From 7122a40f33699ae89521d91249cb893c3037a77c Mon Sep 17 00:00:00 2001 From: qidaijie Date: Wed, 6 Nov 2019 14:23:40 +0800 Subject: [PATCH 01/27] =?UTF-8?q?3.0=20=E5=9F=BA=E7=A1=80=E7=89=88?= =?UTF-8?q?=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 89 ++++++++- properties/core-site.xml | 64 ++++++ properties/hbase-site.xml | 81 ++++++++ properties/hdfs-site.xml | 116 +++++++++++ properties/redis_config.properties | 4 +- properties/service_flow_config.properties | 26 +-- .../cn/ac/iie/bolt/ConnCompletionBolt.java | 23 ++- .../java/cn/ac/iie/bolt/NtcLogSendBolt.java | 2 +- .../iie/bolt/radius/RadiusCompletionBolt.java | 49 +++++ .../cn/ac/iie/common/FlowWriteConfig.java | 2 + .../ac/iie/topology/LogFlowWriteTopology.java | 11 +- .../ac/iie/utils/general/TransFormUtils.java | 35 +++- .../cn/ac/iie/utils/hbase/HbaseUtils.java | 124 ++++++++++++ .../cn/ac/iie/utils/kafka/KafkaLogNtc.java | 4 +- .../cn/ac/iie/utils/redis/RedisPollUtils.java | 16 +- .../cn/ac/iie/utils/system/SnowflakeId.java | 44 +++-- .../iie/utils/zookeeper/DistributedLock.java | 19 +- .../ac/iie/utils/zookeeper/ZooKeeperLock.java | 170 ++++++++++++++++ src/test/java/cn/ac/iie/test/test.java | 35 ++-- .../iie/test/zookeeper/DistributedLock.java | 186 ++++++++++++++++-- .../cn/ac/iie/test/zookeeper/RandomTest.java | 8 + 21 files changed, 1003 insertions(+), 105 deletions(-) create mode 100644 properties/core-site.xml create mode 100644 properties/hbase-site.xml create mode 100644 properties/hdfs-site.xml create mode 100644 src/main/java/cn/ac/iie/bolt/radius/RadiusCompletionBolt.java create mode 100644 src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java create mode 100644 src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java create mode 100644 src/test/java/cn/ac/iie/test/zookeeper/RandomTest.java diff --git a/pom.xml b/pom.xml index 2e03234..59a11e9 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ nexus Team Nexus Repository - http://192.168.10.125:8099/content/groups/public + http://192.168.40.125:8099/content/groups/public @@ -67,6 +67,7 @@ properties **/*.properties + false @@ -84,6 +85,8 @@ UTF-8 1.0.0 1.0.2 + 1.4.9 + 2.7.1 @@ -181,6 +184,90 @@ + + + org.apache.hbase + hbase-client + ${hbase.version} + + + slf4j-log4j12 + org.slf4j + + + log4j-over-slf4j + org.slf4j + + + + + + + org.apache.hbase + hbase-server + ${hbase.version} + + + slf4j-log4j12 + org.slf4j + + + log4j-over-slf4j + org.slf4j + + + + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + slf4j-log4j12 + org.slf4j + + + log4j-over-slf4j + org.slf4j + + + + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + slf4j-log4j12 + org.slf4j + + + log4j-over-slf4j + org.slf4j + + + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + slf4j-log4j12 + org.slf4j + + + log4j-over-slf4j + org.slf4j + + + diff --git a/properties/core-site.xml b/properties/core-site.xml new file mode 100644 index 0000000..66742f0 --- /dev/null +++ b/properties/core-site.xml @@ -0,0 +1,64 @@ + + + + + + + + + fs.defaultFS + hdfs://ns1 + + + hadoop.tmp.dir + file:/opt/hadoop/tmp + + + io.file.buffer.size + 131702 + + + hadoop.proxyuser.root.hosts + * + + + hadoop.proxyuser.root.groups + * + + + hadoop.logfile.size + 10000000 + The max size of each log file + + + + hadoop.logfile.count + 1 + The max number of log files + + + ha.zookeeper.quorum + 192.168.40.202:2181,192.168.40.203:2181,192.168.40.206:2181 + + +io.compression.codecs +com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec + + +io.compression.codec.lzo.class +com.hadoop.compression.lzo.LzoCodec + + + diff --git a/properties/hbase-site.xml b/properties/hbase-site.xml new file mode 100644 index 0000000..b81715c --- /dev/null +++ b/properties/hbase-site.xml @@ -0,0 +1,81 @@ + + + + + + hbase.rootdir + hdfs://ns1/hbase/hbase-1.4.9 + + + hbase.cluster.distributed + true + + + hbase.zookeeper.quorum + 192.168.40.202,192.168.40.203,192.168.40.206 + + +hbase.master.info.port +60010 + + + + phoenix.schema.isNamespaceMappingEnabled + true + + + phoenix.schema.mapSystemTablesToNamespace + true + + + hbase.client.keyvalue.maxsize + 99428800 + + + hbase.server.keyvalue.maxsize + 99428800 + + + hbase.regionserver.wal.codec + org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec + + + phoenix.query.timeoutMs + 1800000 + + + hbase.regionserver.lease.period + 1200000 + + + hbase.rpc.timeout + 1200000 + + + hbase.client.scanner.caching + 1000 + + + hbase.client.scanner.timeout.period + 1200000 + + diff --git a/properties/hdfs-site.xml b/properties/hdfs-site.xml new file mode 100644 index 0000000..5bf742b --- /dev/null +++ b/properties/hdfs-site.xml @@ -0,0 +1,116 @@ + + + + + + + + + dfs.namenode.name.dir + file:/home/bigdata/hadoop/dfs/name + + + dfs.datanode.data.dir + file:/home/bigdata/hadoop/dfs/data + + + dfs.replication + 2 + + + dfs.namenode.secondary.http-address + 192.168.40.202:9001 + + + dfs.webhdfs.enabled + true + + + dfs.permissions + false + + + dfs.permissions.enabled + false + + + dfs.nameservices + ns1 + + + dfs.blocksize + 134217728 + + + dfs.ha.namenodes.ns1 + nn1,nn2 + + + + dfs.namenode.rpc-address.ns1.nn1 + 192.168.40.202:9000 + + + + dfs.namenode.http-address.ns1.nn1 + 192.168.40.202:50070 + + + + dfs.namenode.rpc-address.ns1.nn2 + 192.168.40.203:9000 + + + + dfs.namenode.http-address.ns1.nn2 + 192.168.40.203:50070 + + + + dfs.namenode.shared.edits.dir + qjournal://192.168.40.203:8485;192.168.40.206:8485;192.168.40.202:8485/ns1 + + + + dfs.journalnode.edits.dir + /home/bigdata/hadoop/journal + + + + dfs.client.failover.proxy.provider.ns1 + org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider + + + + dfs.ha.fencing.methods + sshfence + + + + dfs.ha.fencing.ssh.private-key-files + /root/.ssh/id_rsa + + + + dfs.ha.fencing.ssh.connect-timeout + 30000 + + + + dfs.ha.automatic-failover.enabled + true + + + diff --git a/properties/redis_config.properties b/properties/redis_config.properties index f99d396..3c529de 100644 --- a/properties/redis_config.properties +++ b/properties/redis_config.properties @@ -1,8 +1,8 @@ #*****************jedis连接参数设置********************* #redis服务器ip -redis.ip=192.168.40.123 +redis.ip=192.168.40.153 #redis服务器端口号 -redis.port=6379 +redis.port=19000 #与服务器建立连接的超时时间 redis.timeout=3000 #************************jedis池参数设置******************* diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index 3077812..0e64802 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -1,27 +1,29 @@ #管理kafka地址 -#bootstrap.servers=10.4.35.7:9092,10.4.35.8:9092,10.4.35.9:9092 -bootstrap.servers=192.168.6.200:9093,192.168.6.200:9094,192.168.6.200:9095 +bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092 #zookeeper 地址 -zookeeper.servers=192.168.6.200:2181 -#zookeeper.servers=192.168.40.207:2181 +zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 + +#hbase zookeeper地址 +hbase.zookeeper.servers=192.168.40.203:2186 + +#hbase tablename +hbase.table.name=subcriber_info #latest/earliest auto.offset.reset=latest #kafka broker下的topic名称 kafka.topic=SESSION-RECORD-LOG -#kafka.topic=Snowflake-test #读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； -group.id=session-record-log-z +group.id=session-completion-program #输出topic -#results.output.topic=SESSION-TEST-COMPLETED-LOG results.output.topic=SESSION-RECORD-COMPLETED-LOG #storm topology workers -topology.workers=1 +topology.workers=2 #spout并行度 建议与kafka分区数相同 spout.parallelism=3 @@ -33,16 +35,16 @@ datacenter.bolt.parallelism=3 kafka.bolt.parallelism=3 #定位库地址 -ip.library=/dat/ +ip.library=/home/ceiec/topology/dat/ #kafka批量条数 batch.insert.num=5000 #数据中心（UID） -data.center.id.num=15 +data.center.id.num=10 #tick时钟频率 -topology.tick.tuple.freq.secs=5 +topology.tick.tuple.freq.secs=60 #当bolt性能受限时，限制spout接收速度，理论看ack开启才有效 topology.config.max.spout.pending=150000 @@ -66,4 +68,4 @@ influx.ip=http://192.168.40.151:8086 influx.username=admin #influx密码 -influx.password=admin \ No newline at end of file +influx.password=admin diff --git a/src/main/java/cn/ac/iie/bolt/ConnCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/ConnCompletionBolt.java index e67b6cf..109f856 100644 --- a/src/main/java/cn/ac/iie/bolt/ConnCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/ConnCompletionBolt.java @@ -1,5 +1,8 @@ package cn.ac.iie.bolt; +import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.hbase.HbaseUtils; +import cn.ac.iie.utils.system.TupleUtils; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; import org.apache.storm.task.TopologyContext; @@ -10,6 +13,7 @@ import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; +import java.util.HashMap; import java.util.Map; import static cn.ac.iie.utils.general.TransFormUtils.getJsonMessage; @@ -31,15 +35,28 @@ public class ConnCompletionBolt extends BaseBasicBolt { @Override public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { try { - String message = tuple.getString(0); - if (StringUtil.isNotBlank(message)) { - basicOutputCollector.emit(new Values(getJsonMessage(message))); + if (TupleUtils.isTick(tuple)) { +// HbaseUtils.change(); + } else { + String message = tuple.getString(0); + if (StringUtil.isNotBlank(message)) { + basicOutputCollector.emit(new Values(getJsonMessage(message))); + } } + } catch (Exception e) { logger.error("接收解析过程出现异常", e); } } + @Override + public Map getComponentConfiguration() { + Map conf = new HashMap(16); + conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, + FlowWriteConfig.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + return conf; + } + @Override public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { outputFieldsDeclarer.declare(new Fields("connLog")); diff --git a/src/main/java/cn/ac/iie/bolt/NtcLogSendBolt.java b/src/main/java/cn/ac/iie/bolt/NtcLogSendBolt.java index c19acff..e260447 100644 --- a/src/main/java/cn/ac/iie/bolt/NtcLogSendBolt.java +++ b/src/main/java/cn/ac/iie/bolt/NtcLogSendBolt.java @@ -60,7 +60,7 @@ public class NtcLogSendBolt extends BaseBasicBolt { } } } catch (Exception e) { - logger.error("日志发送Kafka过程出现异常 ", e); + logger.error("日志发送Kafka过程出现异常"); e.printStackTrace(); } } diff --git a/src/main/java/cn/ac/iie/bolt/radius/RadiusCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/radius/RadiusCompletionBolt.java new file mode 100644 index 0000000..9b6ebde --- /dev/null +++ b/src/main/java/cn/ac/iie/bolt/radius/RadiusCompletionBolt.java @@ -0,0 +1,49 @@ +package cn.ac.iie.bolt.radius; + +import com.zdjizhi.utils.StringUtil; +import org.apache.log4j.Logger; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.BasicOutputCollector; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseBasicBolt; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; + +import java.util.Map; + +import static cn.ac.iie.utils.general.TransFormUtils.getRadiusMessage; + +/** + * 通联关系日志补全 + * + * @author qidaijie + */ +public class RadiusCompletionBolt extends BaseBasicBolt { + private static final long serialVersionUID = -3657802387129063952L; + private final static Logger logger = Logger.getLogger(RadiusCompletionBolt.class); + + @Override + public void prepare(Map stormConf, TopologyContext context) { + + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { + try { + String message = tuple.getString(0); + if (StringUtil.isNotBlank(message)) { + basicOutputCollector.emit(new Values(getRadiusMessage(message))); + } + } catch (Exception e) { + logger.error("接收解析过程出现异常", e); + } + } + + + @Override + public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { + outputFieldsDeclarer.declare(new Fields("connLog")); + } + +} diff --git a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java index 26e2173..66e124d 100644 --- a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java +++ b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java @@ -40,6 +40,8 @@ public class FlowWriteConfig { */ public static final String BOOTSTRAP_SERVERS = FlowWriteConfigurations.getStringProperty(0, "bootstrap.servers"); public static final String ZOOKEEPER_SERVERS = FlowWriteConfigurations.getStringProperty(0, "zookeeper.servers"); + public static final String HBASE_ZOOKEEPER_SERVERS = FlowWriteConfigurations.getStringProperty(0, "hbase.zookeeper.servers"); + public static final String HBASE_TABLE_NAME = FlowWriteConfigurations.getStringProperty(0, "hbase.table.name"); public static final String GROUP_ID = FlowWriteConfigurations.getStringProperty(0, "group.id"); public static final String RESULTS_OUTPUT_TOPIC = FlowWriteConfigurations.getStringProperty(0, "results.output.topic"); public static final String KAFKA_TOPIC = FlowWriteConfigurations.getStringProperty(0, "kafka.topic"); diff --git a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java index ede06c3..6b677d8 100644 --- a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java +++ b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java @@ -3,7 +3,7 @@ package cn.ac.iie.topology; import cn.ac.iie.bolt.ConnCompletionBolt; import cn.ac.iie.bolt.NtcLogSendBolt; -import cn.ac.iie.bolt.SummaryBolt; +import cn.ac.iie.bolt.radius.RadiusCompletionBolt; import cn.ac.iie.common.FlowWriteConfig; import cn.ac.iie.spout.CustomizedKafkaSpout; import org.apache.log4j.Logger; @@ -58,8 +58,13 @@ public class LogFlowWriteTopology { private void buildTopology() { builder = new TopologyBuilder(); builder.setSpout("LogFlowWriteSpout", new CustomizedKafkaSpout(), FlowWriteConfig.SPOUT_PARALLELISM); - builder.setBolt("ConnCompletionBolt", new ConnCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); - builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ConnCompletionBolt"); + if ("RADIUS-LOG".equals(FlowWriteConfig.KAFKA_TOPIC)) { + builder.setBolt("RadiusCompletionBolt", new RadiusCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt"); + } else { + builder.setBolt("ConnCompletionBolt", new ConnCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ConnCompletionBolt"); + } // builder.setBolt("SummaryBolt", new SummaryBolt(), 1).localOrShuffleGrouping("NtcLogSendBolt"); } diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index 9c0bb8f..3790c35 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -2,6 +2,7 @@ package cn.ac.iie.utils.general; import cn.ac.iie.bean.SessionRecordLog; import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.hbase.HbaseUtils; import cn.ac.iie.utils.redis.RedisPollUtils; import cn.ac.iie.utils.system.SnowflakeId; import cn.ac.iie.utils.zookeeper.DistributedLock; @@ -31,8 +32,6 @@ public class TransFormUtils { .loadAsnDataFileV4(FlowWriteConfig.IP_LIBRARY + "asn_v4.mmdb") .loadAsnDataFileV6(FlowWriteConfig.IP_LIBRARY + "asn_v6.mmdb") .build(); -// private static ZookeeperUtils zookeeperUtils = new ZookeeperUtils(); -// private static SnowflakeId snowflakeId = new SnowflakeId(); /** * 解析日志，并补全 @@ -52,14 +51,40 @@ public class TransFormUtils { sessionRecordLog.setServer_asn(ipLookup.asnLookup(serverIp, true)); sessionRecordLog.setDomain(getTopDomain(sessionRecordLog.getSni(), sessionRecordLog.getHost())); sessionRecordLog.setRecv_time(System.currentTimeMillis() / 1000); -// sessionRecordLog.setSubscribe_id(getSubscribeId(clientIp)); +// sessionRecordLog.setSubscribe_id(HbaseUtils.getAccount(clientIp)); return JSONObject.toJSONString(sessionRecordLog); } catch (Exception e) { - logger.error("日志解析过程出现异常", e); + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); return ""; } } + + /** + * 解析日志，并补全 + * + * @param message radius原始日志 + * @return 补全后的日志 + */ + public static String getRadiusMessage(String message) { + SessionRecordLog sessionRecordLog = JSONObject.parseObject(message, SessionRecordLog.class); + String serverIp = sessionRecordLog.getServer_ip(); + String clientIp = sessionRecordLog.getClient_ip(); + try { + sessionRecordLog.setUid(SnowflakeId.generateId()); + sessionRecordLog.setServer_location(ipLookup.countryLookup(serverIp)); + sessionRecordLog.setClient_location(ipLookup.cityLookupDetail(clientIp)); + sessionRecordLog.setClient_asn(ipLookup.asnLookup(clientIp, true)); + sessionRecordLog.setServer_asn(ipLookup.asnLookup(serverIp, true)); + sessionRecordLog.setRecv_time(System.currentTimeMillis() / 1000); + return JSONObject.toJSONString(sessionRecordLog); + } catch (Exception e) { + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); + return ""; + } + } + + /** * 有sni通过sni获取域名，有hots根据host获取域名 * @@ -67,7 +92,7 @@ public class TransFormUtils { * @param host host * @return 顶级域名 */ - private static String getTopDomain(String sni, String host) { + public static String getTopDomain(String sni, String host) { if (StringUtil.isNotBlank(sni)) { return getDomain(sni); } else if (StringUtil.isNotBlank(host)) { diff --git a/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java new file mode 100644 index 0000000..c7b05e3 --- /dev/null +++ b/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java @@ -0,0 +1,124 @@ +package cn.ac.iie.utils.hbase; + +import cn.ac.iie.common.FlowWriteConfig; +import com.zdjizhi.utils.StringUtil; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.log4j.Logger; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * @author qidaijie + */ +public class HbaseUtils { + private final static Logger logger = Logger.getLogger(HbaseUtils.class); + private static Map subIdMap = new HashMap<>(16); + private static Connection connection; + private static Long time; + +// static { +// // 管理Hbase的配置信息 +// Configuration configuration = HBaseConfiguration.create(); +// // 设置zookeeper节点 +// configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); +// try { +// connection = ConnectionFactory.createConnection(configuration); +// time = System.currentTimeMillis(); +// getAll(); +// } catch (IOException e) { +// logger.error("获取HBase连接失败"); +// e.printStackTrace(); +// } +// } + + /** + * 更新变量 + */ + public static void change() { + Long nowTime = System.currentTimeMillis(); + timestampsFilter(time - 1500, nowTime + 500); + } + + /** + * 获取变更内容 + * + * @param startTime 开始时间 + * @param endTime 结束时间 + */ + private static void timestampsFilter(Long startTime, Long endTime) { + Table table = null; + TableName tableName = TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME); + Admin admin = null; + try { + table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); + Scan scan2 = new Scan(); + scan2.setTimeRange(startTime, endTime); + ResultScanner scanner = table.getScanner(scan2); + for (Result result : scanner) { + Cell[] cells = result.rawCells(); + for (Cell cell : cells) { + subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); + } + } + admin = connection.getAdmin(); + admin.flush(tableName); + logger.warn("当前集合长度" + subIdMap.keySet().size()); + logger.warn("更新后集合keys：" + subIdMap.keySet()); + time = endTime; + scanner.close(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (table != null) { + try { + table.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + /** + * 获取所有的 key value + */ + private static void getAll() { + try { + Table table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); + Scan scan2 = new Scan(); + ResultScanner scanner = table.getScanner(scan2); + for (Result result : scanner) { + Cell[] cells = result.rawCells(); + for (Cell cell : cells) { + subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); + } + } + logger.warn("获取全量后集合keys：" + subIdMap.keySet()); + scanner.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * 获取 account + * + * @param ip + * @return + */ + public static String getAccount(String ip) { + if (StringUtil.isNotBlank(ip)) { + return subIdMap.get(ip); + } else { + return ""; + } + } +} diff --git a/src/main/java/cn/ac/iie/utils/kafka/KafkaLogNtc.java b/src/main/java/cn/ac/iie/utils/kafka/KafkaLogNtc.java index 11ae57a..1bf76eb 100644 --- a/src/main/java/cn/ac/iie/utils/kafka/KafkaLogNtc.java +++ b/src/main/java/cn/ac/iie/utils/kafka/KafkaLogNtc.java @@ -59,7 +59,7 @@ public class KafkaLogNtc { } } kafkaProducer.flush(); - logger.warn("Log sent to National Center successfully!!!!!"); + logger.debug("Log sent to National Center successfully!!!!!"); } /** @@ -75,7 +75,7 @@ public class KafkaLogNtc { properties.put("request.timeout.ms", 60000); properties.put("batch.size", 262144); properties.put("buffer.memory", 33554432); - properties.put("compression.type", "snappy"); +// properties.put("compression.type", "snappy"); kafkaProducer = new KafkaProducer<>(properties); } diff --git a/src/main/java/cn/ac/iie/utils/redis/RedisPollUtils.java b/src/main/java/cn/ac/iie/utils/redis/RedisPollUtils.java index 378bef5..0394378 100644 --- a/src/main/java/cn/ac/iie/utils/redis/RedisPollUtils.java +++ b/src/main/java/cn/ac/iie/utils/redis/RedisPollUtils.java @@ -43,7 +43,7 @@ public class RedisPollUtils { poolConfig.setTestOnBorrow(Boolean.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_TESTONBORROW))); // 根据配置实例化jedis池 jedisPool = new JedisPool(poolConfig, props.getProperty(FlowWriteConfig.REDIS_IP), - Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_PORT))); + Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_PORT)), 3000, "123456"); } catch (Exception e) { logger.error("Redis连接池初始化错误", e); } @@ -96,19 +96,15 @@ public class RedisPollUtils { // return workId; // } - public static Integer getWorkerId(String key) { - int workId = 0; + public static String getWorkerId(String key) { + String sub = ""; try (Jedis jedis = RedisPollUtils.getJedis()) { - if (jedis != null) { - workId = Integer.parseInt(jedis.get(key)); - jedis.set(key, String.valueOf(workId + 2)); - logger.error("\n工作id是：" + workId + "\n"); - } + sub = jedis.get(key); } catch (Exception e) { logger.error("通过Redis获取用户名出现异常", e); - workId = RandomUtils.nextInt(0, 31); + } - return workId; + return sub; } diff --git a/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java b/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java index 5f77996..5c3462d 100644 --- a/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java +++ b/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java @@ -2,6 +2,7 @@ package cn.ac.iie.utils.system; import cn.ac.iie.common.FlowWriteConfig; import cn.ac.iie.utils.zookeeper.DistributedLock; +import cn.ac.iie.utils.zookeeper.ZooKeeperLock; import cn.ac.iie.utils.zookeeper.ZookeeperUtils; import org.apache.log4j.Logger; @@ -98,19 +99,40 @@ public class SnowflakeId { /** * 构造函数 */ +// private SnowflakeId() { +// DistributedLock lock = new DistributedLock(FlowWriteConfig.ZOOKEEPER_SERVERS, "disLocks1"); +// lock.lock(); +// int tmpWorkerId = zookeeperUtils.modifyNode("/Snowflake/" + FlowWriteConfig.KAFKA_TOPIC); +// if (tmpWorkerId > maxWorkerId || tmpWorkerId < 0) { +// throw new IllegalArgumentException(String.format("worker Id can't be greater than %d or less than 0", maxWorkerId)); +// } +// int dataCenterId = FlowWriteConfig.DATA_CENTER_ID_NUM; +// if (dataCenterId > maxDataCenterId || dataCenterId < 0) { +// throw new IllegalArgumentException(String.format("datacenter Id can't be greater than %d or less than 0", maxDataCenterId)); +// } +// this.workerId = tmpWorkerId; +// this.dataCenterId = dataCenterId; +// } + private SnowflakeId() { - DistributedLock lock = new DistributedLock(FlowWriteConfig.ZOOKEEPER_SERVERS, "disLocks1"); - lock.lock(); - int tmpWorkerId = zookeeperUtils.modifyNode("/Snowflake/" + FlowWriteConfig.KAFKA_TOPIC); - if (tmpWorkerId > maxWorkerId || tmpWorkerId < 0) { - throw new IllegalArgumentException(String.format("worker Id can't be greater than %d or less than 0", maxWorkerId)); + ZooKeeperLock lock = new ZooKeeperLock(FlowWriteConfig.ZOOKEEPER_SERVERS, "/locks", "disLocks"); + if (lock.lock()) { + int tmpWorkerId = zookeeperUtils.modifyNode("/Snowflake/" + FlowWriteConfig.KAFKA_TOPIC); + if (tmpWorkerId > maxWorkerId || tmpWorkerId < 0) { + throw new IllegalArgumentException(String.format("worker Id can't be greater than %d or less than 0", maxWorkerId)); + } + int dataCenterId = FlowWriteConfig.DATA_CENTER_ID_NUM; + if (dataCenterId > maxDataCenterId || dataCenterId < 0) { + throw new IllegalArgumentException(String.format("datacenter Id can't be greater than %d or less than 0", maxDataCenterId)); + } + this.workerId = tmpWorkerId; + this.dataCenterId = dataCenterId; + try { + lock.unlock(); + } catch (InterruptedException e) { + e.printStackTrace(); + } } - int dataCenterId = FlowWriteConfig.DATA_CENTER_ID_NUM; - if (dataCenterId > maxDataCenterId || dataCenterId < 0) { - throw new IllegalArgumentException(String.format("datacenter Id can't be greater than %d or less than 0", maxDataCenterId)); - } - this.workerId = tmpWorkerId; - this.dataCenterId = dataCenterId; } // ==============================Methods========================================== diff --git a/src/main/java/cn/ac/iie/utils/zookeeper/DistributedLock.java b/src/main/java/cn/ac/iie/utils/zookeeper/DistributedLock.java index 15f4506..f85a84e 100644 --- a/src/main/java/cn/ac/iie/utils/zookeeper/DistributedLock.java +++ b/src/main/java/cn/ac/iie/utils/zookeeper/DistributedLock.java @@ -190,27 +190,14 @@ public class DistributedLock implements Lock, Watcher { public static void main(String[] args) { ZookeeperUtils zookeeperUtils = new ZookeeperUtils(); - Runnable runnable = new Runnable() { @Override public void run() { - DistributedLock lock = null; - try { - lock = new DistributedLock(FlowWriteConfig.ZOOKEEPER_SERVERS, "disLocks1"); - lock.lock(); -// System.out.println(SnowflakeId.generateId()); - System.out.println(1); - Thread.sleep(3000); - } catch (InterruptedException e) { - e.printStackTrace(); - } finally { - if (lock != null) { - lock.unlock(); - } - } + DistributedLock lock = new DistributedLock(FlowWriteConfig.ZOOKEEPER_SERVERS, "disLocks1"); + lock.lock(); + lock.unlock(); } }; - for (int i = 0; i < 10; i++) { Thread t = new Thread(runnable); t.start(); diff --git a/src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java b/src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java new file mode 100644 index 0000000..6cff343 --- /dev/null +++ b/src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java @@ -0,0 +1,170 @@ +package cn.ac.iie.utils.zookeeper; + +import org.apache.zookeeper.*; +import org.apache.zookeeper.data.Stat; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class ZooKeeperLock implements Watcher { + private ZooKeeper zk = null; + private String rootLockNode; // 锁的根节点 + private String lockName; // 竞争资源，用来生成子节点名称 + private String currentLock; // 当前锁 + private String waitLock; // 等待的锁（前一个锁） + private CountDownLatch countDownLatch; // 计数器（用来在加锁失败时阻塞加锁线程） + private int sessionTimeout = 30000; // 超时时间 + + // 1. 构造器中创建ZK链接，创建锁的根节点 + public ZooKeeperLock(String zkAddress, String rootLockNode, String lockName) { + this.rootLockNode = rootLockNode; + this.lockName = lockName; + try { + // 创建连接，zkAddress格式为：IP:PORT + zk = new ZooKeeper(zkAddress, this.sessionTimeout, this); + // 检测锁的根节点是否存在，不存在则创建 + Stat stat = zk.exists(rootLockNode, false); + if (null == stat) { + zk.create(rootLockNode, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } + } catch (IOException | InterruptedException | KeeperException e) { + e.printStackTrace(); + } + } + + // 2. 加锁方法，先尝试加锁，不能加锁则等待上一个锁的释放 + public boolean lock() { + if (this.tryLock()) { + System.out.println("线程【" + Thread.currentThread().getName() + "】加锁（" + this.currentLock + "）成功！"); + return true; + } else { + return waitOtherLock(this.waitLock, this.sessionTimeout); + } + } + + public boolean tryLock() { + // 分隔符 + String split = "_lock_"; + if (this.lockName.contains("_lock_")) { + throw new RuntimeException("lockName can't contains '_lock_' "); + } + try { + // 创建锁节点（临时有序节点） + this.currentLock = zk.create(this.rootLockNode + "/" + this.lockName + split, new byte[0], + ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL); + System.out.println("线程【" + Thread.currentThread().getName() + + "】创建锁节点（" + this.currentLock + "）成功，开始竞争..."); + // 取所有子节点 + List nodes = zk.getChildren(this.rootLockNode, false); + // 取所有竞争lockName的锁 + List lockNodes = new ArrayList(); + for (String nodeName : nodes) { + if (nodeName.split(split)[0].equals(this.lockName)) { + lockNodes.add(nodeName); + } + } + Collections.sort(lockNodes); + // 取最小节点与当前锁节点比对加锁 + String currentLockPath = this.rootLockNode + "/" + lockNodes.get(0); + if (this.currentLock.equals(currentLockPath)) { + return true; + } + // 加锁失败，设置前一节点为等待锁节点 + String currentLockNode = this.currentLock.substring(this.currentLock.lastIndexOf("/") + 1); + int preNodeIndex = Collections.binarySearch(lockNodes, currentLockNode) - 1; + this.waitLock = lockNodes.get(preNodeIndex); + } catch (KeeperException | InterruptedException e) { + e.printStackTrace(); + } + return false; + } + + private boolean waitOtherLock(String waitLock, int sessionTimeout) { + boolean islock = false; + try { + // 监听等待锁节点 + String waitLockNode = this.rootLockNode + "/" + waitLock; + Stat stat = zk.exists(waitLockNode, true); + if (null != stat) { + System.out.println("线程【" + Thread.currentThread().getName() + + "】锁（" + this.currentLock + "）加锁失败，等待锁（" + waitLockNode + "）释放..."); + // 设置计数器，使用计数器阻塞线程 + this.countDownLatch = new CountDownLatch(1); + islock = this.countDownLatch.await(sessionTimeout, TimeUnit.MILLISECONDS); + this.countDownLatch = null; + if (islock) { + System.out.println("线程【" + Thread.currentThread().getName() + "】锁（" + + this.currentLock + "）加锁成功，锁（" + waitLockNode + "）已经释放"); + } else { + System.out.println("线程【" + Thread.currentThread().getName() + "】锁（" + + this.currentLock + "）加锁失败..."); + } + } else { + islock = true; + } + } catch (KeeperException | InterruptedException e) { + e.printStackTrace(); + } + return islock; + } + + // 3. 释放锁 + public void unlock() throws InterruptedException { + try { + Stat stat = zk.exists(this.currentLock, false); + if (null != stat) { + System.out.println("线程【" + Thread.currentThread().getName() + "】释放锁 " + this.currentLock); + zk.delete(this.currentLock, -1); + this.currentLock = null; + } + } catch (InterruptedException | KeeperException e) { + e.printStackTrace(); + } finally { + zk.close(); + } + } + + // 4. 监听器回调 + @Override + public void process(WatchedEvent watchedEvent) { + if (null != this.countDownLatch && watchedEvent.getType() == Event.EventType.NodeDeleted) { + // 计数器减一，恢复线程操作 + this.countDownLatch.countDown(); + } + } + + + public static void doSomething() { + + } + + public static void main(String[] args) { + Runnable runnable = new Runnable() { + @Override + public void run() { + ZooKeeperLock lock = null; + lock = new ZooKeeperLock("192.168.40.119:2181", "/locks", "test1"); + if (lock.lock()) { + doSomething(); + try { +// Thread.sleep(1000); + lock.unlock(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + }; + + for (int i = 0; i < 10; i++) { + Thread t = new Thread(runnable); + t.start(); + } + } + + +} diff --git a/src/test/java/cn/ac/iie/test/test.java b/src/test/java/cn/ac/iie/test/test.java index 9d2d332..705a0f7 100644 --- a/src/test/java/cn/ac/iie/test/test.java +++ b/src/test/java/cn/ac/iie/test/test.java @@ -4,43 +4,30 @@ import cn.ac.iie.bean.SessionRecordLog; import cn.ac.iie.common.FlowWriteConfig; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.serializer.SerializerFeature; import com.zdjizhi.utils.IpLookup; import org.junit.Test; import javax.servlet.http.HttpServletRequest; import java.math.BigInteger; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; public class test { public static void main(String[] args) { - String message = "{\"str_ea_m-t-r-a-ceid\":\"JSON\"}"; + String message = "{\"str_ea_m-t-r-a-ceid\":\"JSON\",\"uid\":\"0\"}"; SessionRecordLog sessionRecordLog = JSONObject.parseObject(message, SessionRecordLog.class); - System.out.println(sessionRecordLog.getStream_trace_id()); - - String message2 = "{\"streamtraceid\":\"JSON\"}"; - SessionRecordLog sessionRecordLog2 = JSONObject.parseObject(message2, SessionRecordLog.class); - System.out.println(sessionRecordLog2.getStream_trace_id()); - - JSONObject jsonObject = JSON.parseObject(message); - System.out.println("\n" + Arrays.toString(jsonObject.keySet().toArray())); - - HttpServletRequest request = null; - if (request != null) { - String contextPath = request.getScheme() + "://" + request.getServerName() + ":" + request.getServerPort() + request.getContextPath(); - } - - System.out.println(System.currentTimeMillis() / 1000); + System.out.println(JSONObject.toJSONString(sessionRecordLog)); } @Test public void test2() { -// String minTimeStampStr = "00000000000000000000000000000000000000000"; - String minTimeStampStr = "000000000000000000000000000000000000000"; - long minTimeStamp = new BigInteger(minTimeStampStr, 2).longValue(); -// String maxTimeStampStr = "11111111111111111111111111111111111111111"; - String maxTimeStampStr = "111111111111111111111111111111111111111"; - long maxTimeStamp = new BigInteger(maxTimeStampStr, 2).longValue(); - long oneYearMills = 1L * 1000 * 60 * 60 * 24 * 365; - System.out.println((maxTimeStamp - minTimeStamp) / oneYearMills); + Map map = new HashMap<>(); + map.put("a","a"); + map.put("b","a"); + map.put("c","a"); + + System.out.println(map.keySet()); } } diff --git a/src/test/java/cn/ac/iie/test/zookeeper/DistributedLock.java b/src/test/java/cn/ac/iie/test/zookeeper/DistributedLock.java index f442ebe..1423f98 100644 --- a/src/test/java/cn/ac/iie/test/zookeeper/DistributedLock.java +++ b/src/test/java/cn/ac/iie/test/zookeeper/DistributedLock.java @@ -1,26 +1,182 @@ package cn.ac.iie.test.zookeeper; +import cn.ac.iie.common.FlowWriteConfig; +import org.apache.zookeeper.*; +import org.apache.zookeeper.data.Stat; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; -public interface DistributedLock { - /** - * 获取锁，如果没有得到就等待 - */ - public void acquire() throws Exception; +public class DistributedLock implements Lock, Watcher { + private ZooKeeper zk; + private String root = "/locks";//根 + private String lockName;//竞争资源的标志 + private String waitNode;//等待前一个锁 + private String myZnode;//当前锁 + private CountDownLatch latch;//计数器 + private CountDownLatch connectedSignal = new CountDownLatch(1); + private int sessionTimeout = 2000; + public static void main(String[] args) { + DistributedLock lock = new DistributedLock(FlowWriteConfig.ZOOKEEPER_SERVERS, "disLocks1"); + lock.lock(); + System.out.println(1); + if (lock!=null){ + lock.unlock(); + } + } /** - * 获取锁，直到超时 + * 创建分布式锁,使用前请确认config配置的zookeeper服务可用 * - * @param unit time参数的单位 - * @throws Exception - * @return是否获取到锁 + * @param config 192.168.1.127:2181 + * @param lockName 竞争资源标志,lockName中不能包含单词_lock_ */ - public boolean acquire(long time, TimeUnit unit) throws Exception; + public DistributedLock(String config, String lockName) { + this.lockName = lockName; + // 创建一个与服务器的连接 + try { + zk = new ZooKeeper(config, sessionTimeout, this); + connectedSignal.await(); + Stat stat = zk.exists(root, false);//此去不执行 Watcher + if (stat == null) { + // 创建根节点 + zk.create(root, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } + } catch (IOException e) { + throw new LockException(e); + } catch (KeeperException e) { + throw new LockException(e); + } catch (InterruptedException e) { + throw new LockException(e); + } + } /** - * 释放锁 - * - * @throws Exception + * zookeeper节点的监视器 */ - public void release() throws Exception; -} + public void process(WatchedEvent event) { + //建立连接用 + if (event.getState() == Event.KeeperState.SyncConnected) { + connectedSignal.countDown(); + return; + } + //其他线程放弃锁的标志 + if (this.latch != null) { + this.latch.countDown(); + } + } + + public void lock() { + try { + if (this.tryLock()) { + System.out.println("Thread " + Thread.currentThread().getId() + " " + myZnode + " get lock true"); + return; + } else { + waitForLock(waitNode, sessionTimeout);//等待锁 + } + } catch (KeeperException e) { + throw new LockException(e); + } catch (InterruptedException e) { + throw new LockException(e); + } + } + + public boolean tryLock() { + try { + String splitStr = "_lock_"; + if (lockName.contains(splitStr)) + throw new LockException("lockName can not contains \\u000B"); + //创建临时子节点 + myZnode = zk.create(root + "/" + lockName + splitStr, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL); + System.out.println(myZnode + " is created "); + //取出所有子节点 + List subNodes = zk.getChildren(root, false); + //取出所有lockName的锁 + List lockObjNodes = new ArrayList(); + for (String node : subNodes) { + String _node = node.split(splitStr)[0]; + if (_node.equals(lockName)) { + lockObjNodes.add(node); + } + } + Collections.sort(lockObjNodes); + + if (myZnode.equals(root + "/" + lockObjNodes.get(0))) { + //如果是最小的节点,则表示取得锁 + System.out.println(myZnode + "==" + lockObjNodes.get(0)); + return true; + } + //如果不是最小的节点，找到比自己小1的节点 + String subMyZnode = myZnode.substring(myZnode.lastIndexOf("/") + 1); + waitNode = lockObjNodes.get(Collections.binarySearch(lockObjNodes, subMyZnode) - 1);//找到前一个子节点 + } catch (KeeperException e) { + throw new LockException(e); + } catch (InterruptedException e) { + throw new LockException(e); + } + return false; + } + + public boolean tryLock(long time, TimeUnit unit) { + try { + if (this.tryLock()) { + return true; + } + return waitForLock(waitNode, time); + } catch (Exception e) { + e.printStackTrace(); + } + return false; + } + + private boolean waitForLock(String lower, long waitTime) throws InterruptedException, KeeperException { + Stat stat = zk.exists(root + "/" + lower, true);//同时注册监听。 + //判断比自己小一个数的节点是否存在,如果不存在则无需等待锁,同时注册监听 + if (stat != null) { + System.out.println("Thread " + Thread.currentThread().getId() + " waiting for " + root + "/" + lower); + this.latch = new CountDownLatch(1); + this.latch.await(waitTime, TimeUnit.MILLISECONDS);//等待，这里应该一直等待其他线程释放锁 + this.latch = null; + } + return true; + } + + public void unlock() { + try { + System.out.println("unlock " + myZnode); + zk.delete(myZnode, -1); + myZnode = null; + zk.close(); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (KeeperException e) { + e.printStackTrace(); + } + } + + public void lockInterruptibly() throws InterruptedException { + this.lock(); + } + + public Condition newCondition() { + return null; + } + + public class LockException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public LockException(String e) { + super(e); + } + + public LockException(Exception e) { + super(e); + } + } +} \ No newline at end of file diff --git a/src/test/java/cn/ac/iie/test/zookeeper/RandomTest.java b/src/test/java/cn/ac/iie/test/zookeeper/RandomTest.java new file mode 100644 index 0000000..3730073 --- /dev/null +++ b/src/test/java/cn/ac/iie/test/zookeeper/RandomTest.java @@ -0,0 +1,8 @@ +package cn.ac.iie.test.zookeeper; + +public class RandomTest { + public static void main(String[] args) { + + + } +} From 11a389c14f3bb039b8b2dd26ca09281b72198d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Tue, 12 Nov 2019 11:29:19 +0800 Subject: [PATCH 02/27] =?UTF-8?q?radius=E6=97=A5=E5=BF=97=E8=A1=A5?= =?UTF-8?q?=E5=85=A8=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 8 +- properties/hbase-site.xml | 36 +- properties/hdfs-site.xml | 12 +- properties/service_flow_config.properties | 24 +- .../ac/iie/bean/CollectProtocolRecordLog.java | 677 ++++++++++++++++++ .../cn/ac/iie/bean/ProxySessionRecordLog.java | 198 +++++ .../ac/iie/bean/PublicSessionRecordLog.java | 380 ++++++++++ .../ac/iie/bean/RadiusSessionRecordLog.java | 84 +++ .../cn/ac/iie/bean/SecurityPolicyLog.java | 677 ++++++++++++++++++ .../java/cn/ac/iie/bean/SessionRecordLog.java | 672 ----------------- .../CollectCompletedBolt.java} | 21 +- .../iie/bolt/proxy/ProxyCompletionBolt.java | 65 ++ .../bolt/security/SecurityCompletionBolt.java | 65 ++ .../ac/iie/topology/LogFlowWriteTopology.java | 52 +- .../ac/iie/utils/general/TransFormUtils.java | 165 +++-- .../cn/ac/iie/utils/hbase/HbaseUtils.java | 29 +- .../ac/iie/utils/redis/RedisClusterUtils.java | 79 -- .../cn/ac/iie/utils/redis/RedisPollUtils.java | 111 --- 18 files changed, 2340 insertions(+), 1015 deletions(-) create mode 100644 src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java create mode 100644 src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java create mode 100644 src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java create mode 100644 src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java create mode 100644 src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java delete mode 100644 src/main/java/cn/ac/iie/bean/SessionRecordLog.java rename src/main/java/cn/ac/iie/bolt/{ConnCompletionBolt.java => collectProtocol/CollectCompletedBolt.java} (76%) create mode 100644 src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java create mode 100644 src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java delete mode 100644 src/main/java/cn/ac/iie/utils/redis/RedisClusterUtils.java delete mode 100644 src/main/java/cn/ac/iie/utils/redis/RedisPollUtils.java diff --git a/pom.xml b/pom.xml index 59a11e9..88624d0 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ properties **/*.properties - + **/*.xml false @@ -132,12 +132,6 @@ ${storm.version} - - redis.clients - jedis - 2.8.1 - - junit junit diff --git a/properties/hbase-site.xml b/properties/hbase-site.xml index b81715c..d26f7df 100644 --- a/properties/hbase-site.xml +++ b/properties/hbase-site.xml @@ -31,13 +31,13 @@ hbase.zookeeper.quorum - 192.168.40.202,192.168.40.203,192.168.40.206 + master,slave1,slave2 hbase.master.info.port 60010 - +#开启schema支持 对应hbase的namespace phoenix.schema.isNamespaceMappingEnabled true @@ -46,36 +46,4 @@ phoenix.schema.mapSystemTablesToNamespace true - - hbase.client.keyvalue.maxsize - 99428800 - - - hbase.server.keyvalue.maxsize - 99428800 - - - hbase.regionserver.wal.codec - org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec - - - phoenix.query.timeoutMs - 1800000 - - - hbase.regionserver.lease.period - 1200000 - - - hbase.rpc.timeout - 1200000 - - - hbase.client.scanner.caching - 1000 - - - hbase.client.scanner.timeout.period - 1200000 - diff --git a/properties/hdfs-site.xml b/properties/hdfs-site.xml index 5bf742b..845b195 100644 --- a/properties/hdfs-site.xml +++ b/properties/hdfs-site.xml @@ -31,7 +31,7 @@ dfs.namenode.secondary.http-address - 192.168.40.202:9001 + master:9001 dfs.webhdfs.enabled @@ -60,27 +60,27 @@ dfs.namenode.rpc-address.ns1.nn1 - 192.168.40.202:9000 + master:9000 dfs.namenode.http-address.ns1.nn1 - 192.168.40.202:50070 + master:50070 dfs.namenode.rpc-address.ns1.nn2 - 192.168.40.203:9000 + slave1:9000 dfs.namenode.http-address.ns1.nn2 - 192.168.40.203:50070 + slave1:50070 dfs.namenode.shared.edits.dir - qjournal://192.168.40.203:8485;192.168.40.206:8485;192.168.40.202:8485/ns1 + qjournal://slave1:8485;slave2:8485;master:8485/ns1 diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index 0e64802..e07af38 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -1,11 +1,11 @@ #管理kafka地址 -bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092 +bootstrap.servers=192.168.40.152:9092 #zookeeper 地址 -zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 +zookeeper.servers=192.168.40.152:2181 #hbase zookeeper地址 -hbase.zookeeper.servers=192.168.40.203:2186 +hbase.zookeeper.servers=192.168.40.203:2181,192.168.40.206:2181 #hbase tablename hbase.table.name=subcriber_info @@ -14,29 +14,29 @@ hbase.table.name=subcriber_info auto.offset.reset=latest #kafka broker下的topic名称 -kafka.topic=SESSION-RECORD-LOG +kafka.topic=SECURITY-POLICY-LOG #读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； group.id=session-completion-program #输出topic -results.output.topic=SESSION-RECORD-COMPLETED-LOG +results.output.topic=SECURITY-POLICY-COMPLETED-LOG #storm topology workers -topology.workers=2 +topology.workers=1 #spout并行度 建议与kafka分区数相同 -spout.parallelism=3 +spout.parallelism=1 #处理补全操作的bolt并行度-worker的倍数 -datacenter.bolt.parallelism=3 +datacenter.bolt.parallelism=1 -#写入kafkad的并行度 -kafka.bolt.parallelism=3 +#写入kafka的并行度 +kafka.bolt.parallelism=1 #定位库地址 -ip.library=/home/ceiec/topology/dat/ - +#ip.library=/home/ceiec/topology/dat/ +ip.library=D:/dat/ #kafka批量条数 batch.insert.num=5000 diff --git a/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java b/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java new file mode 100644 index 0000000..6131846 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java @@ -0,0 +1,677 @@ +package cn.ac.iie.bean; + +public class CollectProtocolRecordLog extends PublicSessionRecordLog{ + + +//TODO HTTP协议属性 21 + + private String http_url; + private String http_host; + private String http_domain; + private String http_request_line; + private String http_response_line; + private String http_request_header; + private String http_response_header; + private String http_request_body; + private String http_response_body; + private String http_request_body_key; + private String http_response_body_key; + private int http_proxy_flag; + private int http_sequence; + private String http_snapshot; + private String http_cookie; + private String http_referer; + private String http_user_agent; + private String http_content_length; + private String http_content_type; + private String http_set_cookie; + private String http_version; + +//TODO MAIL协议属性 9 + + private String mail_protocol_type; + private String mail_sender; + private String mail_receiver; + private String mail_subject; + private String mail_content; + private String mail_attachment_name; + private String mail_attachment_content; + private String mail_eml_file; + private String mail_snapshot; + +//TODO 3DNS协议属性 18 + + private int dns_message_id; + private int dns_qr; + private int dns_opcode; + private int dns_aa; + private int dns_tc; + private int dns_rd; + private int dns_ra; + private int dns_rcode; + private int dns_qdcount; + private int dns_ancount; + private int dns_nscount; + private int dns_arcount; + private String dns_qname; + private int dns_qtype; + private int dns_qclass; + private String dns_cname; + private int dns_sub; + private String dns_rr; + +//TODO SSL协议属性 13 + + + private int ssl_pinningst; + private int ssl_intercept_state; + private int ssl_server_side_latency; + private int ssl_client_side_latency; + private int ssl_cert_verify; + private int ssl_con_latency_ms; + private String ssl_version; + private String ssl_sni; + private String ssl_san; + private String ssl_cn; + private String ssl_server_side_version; + private String ssl_client_side_version; + private String ssl_error; + +//TODO FTP协议属性 2 + private String ftp_url; + private String ftp_content; + + +//TODO BGP协议属性 3 + private int bgp_type; + private String bgp_as_num; + private String bgp_route; + +//TODO VOIP协议属性 4 + private String voip_calling_account; + private String voip_called_account; + private String voip_calling_number; + private String voip_called_number; + +//TODO STREAMING_MEDIA协议属性 2 + private String streaming_media_url; + private String streaming_media_protocol; + + + public String getHttp_url() { + return http_url; + } + + public void setHttp_url(String http_url) { + this.http_url = http_url; + } + + public String getHttp_host() { + return http_host; + } + + public void setHttp_host(String http_host) { + this.http_host = http_host; + } + + public String getHttp_domain() { + return http_domain; + } + + public void setHttp_domain(String http_domain) { + this.http_domain = http_domain; + } + + public String getHttp_request_line() { + return http_request_line; + } + + public void setHttp_request_line(String http_request_line) { + this.http_request_line = http_request_line; + } + + public String getHttp_response_line() { + return http_response_line; + } + + public void setHttp_response_line(String http_response_line) { + this.http_response_line = http_response_line; + } + + public String getHttp_request_header() { + return http_request_header; + } + + public void setHttp_request_header(String http_request_header) { + this.http_request_header = http_request_header; + } + + public String getHttp_response_header() { + return http_response_header; + } + + public void setHttp_response_header(String http_response_header) { + this.http_response_header = http_response_header; + } + + public String getHttp_request_body() { + return http_request_body; + } + + public void setHttp_request_body(String http_request_body) { + this.http_request_body = http_request_body; + } + + public String getHttp_response_body() { + return http_response_body; + } + + public void setHttp_response_body(String http_response_body) { + this.http_response_body = http_response_body; + } + + public String getHttp_request_body_key() { + return http_request_body_key; + } + + public void setHttp_request_body_key(String http_request_body_key) { + this.http_request_body_key = http_request_body_key; + } + + public String getHttp_response_body_key() { + return http_response_body_key; + } + + public void setHttp_response_body_key(String http_response_body_key) { + this.http_response_body_key = http_response_body_key; + } + + public int getHttp_proxy_flag() { + return http_proxy_flag; + } + + public void setHttp_proxy_flag(int http_proxy_flag) { + this.http_proxy_flag = http_proxy_flag; + } + + public int getHttp_sequence() { + return http_sequence; + } + + public void setHttp_sequence(int http_sequence) { + this.http_sequence = http_sequence; + } + + public String getHttp_snapshot() { + return http_snapshot; + } + + public void setHttp_snapshot(String http_snapshot) { + this.http_snapshot = http_snapshot; + } + + public String getHttp_cookie() { + return http_cookie; + } + + public void setHttp_cookie(String http_cookie) { + this.http_cookie = http_cookie; + } + + public String getHttp_referer() { + return http_referer; + } + + public void setHttp_referer(String http_referer) { + this.http_referer = http_referer; + } + + public String getHttp_user_agent() { + return http_user_agent; + } + + public void setHttp_user_agent(String http_user_agent) { + this.http_user_agent = http_user_agent; + } + + public String getHttp_content_length() { + return http_content_length; + } + + public void setHttp_content_length(String http_content_length) { + this.http_content_length = http_content_length; + } + + public String getHttp_content_type() { + return http_content_type; + } + + public void setHttp_content_type(String http_content_type) { + this.http_content_type = http_content_type; + } + + public String getHttp_set_cookie() { + return http_set_cookie; + } + + public void setHttp_set_cookie(String http_set_cookie) { + this.http_set_cookie = http_set_cookie; + } + + public String getHttp_version() { + return http_version; + } + + public void setHttp_version(String http_version) { + this.http_version = http_version; + } + + public String getMail_protocol_type() { + return mail_protocol_type; + } + + public void setMail_protocol_type(String mail_protocol_type) { + this.mail_protocol_type = mail_protocol_type; + } + + public String getMail_sender() { + return mail_sender; + } + + public void setMail_sender(String mail_sender) { + this.mail_sender = mail_sender; + } + + public String getMail_receiver() { + return mail_receiver; + } + + public void setMail_receiver(String mail_receiver) { + this.mail_receiver = mail_receiver; + } + + public String getMail_subject() { + return mail_subject; + } + + public void setMail_subject(String mail_subject) { + this.mail_subject = mail_subject; + } + + public String getMail_content() { + return mail_content; + } + + public void setMail_content(String mail_content) { + this.mail_content = mail_content; + } + + public String getMail_attachment_name() { + return mail_attachment_name; + } + + public void setMail_attachment_name(String mail_attachment_name) { + this.mail_attachment_name = mail_attachment_name; + } + + public String getMail_attachment_content() { + return mail_attachment_content; + } + + public void setMail_attachment_content(String mail_attachment_content) { + this.mail_attachment_content = mail_attachment_content; + } + + public String getMail_eml_file() { + return mail_eml_file; + } + + public void setMail_eml_file(String mail_eml_file) { + this.mail_eml_file = mail_eml_file; + } + + public String getMail_snapshot() { + return mail_snapshot; + } + + public void setMail_snapshot(String mail_snapshot) { + this.mail_snapshot = mail_snapshot; + } + + public int getDns_message_id() { + return dns_message_id; + } + + public void setDns_message_id(int dns_message_id) { + this.dns_message_id = dns_message_id; + } + + public int getDns_qr() { + return dns_qr; + } + + public void setDns_qr(int dns_qr) { + this.dns_qr = dns_qr; + } + + public int getDns_opcode() { + return dns_opcode; + } + + public void setDns_opcode(int dns_opcode) { + this.dns_opcode = dns_opcode; + } + + public int getDns_aa() { + return dns_aa; + } + + public void setDns_aa(int dns_aa) { + this.dns_aa = dns_aa; + } + + public int getDns_tc() { + return dns_tc; + } + + public void setDns_tc(int dns_tc) { + this.dns_tc = dns_tc; + } + + public int getDns_rd() { + return dns_rd; + } + + public void setDns_rd(int dns_rd) { + this.dns_rd = dns_rd; + } + + public int getDns_ra() { + return dns_ra; + } + + public void setDns_ra(int dns_ra) { + this.dns_ra = dns_ra; + } + + public int getDns_rcode() { + return dns_rcode; + } + + public void setDns_rcode(int dns_rcode) { + this.dns_rcode = dns_rcode; + } + + public int getDns_qdcount() { + return dns_qdcount; + } + + public void setDns_qdcount(int dns_qdcount) { + this.dns_qdcount = dns_qdcount; + } + + public int getDns_ancount() { + return dns_ancount; + } + + public void setDns_ancount(int dns_ancount) { + this.dns_ancount = dns_ancount; + } + + public int getDns_nscount() { + return dns_nscount; + } + + public void setDns_nscount(int dns_nscount) { + this.dns_nscount = dns_nscount; + } + + public int getDns_arcount() { + return dns_arcount; + } + + public void setDns_arcount(int dns_arcount) { + this.dns_arcount = dns_arcount; + } + + public String getDns_qname() { + return dns_qname; + } + + public void setDns_qname(String dns_qname) { + this.dns_qname = dns_qname; + } + + public int getDns_qtype() { + return dns_qtype; + } + + public void setDns_qtype(int dns_qtype) { + this.dns_qtype = dns_qtype; + } + + public int getDns_qclass() { + return dns_qclass; + } + + public void setDns_qclass(int dns_qclass) { + this.dns_qclass = dns_qclass; + } + + public String getDns_cname() { + return dns_cname; + } + + public void setDns_cname(String dns_cname) { + this.dns_cname = dns_cname; + } + + public int getDns_sub() { + return dns_sub; + } + + public void setDns_sub(int dns_sub) { + this.dns_sub = dns_sub; + } + + public String getDns_rr() { + return dns_rr; + } + + public void setDns_rr(String dns_rr) { + this.dns_rr = dns_rr; + } + + public int getSsl_pinningst() { + return ssl_pinningst; + } + + public void setSsl_pinningst(int ssl_pinningst) { + this.ssl_pinningst = ssl_pinningst; + } + + public int getSsl_intercept_state() { + return ssl_intercept_state; + } + + public void setSsl_intercept_state(int ssl_intercept_state) { + this.ssl_intercept_state = ssl_intercept_state; + } + + public int getSsl_server_side_latency() { + return ssl_server_side_latency; + } + + public void setSsl_server_side_latency(int ssl_server_side_latency) { + this.ssl_server_side_latency = ssl_server_side_latency; + } + + public int getSsl_client_side_latency() { + return ssl_client_side_latency; + } + + public void setSsl_client_side_latency(int ssl_client_side_latency) { + this.ssl_client_side_latency = ssl_client_side_latency; + } + + public int getSsl_cert_verify() { + return ssl_cert_verify; + } + + public void setSsl_cert_verify(int ssl_cert_verify) { + this.ssl_cert_verify = ssl_cert_verify; + } + + public int getSsl_con_latency_ms() { + return ssl_con_latency_ms; + } + + public void setSsl_con_latency_ms(int ssl_con_latency_ms) { + this.ssl_con_latency_ms = ssl_con_latency_ms; + } + + public String getSsl_version() { + return ssl_version; + } + + public void setSsl_version(String ssl_version) { + this.ssl_version = ssl_version; + } + + public String getSsl_sni() { + return ssl_sni; + } + + public void setSsl_sni(String ssl_sni) { + this.ssl_sni = ssl_sni; + } + + public String getSsl_san() { + return ssl_san; + } + + public void setSsl_san(String ssl_san) { + this.ssl_san = ssl_san; + } + + public String getSsl_cn() { + return ssl_cn; + } + + public void setSsl_cn(String ssl_cn) { + this.ssl_cn = ssl_cn; + } + + public String getSsl_server_side_version() { + return ssl_server_side_version; + } + + public void setSsl_server_side_version(String ssl_server_side_version) { + this.ssl_server_side_version = ssl_server_side_version; + } + + public String getSsl_client_side_version() { + return ssl_client_side_version; + } + + public void setSsl_client_side_version(String ssl_client_side_version) { + this.ssl_client_side_version = ssl_client_side_version; + } + + public String getSsl_error() { + return ssl_error; + } + + public void setSsl_error(String ssl_error) { + this.ssl_error = ssl_error; + } + + public String getFtp_url() { + return ftp_url; + } + + public void setFtp_url(String ftp_url) { + this.ftp_url = ftp_url; + } + + public String getFtp_content() { + return ftp_content; + } + + public void setFtp_content(String ftp_content) { + this.ftp_content = ftp_content; + } + + public int getBgp_type() { + return bgp_type; + } + + public void setBgp_type(int bgp_type) { + this.bgp_type = bgp_type; + } + + public String getBgp_as_num() { + return bgp_as_num; + } + + public void setBgp_as_num(String bgp_as_num) { + this.bgp_as_num = bgp_as_num; + } + + public String getBgp_route() { + return bgp_route; + } + + public void setBgp_route(String bgp_route) { + this.bgp_route = bgp_route; + } + + public String getVoip_calling_account() { + return voip_calling_account; + } + + public void setVoip_calling_account(String voip_calling_account) { + this.voip_calling_account = voip_calling_account; + } + + public String getVoip_called_account() { + return voip_called_account; + } + + public void setVoip_called_account(String voip_called_account) { + this.voip_called_account = voip_called_account; + } + + public String getVoip_calling_number() { + return voip_calling_number; + } + + public void setVoip_calling_number(String voip_calling_number) { + this.voip_calling_number = voip_calling_number; + } + + public String getVoip_called_number() { + return voip_called_number; + } + + public void setVoip_called_number(String voip_called_number) { + this.voip_called_number = voip_called_number; + } + + + public String getStreaming_media_url() { + return streaming_media_url; + } + + public void setStreaming_media_url(String streaming_media_url) { + this.streaming_media_url = streaming_media_url; + } + + public String getStreaming_media_protocol() { + return streaming_media_protocol; + } + + public void setStreaming_media_protocol(String streaming_media_protocol) { + this.streaming_media_protocol = streaming_media_protocol; + } +} diff --git a/src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java b/src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java new file mode 100644 index 0000000..53b8bd7 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java @@ -0,0 +1,198 @@ +package cn.ac.iie.bean; +import com.alibaba.fastjson.annotation.JSONField; +import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter; +public class ProxySessionRecordLog extends PublicSessionRecordLog{ + + +//TODO HTTP协议属性 21 + + private String http_url; + private String http_host; + private String http_domain; + private String http_request_line; + private String http_response_line; + private String http_request_header; + private String http_response_header; + private String http_request_body; + private String http_response_body; + private String http_request_body_key; + private String http_response_body_key; + private int http_proxy_flag; + private int http_sequence; + private String http_snapshot; + private String http_cookie; + private String http_referer; + private String http_user_agent; + private String http_content_length; + private String http_content_type; + private String http_set_cookie; + private String http_version; + + public String getHttp_url() { + return http_url; + } + + public void setHttp_url(String http_url) { + this.http_url = http_url; + } + + public String getHttp_host() { + return http_host; + } + + public void setHttp_host(String http_host) { + this.http_host = http_host; + } + + public String getHttp_domain() { + return http_domain; + } + + public void setHttp_domain(String http_domain) { + this.http_domain = http_domain; + } + + public String getHttp_request_line() { + return http_request_line; + } + + public void setHttp_request_line(String http_request_line) { + this.http_request_line = http_request_line; + } + + public String getHttp_response_line() { + return http_response_line; + } + + public void setHttp_response_line(String http_response_line) { + this.http_response_line = http_response_line; + } + + public String getHttp_request_header() { + return http_request_header; + } + + public void setHttp_request_header(String http_request_header) { + this.http_request_header = http_request_header; + } + + public String getHttp_response_header() { + return http_response_header; + } + + public void setHttp_response_header(String http_response_header) { + this.http_response_header = http_response_header; + } + + public String getHttp_request_body() { + return http_request_body; + } + + public void setHttp_request_body(String http_request_body) { + this.http_request_body = http_request_body; + } + + public String getHttp_response_body() { + return http_response_body; + } + + public void setHttp_response_body(String http_response_body) { + this.http_response_body = http_response_body; + } + + public String getHttp_request_body_key() { + return http_request_body_key; + } + + public void setHttp_request_body_key(String http_request_body_key) { + this.http_request_body_key = http_request_body_key; + } + + public String getHttp_response_body_key() { + return http_response_body_key; + } + + public void setHttp_response_body_key(String http_response_body_key) { + this.http_response_body_key = http_response_body_key; + } + + public int getHttp_proxy_flag() { + return http_proxy_flag; + } + + public void setHttp_proxy_flag(int http_proxy_flag) { + this.http_proxy_flag = http_proxy_flag; + } + + public int getHttp_sequence() { + return http_sequence; + } + + public void setHttp_sequence(int http_sequence) { + this.http_sequence = http_sequence; + } + + public String getHttp_snapshot() { + return http_snapshot; + } + + public void setHttp_snapshot(String http_snapshot) { + this.http_snapshot = http_snapshot; + } + + public String getHttp_cookie() { + return http_cookie; + } + + public void setHttp_cookie(String http_cookie) { + this.http_cookie = http_cookie; + } + + public String getHttp_referer() { + return http_referer; + } + + public void setHttp_referer(String http_referer) { + this.http_referer = http_referer; + } + + public String getHttp_user_agent() { + return http_user_agent; + } + + public void setHttp_user_agent(String http_user_agent) { + this.http_user_agent = http_user_agent; + } + + public String getHttp_content_length() { + return http_content_length; + } + + public void setHttp_content_length(String http_content_length) { + this.http_content_length = http_content_length; + } + + public String getHttp_content_type() { + return http_content_type; + } + + public void setHttp_content_type(String http_content_type) { + this.http_content_type = http_content_type; + } + + public String getHttp_set_cookie() { + return http_set_cookie; + } + + public void setHttp_set_cookie(String http_set_cookie) { + this.http_set_cookie = http_set_cookie; + } + + public String getHttp_version() { + return http_version; + } + + public void setHttp_version(String http_version) { + this.http_version = http_version; + } +} diff --git a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java new file mode 100644 index 0000000..7d853d4 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java @@ -0,0 +1,380 @@ +package cn.ac.iie.bean; + +import com.alibaba.fastjson.annotation.JSONField; +import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter; + +public class PublicSessionRecordLog { + +//TODO 基础属性 40 int22 + private int common_service; + private int common_direction; + private long common_recv_time; + private int common_address_type; + private int common_policy_id; + private int common_action; + private int common_client_port; + private int common_entrance_id; + private int common_link_id; + private int common_encapsulation; + private int common_server_port; + private int common_app_id; + private int common_protocol_id; + private int common_c2s_pkt_num; + private int common_s2c_pkt_num; + private long common_c2s_byte_num; + private long common_s2c_byte_num; + private int common_start_time; + private int common_end_time; + private int common_con_duration_ms; + private int common_stream_dir; + private int common_has_dup_traffic; + private long common_log_id; + + private String common_schema_type; + + private String common_l4_protocol; + private String common_user_tags; + private String common_user_region; + private String common_client_ip; + private String common_device_id; + private String common_isp; + private String common_sled_ip; + private String common_client_location; + private String common_client_asn; + private String common_subscriber_id; + private String common_server_ip; + private String common_server_location; + private String common_server_asn; + private String common_app_label; + private String common_address_list; + private String common_stream_error; + private String common_stream_trace_id; + + public int getCommon_service() { + return common_service; + } + + public void setCommon_service(int common_service) { + this.common_service = common_service; + } + + public int getCommon_direction() { + return common_direction; + } + + public void setCommon_direction(int common_direction) { + this.common_direction = common_direction; + } + + public long getCommon_recv_time() { + return common_recv_time; + } + + public void setCommon_recv_time(long common_recv_time) { + this.common_recv_time = common_recv_time; + } + + public int getCommon_address_type() { + return common_address_type; + } + + public void setCommon_address_type(int common_address_type) { + this.common_address_type = common_address_type; + } + + public int getCommon_policy_id() { + return common_policy_id; + } + + public void setCommon_policy_id(int common_policy_id) { + this.common_policy_id = common_policy_id; + } + + public int getCommon_action() { + return common_action; + } + + public void setCommon_action(int common_action) { + this.common_action = common_action; + } + + public int getCommon_client_port() { + return common_client_port; + } + + public void setCommon_client_port(int common_client_port) { + this.common_client_port = common_client_port; + } + + public int getCommon_entrance_id() { + return common_entrance_id; + } + + public void setCommon_entrance_id(int common_entrance_id) { + this.common_entrance_id = common_entrance_id; + } + + public int getCommon_link_id() { + return common_link_id; + } + + public void setCommon_link_id(int common_link_id) { + this.common_link_id = common_link_id; + } + + public int getCommon_encapsulation() { + return common_encapsulation; + } + + public void setCommon_encapsulation(int common_encapsulation) { + this.common_encapsulation = common_encapsulation; + } + + public int getCommon_server_port() { + return common_server_port; + } + + public void setCommon_server_port(int common_server_port) { + this.common_server_port = common_server_port; + } + + public int getCommon_app_id() { + return common_app_id; + } + + public void setCommon_app_id(int common_app_id) { + this.common_app_id = common_app_id; + } + + public int getCommon_protocol_id() { + return common_protocol_id; + } + + public void setCommon_protocol_id(int common_protocol_id) { + this.common_protocol_id = common_protocol_id; + } + + public int getCommon_c2s_pkt_num() { + return common_c2s_pkt_num; + } + + public void setCommon_c2s_pkt_num(int common_c2s_pkt_num) { + this.common_c2s_pkt_num = common_c2s_pkt_num; + } + + public int getCommon_s2c_pkt_num() { + return common_s2c_pkt_num; + } + + public void setCommon_s2c_pkt_num(int common_s2c_pkt_num) { + this.common_s2c_pkt_num = common_s2c_pkt_num; + } + + public long getCommon_c2s_byte_num() { + return common_c2s_byte_num; + } + + public void setCommon_c2s_byte_num(long common_c2s_byte_num) { + this.common_c2s_byte_num = common_c2s_byte_num; + } + + public long getCommon_s2c_byte_num() { + return common_s2c_byte_num; + } + + public void setCommon_s2c_byte_num(long common_s2c_byte_num) { + this.common_s2c_byte_num = common_s2c_byte_num; + } + + public int getCommon_start_time() { + return common_start_time; + } + + public void setCommon_start_time(int common_start_time) { + this.common_start_time = common_start_time; + } + + public int getCommon_end_time() { + return common_end_time; + } + + public void setCommon_end_time(int common_end_time) { + this.common_end_time = common_end_time; + } + + public int getCommon_con_duration_ms() { + return common_con_duration_ms; + } + + public void setCommon_con_duration_ms(int common_con_duration_ms) { + this.common_con_duration_ms = common_con_duration_ms; + } + + public int getCommon_stream_dir() { + return common_stream_dir; + } + + public void setCommon_stream_dir(int common_stream_dir) { + this.common_stream_dir = common_stream_dir; + } + + public int getCommon_has_dup_traffic() { + return common_has_dup_traffic; + } + + public void setCommon_has_dup_traffic(int common_has_dup_traffic) { + this.common_has_dup_traffic = common_has_dup_traffic; + } + + public String getCommon_schema_type() { + return common_schema_type; + } + + public void setCommon_schema_type(String common_schema_type) { + this.common_schema_type = common_schema_type; + } + + public long getCommon_log_id() { + return common_log_id; + } + + public void setCommon_log_id(long common_log_id) { + this.common_log_id = common_log_id; + } + + public String getCommon_l4_protocol() { + return common_l4_protocol; + } + + public void setCommon_l4_protocol(String common_l4_protocol) { + this.common_l4_protocol = common_l4_protocol; + } + + public String getCommon_user_tags() { + return common_user_tags; + } + + public void setCommon_user_tags(String common_user_tags) { + this.common_user_tags = common_user_tags; + } + + public String getCommon_user_region() { + return common_user_region; + } + + public void setCommon_user_region(String common_user_region) { + this.common_user_region = common_user_region; + } + + public String getCommon_client_ip() { + return common_client_ip; + } + + public void setCommon_client_ip(String common_client_ip) { + this.common_client_ip = common_client_ip; + } + + public String getCommon_device_id() { + return common_device_id; + } + + public void setCommon_device_id(String common_device_id) { + this.common_device_id = common_device_id; + } + + public String getCommon_isp() { + return common_isp; + } + + public void setCommon_isp(String common_isp) { + this.common_isp = common_isp; + } + + public String getCommon_sled_ip() { + return common_sled_ip; + } + + public void setCommon_sled_ip(String common_sled_ip) { + this.common_sled_ip = common_sled_ip; + } + + public String getCommon_client_location() { + return common_client_location; + } + + public void setCommon_client_location(String common_client_location) { + this.common_client_location = common_client_location; + } + + public String getCommon_client_asn() { + return common_client_asn; + } + + public void setCommon_client_asn(String common_client_asn) { + this.common_client_asn = common_client_asn; + } + + public String getCommon_subscriber_id() { + return common_subscriber_id; + } + + public void setCommon_subscriber_id(String common_subscriber_id) { + this.common_subscriber_id = common_subscriber_id; + } + + public String getCommon_server_ip() { + return common_server_ip; + } + + public void setCommon_server_ip(String common_server_ip) { + this.common_server_ip = common_server_ip; + } + + public String getCommon_server_location() { + return common_server_location; + } + + public void setCommon_server_location(String common_server_location) { + this.common_server_location = common_server_location; + } + + public String getCommon_server_asn() { + return common_server_asn; + } + + public void setCommon_server_asn(String common_server_asn) { + this.common_server_asn = common_server_asn; + } + + public String getCommon_app_label() { + return common_app_label; + } + + public void setCommon_app_label(String common_app_label) { + this.common_app_label = common_app_label; + } + + public String getCommon_address_list() { + return common_address_list; + } + + public void setCommon_address_list(String common_address_list) { + this.common_address_list = common_address_list; + } + + public String getCommon_stream_error() { + return common_stream_error; + } + + public void setCommon_stream_error(String common_stream_error) { + this.common_stream_error = common_stream_error; + } + + public String getCommon_stream_trace_id() { + return common_stream_trace_id; + } + + public void setCommon_stream_trace_id(String common_stream_trace_id) { + this.common_stream_trace_id = common_stream_trace_id; + } +} diff --git a/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java new file mode 100644 index 0000000..d467afc --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java @@ -0,0 +1,84 @@ +package cn.ac.iie.bean; +import com.alibaba.fastjson.annotation.JSONField; +import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter; +public class RadiusSessionRecordLog extends PublicSessionRecordLog{ + + +//TODO RADIUS协议属性 4 + + private int radius_session_timeout; + private int radius_idle_timeout; + private int radius_acct_status_type; + private int radius_acct_terminate_cause; + + public int getRadius_session_timeout() { + return radius_session_timeout; + } + + public void setRadius_session_timeout(int radius_session_timeout) { + this.radius_session_timeout = radius_session_timeout; + } + + public int getRadius_idle_timeout() { + return radius_idle_timeout; + } + + public void setRadius_idle_timeout(int radius_idle_timeout) { + this.radius_idle_timeout = radius_idle_timeout; + } + + public int getRadius_acct_status_type() { + return radius_acct_status_type; + } + + public void setRadius_acct_status_type(int radius_acct_status_type) { + this.radius_acct_status_type = radius_acct_status_type; + } + + public int getRadius_acct_terminate_cause() { + return radius_acct_terminate_cause; + } + + public void setRadius_acct_terminate_cause(int radius_acct_terminate_cause) { + this.radius_acct_terminate_cause = radius_acct_terminate_cause; + } + + private int radius_packet_type; + + private String radius_nas_ip; + private String radius_framed_ip; + private String radius_account; + + + public int getRadius_packet_type() { + return radius_packet_type; + } + + public void setRadius_packet_type(int radius_packet_type) { + this.radius_packet_type = radius_packet_type; + } + + public String getRadius_nas_ip() { + return radius_nas_ip; + } + + public void setRadius_nas_ip(String radius_nas_ip) { + this.radius_nas_ip = radius_nas_ip; + } + + public String getRadius_framed_ip() { + return radius_framed_ip; + } + + public void setRadius_framed_ip(String radius_framed_ip) { + this.radius_framed_ip = radius_framed_ip; + } + + public String getRadius_account() { + return radius_account; + } + + public void setRadius_account(String radius_account) { + this.radius_account = radius_account; + } +} diff --git a/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java b/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java new file mode 100644 index 0000000..775f1c7 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java @@ -0,0 +1,677 @@ +package cn.ac.iie.bean; + +public class SecurityPolicyLog extends PublicSessionRecordLog{ + + +//TODO HTTP协议属性 21 + + private String http_url; + private String http_host; + private String http_domain; + private String http_request_line; + private String http_response_line; + private String http_request_header; + private String http_response_header; + private String http_request_body; + private String http_response_body; + private String http_request_body_key; + private String http_response_body_key; + private int http_proxy_flag; + private int http_sequence; + private String http_snapshot; + private String http_cookie; + private String http_referer; + private String http_user_agent; + private String http_content_length; + private String http_content_type; + private String http_set_cookie; + private String http_version; + +//TODO MAIL协议属性 9 + + private String mail_protocol_type; + private String mail_sender; + private String mail_receiver; + private String mail_subject; + private String mail_content; + private String mail_attachment_name; + private String mail_attachment_content; + private String mail_eml_file; + private String mail_snapshot; + +//TODO 3DNS协议属性 18 + + private int dns_message_id; + private int dns_qr; + private int dns_opcode; + private int dns_aa; + private int dns_tc; + private int dns_rd; + private int dns_ra; + private int dns_rcode; + private int dns_qdcount; + private int dns_ancount; + private int dns_nscount; + private int dns_arcount; + private String dns_qname; + private int dns_qtype; + private int dns_qclass; + private String dns_cname; + private int dns_sub; + private String dns_rr; + +//TODO SSL协议属性 13 + + + private int ssl_pinningst; + private int ssl_intercept_state; + private int ssl_server_side_latency; + private int ssl_client_side_latency; + private int ssl_cert_verify; + private int ssl_con_latency_ms; + private String ssl_version; + private String ssl_sni; + private String ssl_san; + private String ssl_cn; + private String ssl_server_side_version; + private String ssl_client_side_version; + private String ssl_error; + +//TODO FTP协议属性 2 + private String ftp_url; + private String ftp_content; + + +//TODO BGP协议属性 3 + private int bgp_type; + private String bgp_as_num; + private String bgp_route; + +//TODO VOIP协议属性 4 + private String voip_calling_account; + private String voip_called_account; + private String voip_calling_number; + private String voip_called_number; + +//TODO STREAMING_MEDIA协议属性 2 + private String streaming_media_url; + private String streaming_media_protocol; + + + public String getHttp_url() { + return http_url; + } + + public void setHttp_url(String http_url) { + this.http_url = http_url; + } + + public String getHttp_host() { + return http_host; + } + + public void setHttp_host(String http_host) { + this.http_host = http_host; + } + + public String getHttp_domain() { + return http_domain; + } + + public void setHttp_domain(String http_domain) { + this.http_domain = http_domain; + } + + public String getHttp_request_line() { + return http_request_line; + } + + public void setHttp_request_line(String http_request_line) { + this.http_request_line = http_request_line; + } + + public String getHttp_response_line() { + return http_response_line; + } + + public void setHttp_response_line(String http_response_line) { + this.http_response_line = http_response_line; + } + + public String getHttp_request_header() { + return http_request_header; + } + + public void setHttp_request_header(String http_request_header) { + this.http_request_header = http_request_header; + } + + public String getHttp_response_header() { + return http_response_header; + } + + public void setHttp_response_header(String http_response_header) { + this.http_response_header = http_response_header; + } + + public String getHttp_request_body() { + return http_request_body; + } + + public void setHttp_request_body(String http_request_body) { + this.http_request_body = http_request_body; + } + + public String getHttp_response_body() { + return http_response_body; + } + + public void setHttp_response_body(String http_response_body) { + this.http_response_body = http_response_body; + } + + public String getHttp_request_body_key() { + return http_request_body_key; + } + + public void setHttp_request_body_key(String http_request_body_key) { + this.http_request_body_key = http_request_body_key; + } + + public String getHttp_response_body_key() { + return http_response_body_key; + } + + public void setHttp_response_body_key(String http_response_body_key) { + this.http_response_body_key = http_response_body_key; + } + + public int getHttp_proxy_flag() { + return http_proxy_flag; + } + + public void setHttp_proxy_flag(int http_proxy_flag) { + this.http_proxy_flag = http_proxy_flag; + } + + public int getHttp_sequence() { + return http_sequence; + } + + public void setHttp_sequence(int http_sequence) { + this.http_sequence = http_sequence; + } + + public String getHttp_snapshot() { + return http_snapshot; + } + + public void setHttp_snapshot(String http_snapshot) { + this.http_snapshot = http_snapshot; + } + + public String getHttp_cookie() { + return http_cookie; + } + + public void setHttp_cookie(String http_cookie) { + this.http_cookie = http_cookie; + } + + public String getHttp_referer() { + return http_referer; + } + + public void setHttp_referer(String http_referer) { + this.http_referer = http_referer; + } + + public String getHttp_user_agent() { + return http_user_agent; + } + + public void setHttp_user_agent(String http_user_agent) { + this.http_user_agent = http_user_agent; + } + + public String getHttp_content_length() { + return http_content_length; + } + + public void setHttp_content_length(String http_content_length) { + this.http_content_length = http_content_length; + } + + public String getHttp_content_type() { + return http_content_type; + } + + public void setHttp_content_type(String http_content_type) { + this.http_content_type = http_content_type; + } + + public String getHttp_set_cookie() { + return http_set_cookie; + } + + public void setHttp_set_cookie(String http_set_cookie) { + this.http_set_cookie = http_set_cookie; + } + + public String getHttp_version() { + return http_version; + } + + public void setHttp_version(String http_version) { + this.http_version = http_version; + } + + public String getMail_protocol_type() { + return mail_protocol_type; + } + + public void setMail_protocol_type(String mail_protocol_type) { + this.mail_protocol_type = mail_protocol_type; + } + + public String getMail_sender() { + return mail_sender; + } + + public void setMail_sender(String mail_sender) { + this.mail_sender = mail_sender; + } + + public String getMail_receiver() { + return mail_receiver; + } + + public void setMail_receiver(String mail_receiver) { + this.mail_receiver = mail_receiver; + } + + public String getMail_subject() { + return mail_subject; + } + + public void setMail_subject(String mail_subject) { + this.mail_subject = mail_subject; + } + + public String getMail_content() { + return mail_content; + } + + public void setMail_content(String mail_content) { + this.mail_content = mail_content; + } + + public String getMail_attachment_name() { + return mail_attachment_name; + } + + public void setMail_attachment_name(String mail_attachment_name) { + this.mail_attachment_name = mail_attachment_name; + } + + public String getMail_attachment_content() { + return mail_attachment_content; + } + + public void setMail_attachment_content(String mail_attachment_content) { + this.mail_attachment_content = mail_attachment_content; + } + + public String getMail_eml_file() { + return mail_eml_file; + } + + public void setMail_eml_file(String mail_eml_file) { + this.mail_eml_file = mail_eml_file; + } + + public String getMail_snapshot() { + return mail_snapshot; + } + + public void setMail_snapshot(String mail_snapshot) { + this.mail_snapshot = mail_snapshot; + } + + public int getDns_message_id() { + return dns_message_id; + } + + public void setDns_message_id(int dns_message_id) { + this.dns_message_id = dns_message_id; + } + + public int getDns_qr() { + return dns_qr; + } + + public void setDns_qr(int dns_qr) { + this.dns_qr = dns_qr; + } + + public int getDns_opcode() { + return dns_opcode; + } + + public void setDns_opcode(int dns_opcode) { + this.dns_opcode = dns_opcode; + } + + public int getDns_aa() { + return dns_aa; + } + + public void setDns_aa(int dns_aa) { + this.dns_aa = dns_aa; + } + + public int getDns_tc() { + return dns_tc; + } + + public void setDns_tc(int dns_tc) { + this.dns_tc = dns_tc; + } + + public int getDns_rd() { + return dns_rd; + } + + public void setDns_rd(int dns_rd) { + this.dns_rd = dns_rd; + } + + public int getDns_ra() { + return dns_ra; + } + + public void setDns_ra(int dns_ra) { + this.dns_ra = dns_ra; + } + + public int getDns_rcode() { + return dns_rcode; + } + + public void setDns_rcode(int dns_rcode) { + this.dns_rcode = dns_rcode; + } + + public int getDns_qdcount() { + return dns_qdcount; + } + + public void setDns_qdcount(int dns_qdcount) { + this.dns_qdcount = dns_qdcount; + } + + public int getDns_ancount() { + return dns_ancount; + } + + public void setDns_ancount(int dns_ancount) { + this.dns_ancount = dns_ancount; + } + + public int getDns_nscount() { + return dns_nscount; + } + + public void setDns_nscount(int dns_nscount) { + this.dns_nscount = dns_nscount; + } + + public int getDns_arcount() { + return dns_arcount; + } + + public void setDns_arcount(int dns_arcount) { + this.dns_arcount = dns_arcount; + } + + public String getDns_qname() { + return dns_qname; + } + + public void setDns_qname(String dns_qname) { + this.dns_qname = dns_qname; + } + + public int getDns_qtype() { + return dns_qtype; + } + + public void setDns_qtype(int dns_qtype) { + this.dns_qtype = dns_qtype; + } + + public int getDns_qclass() { + return dns_qclass; + } + + public void setDns_qclass(int dns_qclass) { + this.dns_qclass = dns_qclass; + } + + public String getDns_cname() { + return dns_cname; + } + + public void setDns_cname(String dns_cname) { + this.dns_cname = dns_cname; + } + + public int getDns_sub() { + return dns_sub; + } + + public void setDns_sub(int dns_sub) { + this.dns_sub = dns_sub; + } + + public String getDns_rr() { + return dns_rr; + } + + public void setDns_rr(String dns_rr) { + this.dns_rr = dns_rr; + } + + public int getSsl_pinningst() { + return ssl_pinningst; + } + + public void setSsl_pinningst(int ssl_pinningst) { + this.ssl_pinningst = ssl_pinningst; + } + + public int getSsl_intercept_state() { + return ssl_intercept_state; + } + + public void setSsl_intercept_state(int ssl_intercept_state) { + this.ssl_intercept_state = ssl_intercept_state; + } + + public int getSsl_server_side_latency() { + return ssl_server_side_latency; + } + + public void setSsl_server_side_latency(int ssl_server_side_latency) { + this.ssl_server_side_latency = ssl_server_side_latency; + } + + public int getSsl_client_side_latency() { + return ssl_client_side_latency; + } + + public void setSsl_client_side_latency(int ssl_client_side_latency) { + this.ssl_client_side_latency = ssl_client_side_latency; + } + + public int getSsl_cert_verify() { + return ssl_cert_verify; + } + + public void setSsl_cert_verify(int ssl_cert_verify) { + this.ssl_cert_verify = ssl_cert_verify; + } + + public int getSsl_con_latency_ms() { + return ssl_con_latency_ms; + } + + public void setSsl_con_latency_ms(int ssl_con_latency_ms) { + this.ssl_con_latency_ms = ssl_con_latency_ms; + } + + public String getSsl_version() { + return ssl_version; + } + + public void setSsl_version(String ssl_version) { + this.ssl_version = ssl_version; + } + + public String getSsl_sni() { + return ssl_sni; + } + + public void setSsl_sni(String ssl_sni) { + this.ssl_sni = ssl_sni; + } + + public String getSsl_san() { + return ssl_san; + } + + public void setSsl_san(String ssl_san) { + this.ssl_san = ssl_san; + } + + public String getSsl_cn() { + return ssl_cn; + } + + public void setSsl_cn(String ssl_cn) { + this.ssl_cn = ssl_cn; + } + + public String getSsl_server_side_version() { + return ssl_server_side_version; + } + + public void setSsl_server_side_version(String ssl_server_side_version) { + this.ssl_server_side_version = ssl_server_side_version; + } + + public String getSsl_client_side_version() { + return ssl_client_side_version; + } + + public void setSsl_client_side_version(String ssl_client_side_version) { + this.ssl_client_side_version = ssl_client_side_version; + } + + public String getSsl_error() { + return ssl_error; + } + + public void setSsl_error(String ssl_error) { + this.ssl_error = ssl_error; + } + + public String getFtp_url() { + return ftp_url; + } + + public void setFtp_url(String ftp_url) { + this.ftp_url = ftp_url; + } + + public String getFtp_content() { + return ftp_content; + } + + public void setFtp_content(String ftp_content) { + this.ftp_content = ftp_content; + } + + public int getBgp_type() { + return bgp_type; + } + + public void setBgp_type(int bgp_type) { + this.bgp_type = bgp_type; + } + + public String getBgp_as_num() { + return bgp_as_num; + } + + public void setBgp_as_num(String bgp_as_num) { + this.bgp_as_num = bgp_as_num; + } + + public String getBgp_route() { + return bgp_route; + } + + public void setBgp_route(String bgp_route) { + this.bgp_route = bgp_route; + } + + public String getVoip_calling_account() { + return voip_calling_account; + } + + public void setVoip_calling_account(String voip_calling_account) { + this.voip_calling_account = voip_calling_account; + } + + public String getVoip_called_account() { + return voip_called_account; + } + + public void setVoip_called_account(String voip_called_account) { + this.voip_called_account = voip_called_account; + } + + public String getVoip_calling_number() { + return voip_calling_number; + } + + public void setVoip_calling_number(String voip_calling_number) { + this.voip_calling_number = voip_calling_number; + } + + public String getVoip_called_number() { + return voip_called_number; + } + + public void setVoip_called_number(String voip_called_number) { + this.voip_called_number = voip_called_number; + } + + + public String getStreaming_media_url() { + return streaming_media_url; + } + + public void setStreaming_media_url(String streaming_media_url) { + this.streaming_media_url = streaming_media_url; + } + + public String getStreaming_media_protocol() { + return streaming_media_protocol; + } + + public void setStreaming_media_protocol(String streaming_media_protocol) { + this.streaming_media_protocol = streaming_media_protocol; + } +} diff --git a/src/main/java/cn/ac/iie/bean/SessionRecordLog.java b/src/main/java/cn/ac/iie/bean/SessionRecordLog.java deleted file mode 100644 index baa3f81..0000000 --- a/src/main/java/cn/ac/iie/bean/SessionRecordLog.java +++ /dev/null @@ -1,672 +0,0 @@ -package cn.ac.iie.bean; - -import com.alibaba.fastjson.annotation.JSONField; -import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter; - -/** - * @author qidaijie - */ -public class SessionRecordLog { - private long uid; - private int policy_id; - private long action; - private int start_time; - private int end_time; - private long recv_time; - private String trans_proto; - private String app_proto; - private int addr_type; - private String server_ip; - private String client_ip; - private int server_port; - private int client_port; - private int service; - private int entrance_id; - private int device_id; - private int Link_id; - private String isp; - private int encap_type; - private int direction; - private int stream_dir; - private String cap_ip; - private String addr_list; - private String server_location; - private String client_location; - private String client_asn; - private String server_asn; - private String subscribe_id; - private long con_duration_ms; - private String url; - private String host; - private String domain; - private String category; - private String req_line; - private String res_line; - private String cookie; - private String referer; - private String user_agent; - private String content_len; - private String content_type; - private String set_cookie; - private String req_header; - private String resp_header; - private String req_body_key; - private String req_body; - private String res_body_key; - private String resp_body; - private String version; - private String sni; - private String san; - private String cn; - private int app_id; - private int protocol_id; - private long con_latency_ms; - private int pinningst; - private int intercept_state; - private long ssl_server_side_latency; - private long ssl_client_side_latency; - private String ssl_server_side_version; - private String ssl_client_side_version; - private int ssl_cert_verify; - private String stream_trace_id; - private String ssl_error; - private long c2s_pkt_num; - private long S2c_pkt_num; - private long c2s_byte_num; - private long s2c_byte_num; - private String nas_ip; - private String framed_ip; - private String account; - private int packet_type; - private int has_dup_traffic; - private String stream_error; - - - public SessionRecordLog() { - } - - public long getUid() { - return uid; - } - - public void setUid(long uid) { - this.uid = uid; - } - - public int getPolicy_id() { - return policy_id; - } - - public void setPolicy_id(int policy_id) { - this.policy_id = policy_id; - } - - public long getAction() { - return action; - } - - public void setAction(long action) { - this.action = action; - } - - public int getStart_time() { - return start_time; - } - - public void setStart_time(int start_time) { - this.start_time = start_time; - } - - public int getEnd_time() { - return end_time; - } - - public void setEnd_time(int end_time) { - this.end_time = end_time; - } - - public String getSsl_error() { - return ssl_error; - } - - public void setSsl_error(String ssl_error) { - this.ssl_error = ssl_error; - } - - public String getApp_proto() { - return app_proto; - } - - public void setApp_proto(String app_proto) { - this.app_proto = app_proto; - } - - public long getRecv_time() { - return recv_time; - } - - public void setRecv_time(long recv_time) { - this.recv_time = recv_time; - } - - public String getTrans_proto() { - return trans_proto; - } - - public void setTrans_proto(String trans_proto) { - this.trans_proto = trans_proto; - } - - public int getAddr_type() { - return addr_type; - } - - public void setAddr_type(int addr_type) { - this.addr_type = addr_type; - } - - public String getServer_ip() { - return server_ip; - } - - public void setServer_ip(String server_ip) { - this.server_ip = server_ip; - } - - public String getClient_ip() { - return client_ip; - } - - public void setClient_ip(String client_ip) { - this.client_ip = client_ip; - } - - public int getServer_port() { - return server_port; - } - - public void setServer_port(int server_port) { - this.server_port = server_port; - } - - public int getClient_port() { - return client_port; - } - - public void setClient_port(int client_port) { - this.client_port = client_port; - } - - public int getService() { - return service; - } - - public void setService(int service) { - this.service = service; - } - - public int getEntrance_id() { - return entrance_id; - } - - public void setEntrance_id(int entrance_id) { - this.entrance_id = entrance_id; - } - - public int getDevice_id() { - return device_id; - } - - public void setDevice_id(int device_id) { - this.device_id = device_id; - } - - public int getLink_id() { - return Link_id; - } - - public void setLink_id(int link_id) { - Link_id = link_id; - } - - public String getIsp() { - return isp; - } - - public void setIsp(String isp) { - this.isp = isp; - } - - public int getEncap_type() { - return encap_type; - } - - public void setEncap_type(int encap_type) { - this.encap_type = encap_type; - } - - public int getDirection() { - return direction; - } - - public void setDirection(int direction) { - this.direction = direction; - } - - public int getStream_dir() { - return stream_dir; - } - - public void setStream_dir(int stream_dir) { - this.stream_dir = stream_dir; - } - - public String getCap_ip() { - return cap_ip; - } - - public void setCap_ip(String cap_ip) { - this.cap_ip = cap_ip; - } - - public String getAddr_list() { - return addr_list; - } - - public void setAddr_list(String addr_list) { - this.addr_list = addr_list; - } - - public String getServer_location() { - return server_location; - } - - public void setServer_location(String server_location) { - this.server_location = server_location; - } - - public String getClient_location() { - return client_location; - } - - public void setClient_location(String client_location) { - this.client_location = client_location; - } - - public String getClient_asn() { - return client_asn; - } - - public void setClient_asn(String client_asn) { - this.client_asn = client_asn; - } - - public String getServer_asn() { - return server_asn; - } - - public void setServer_asn(String server_asn) { - this.server_asn = server_asn; - } - - public String getSubscribe_id() { - return subscribe_id; - } - - public void setSubscribe_id(String subscribe_id) { - this.subscribe_id = subscribe_id; - } - - public long getCon_duration_ms() { - return con_duration_ms; - } - - public void setCon_duration_ms(long con_duration_ms) { - this.con_duration_ms = con_duration_ms; - } - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public String getHost() { - return host; - } - - public void setHost(String host) { - this.host = host; - } - - public String getDomain() { - return domain; - } - - public void setDomain(String domain) { - this.domain = domain; - } - - public String getCategory() { - return category; - } - - public void setCategory(String category) { - this.category = category; - } - - public String getReq_line() { - return req_line; - } - - public void setReq_line(String req_line) { - this.req_line = req_line; - } - - public String getRes_line() { - return res_line; - } - - public void setRes_line(String res_line) { - this.res_line = res_line; - } - - public String getCookie() { - return cookie; - } - - public void setCookie(String cookie) { - this.cookie = cookie; - } - - public String getReferer() { - return referer; - } - - public void setReferer(String referer) { - this.referer = referer; - } - - public String getUser_agent() { - return user_agent; - } - - public void setUser_agent(String user_agent) { - this.user_agent = user_agent; - } - - public String getContent_len() { - return content_len; - } - - public void setContent_len(String content_len) { - this.content_len = content_len; - } - - public String getContent_type() { - return content_type; - } - - public void setContent_type(String content_type) { - this.content_type = content_type; - } - - public String getSet_cookie() { - return set_cookie; - } - - public void setSet_cookie(String set_cookie) { - this.set_cookie = set_cookie; - } - - public String getReq_header() { - return req_header; - } - - public void setReq_header(String req_header) { - this.req_header = req_header; - } - - public String getResp_header() { - return resp_header; - } - - public void setResp_header(String resp_header) { - this.resp_header = resp_header; - } - - public String getReq_body_key() { - return req_body_key; - } - - public void setReq_body_key(String req_body_key) { - this.req_body_key = req_body_key; - } - - public String getReq_body() { - return req_body; - } - - public void setReq_body(String req_body) { - this.req_body = req_body; - } - - public String getRes_body_key() { - return res_body_key; - } - - public void setRes_body_key(String res_body_key) { - this.res_body_key = res_body_key; - } - - public String getResp_body() { - return resp_body; - } - - public void setResp_body(String resp_body) { - this.resp_body = resp_body; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public String getSni() { - return sni; - } - - public void setSni(String sni) { - this.sni = sni; - } - - public String getSan() { - return san; - } - - public void setSan(String san) { - this.san = san; - } - - public String getCn() { - return cn; - } - - public void setCn(String cn) { - this.cn = cn; - } - - public int getApp_id() { - return app_id; - } - - public void setApp_id(int app_id) { - this.app_id = app_id; - } - - public int getProtocol_id() { - return protocol_id; - } - - public void setProtocol_id(int protocol_id) { - this.protocol_id = protocol_id; - } - - public int getIntercept_state() { - return intercept_state; - } - - public void setIntercept_state(int intercept_state) { - this.intercept_state = intercept_state; - } - - public long getSsl_server_side_latency() { - return ssl_server_side_latency; - } - - public void setSsl_server_side_latency(long ssl_server_side_latency) { - this.ssl_server_side_latency = ssl_server_side_latency; - } - - public long getSsl_client_side_latency() { - return ssl_client_side_latency; - } - - public void setSsl_client_side_latency(long ssl_client_side_latency) { - this.ssl_client_side_latency = ssl_client_side_latency; - } - - public String getSsl_server_side_version() { - return ssl_server_side_version; - } - - public void setSsl_server_side_version(String ssl_server_side_version) { - this.ssl_server_side_version = ssl_server_side_version; - } - - public String getSsl_client_side_version() { - return ssl_client_side_version; - } - - public void setSsl_client_side_version(String ssl_client_side_version) { - this.ssl_client_side_version = ssl_client_side_version; - } - - public int getSsl_cert_verify() { - return ssl_cert_verify; - } - - public void setSsl_cert_verify(int ssl_cert_verify) { - this.ssl_cert_verify = ssl_cert_verify; - } - - public String getStream_trace_id() { - return stream_trace_id; - } - - public void setStream_trace_id(String stream_trace_id) { - this.stream_trace_id = stream_trace_id; - } - - public long getCon_latency_ms() { - return con_latency_ms; - } - - public void setCon_latency_ms(long con_latency_ms) { - this.con_latency_ms = con_latency_ms; - } - - public int getPinningst() { - return pinningst; - } - - public void setPinningst(int pinningst) { - this.pinningst = pinningst; - } - - - public long getC2s_pkt_num() { - return c2s_pkt_num; - } - - public void setC2s_pkt_num(long c2s_pkt_num) { - this.c2s_pkt_num = c2s_pkt_num; - } - - public long getS2c_pkt_num() { - return S2c_pkt_num; - } - - public void setS2c_pkt_num(long s2c_pkt_num) { - S2c_pkt_num = s2c_pkt_num; - } - - public long getC2s_byte_num() { - return c2s_byte_num; - } - - public void setC2s_byte_num(long c2s_byte_num) { - this.c2s_byte_num = c2s_byte_num; - } - - public long getS2c_byte_num() { - return s2c_byte_num; - } - - public void setS2c_byte_num(long s2c_byte_num) { - this.s2c_byte_num = s2c_byte_num; - } - - public String getNas_ip() { - return nas_ip; - } - - public void setNas_ip(String nas_ip) { - this.nas_ip = nas_ip; - } - - public String getFramed_ip() { - return framed_ip; - } - - public void setFramed_ip(String framed_ip) { - this.framed_ip = framed_ip; - } - - public String getAccount() { - return account; - } - - public void setAccount(String account) { - this.account = account; - } - - public int getPacket_type() { - return packet_type; - } - - public void setPacket_type(int packet_type) { - this.packet_type = packet_type; - } - - public int getHas_dup_traffic() { - return has_dup_traffic; - } - - public void setHas_dup_traffic(int has_dup_traffic) { - this.has_dup_traffic = has_dup_traffic; - } - - public String getStream_error() { - return stream_error; - } - - public void setStream_error(String stream_error) { - this.stream_error = stream_error; - } -} diff --git a/src/main/java/cn/ac/iie/bolt/ConnCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java similarity index 76% rename from src/main/java/cn/ac/iie/bolt/ConnCompletionBolt.java rename to src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java index 109f856..1cd40fa 100644 --- a/src/main/java/cn/ac/iie/bolt/ConnCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java @@ -1,7 +1,6 @@ -package cn.ac.iie.bolt; +package cn.ac.iie.bolt.collectProtocol; import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HbaseUtils; import cn.ac.iie.utils.system.TupleUtils; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; @@ -16,16 +15,18 @@ import org.apache.storm.tuple.Values; import java.util.HashMap; import java.util.Map; -import static cn.ac.iie.utils.general.TransFormUtils.getJsonMessage; +import static cn.ac.iie.utils.general.TransFormUtils.getCollectProtocolMessage; +import static cn.ac.iie.utils.hbase.HbaseUtils.change; /** * 通联关系日志补全 * * @author qidaijie */ -public class ConnCompletionBolt extends BaseBasicBolt { - private static final long serialVersionUID = -1059151670138465894L; - private final static Logger logger = Logger.getLogger(ConnCompletionBolt.class); +@SuppressWarnings("all") +public class CollectCompletedBolt extends BaseBasicBolt { + private static final long serialVersionUID = -105915167013846589L; + private final static Logger logger = Logger.getLogger(CollectCompletedBolt.class); @Override public void prepare(Map stormConf, TopologyContext context) { @@ -36,14 +37,13 @@ public class ConnCompletionBolt extends BaseBasicBolt { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { try { if (TupleUtils.isTick(tuple)) { -// HbaseUtils.change(); - } else { + change(); + }else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { - basicOutputCollector.emit(new Values(getJsonMessage(message))); + basicOutputCollector.emit(new Values(getCollectProtocolMessage(message))); } } - } catch (Exception e) { logger.error("接收解析过程出现异常", e); } @@ -57,6 +57,7 @@ public class ConnCompletionBolt extends BaseBasicBolt { return conf; } + @Override public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { outputFieldsDeclarer.declare(new Fields("connLog")); diff --git a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java new file mode 100644 index 0000000..123095a --- /dev/null +++ b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java @@ -0,0 +1,65 @@ +package cn.ac.iie.bolt.proxy; + +import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.system.TupleUtils; +import com.zdjizhi.utils.StringUtil; +import org.apache.log4j.Logger; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.BasicOutputCollector; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseBasicBolt; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; + +import java.util.HashMap; +import java.util.Map; + +import static cn.ac.iie.utils.general.TransFormUtils.getProxyMessage; +import static cn.ac.iie.utils.hbase.HbaseUtils.change; + +/** + * 通联关系日志补全 + * + * @author qidaijie + */ +@SuppressWarnings("all") +public class ProxyCompletionBolt extends BaseBasicBolt { + private static final long serialVersionUID = -1059151670838465894L; + private final static Logger logger = Logger.getLogger(ProxyCompletionBolt.class); + + @Override + public void prepare(Map stormConf, TopologyContext context) { + + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { + try { + if (TupleUtils.isTick(tuple)) { + change(); + }else { + String message = tuple.getString(0); + if (StringUtil.isNotBlank(message)) { + basicOutputCollector.emit(new Values(getProxyMessage(message))); + } + } + } catch (Exception e) { + logger.error("接收解析过程出现异常", e); + } + } + + @Override + public Map getComponentConfiguration() { + Map conf = new HashMap(16); + conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, + FlowWriteConfig.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + return conf; + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { + outputFieldsDeclarer.declare(new Fields("connLog")); + } + +} diff --git a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java new file mode 100644 index 0000000..ebb33b7 --- /dev/null +++ b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java @@ -0,0 +1,65 @@ +package cn.ac.iie.bolt.security; + +import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.hbase.HbaseUtils; +import cn.ac.iie.utils.system.TupleUtils; +import com.zdjizhi.utils.StringUtil; +import org.apache.log4j.Logger; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.BasicOutputCollector; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseBasicBolt; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; + +import java.util.HashMap; +import java.util.Map; + +import static cn.ac.iie.utils.general.TransFormUtils.getSecurityMessage; + +/** + * 通联关系日志补全 + * + * @author qidaijie + */ +@SuppressWarnings("all") +public class SecurityCompletionBolt extends BaseBasicBolt { + private static final long serialVersionUID = -1059151770138464874L; + private final static Logger logger = Logger.getLogger(SecurityCompletionBolt.class); + + @Override + public void prepare(Map stormConf, TopologyContext context) { + + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { + try { + if (TupleUtils.isTick(tuple)) { + HbaseUtils.change(); + }else { + String message = tuple.getString(0); + if (StringUtil.isNotBlank(message)) { + basicOutputCollector.emit(new Values(getSecurityMessage(message))); + } + } + } catch (Exception e) { + logger.error("接收解析过程出现异常", e); + } + } + + @Override + public Map getComponentConfiguration() { + Map conf = new HashMap(16); + conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, + FlowWriteConfig.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + return conf; + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { + outputFieldsDeclarer.declare(new Fields("connLog")); + } + +} diff --git a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java index 6b677d8..80ecdb8 100644 --- a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java +++ b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java @@ -1,9 +1,13 @@ package cn.ac.iie.topology; -import cn.ac.iie.bolt.ConnCompletionBolt; import cn.ac.iie.bolt.NtcLogSendBolt; +import cn.ac.iie.bolt.collectProtocol.CollectCompletedBolt; import cn.ac.iie.bolt.radius.RadiusCompletionBolt; +import cn.ac.iie.bolt.security.SecurityCompletionBolt; + +import cn.ac.iie.bolt.proxy.ProxyCompletionBolt; + import cn.ac.iie.common.FlowWriteConfig; import cn.ac.iie.spout.CustomizedKafkaSpout; import org.apache.log4j.Logger; @@ -58,15 +62,45 @@ public class LogFlowWriteTopology { private void buildTopology() { builder = new TopologyBuilder(); builder.setSpout("LogFlowWriteSpout", new CustomizedKafkaSpout(), FlowWriteConfig.SPOUT_PARALLELISM); - if ("RADIUS-LOG".equals(FlowWriteConfig.KAFKA_TOPIC)) { - builder.setBolt("RadiusCompletionBolt", new RadiusCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); - builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt"); - } else { - builder.setBolt("ConnCompletionBolt", new ConnCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); - builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ConnCompletionBolt"); - } + + String topic_name = FlowWriteConfig.KAFKA_TOPIC; + + switch(topic_name){ + case "PROXY-POLICY-LOG": + builder.setBolt("ProxyCompletionBolt", new ProxyCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ProxyCompletionBolt"); + break; + + case "COLLECT-RADIUS-RECORD-LOG": + builder.setBolt("RadiusCompletionBolt", new RadiusCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt"); + break; + + case "COLLECT-CONNECTION-RECORD-LOG": + builder.setBolt("CollectCompletedBolt", new CollectCompletedBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("CollectCompletedBolt"); + break; + + case "SECURITY-POLICY-LOG": + builder.setBolt("SecurityCompletionBolt", new SecurityCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("SecurityCompletionBolt"); + break; + + + } + + /*if ("PROXY_POLICY_LOG".equals(FlowWriteConfig.KAFKA_TOPIC) || "COLLECT_HTTP_META_LOG".equals(FlowWriteConfig.KAFKA_TOPIC)) { + builder.setBolt("HttpCompletionBolt", new HttpCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("HttpCompletionBolt"); + } else if ("COLLECT_RADIUS_RECORD_LOG".equals(FlowWriteConfig.KAFKA_TOPIC)) { + builder.setBolt("RadiusCompletionBolt", new ConnCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt"); + } else { + builder.setBolt("ConnCompletionBolt", new ConnCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); + builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ConnCompletionBolt"); + }*/ // builder.setBolt("SummaryBolt", new SummaryBolt(), 1).localOrShuffleGrouping("NtcLogSendBolt"); - } + } public static void main(String[] args) throws Exception { LogFlowWriteTopology csst = null; diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index 3790c35..c515689 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -1,17 +1,13 @@ package cn.ac.iie.utils.general; -import cn.ac.iie.bean.SessionRecordLog; +import cn.ac.iie.bean.*; import cn.ac.iie.common.FlowWriteConfig; import cn.ac.iie.utils.hbase.HbaseUtils; -import cn.ac.iie.utils.redis.RedisPollUtils; import cn.ac.iie.utils.system.SnowflakeId; -import cn.ac.iie.utils.zookeeper.DistributedLock; -import cn.ac.iie.utils.zookeeper.ZookeeperUtils; import com.alibaba.fastjson.JSONObject; import com.zdjizhi.utils.IpLookup; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; -import redis.clients.jedis.Jedis; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -23,6 +19,7 @@ import java.util.regex.Pattern; * @author qidaijie * @create 2018-08-13 15:11 */ +@SuppressWarnings("all") public class TransFormUtils { private static Logger logger = Logger.getLogger(TransFormUtils.class); private static Pattern WEB_PATTERN = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.cc|\\.me|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)"); @@ -33,51 +30,25 @@ public class TransFormUtils { .loadAsnDataFileV6(FlowWriteConfig.IP_LIBRARY + "asn_v6.mmdb") .build(); - /** - * 解析日志，并补全 - * - * @param message 原始日志 - * @return 补全后的日志 - */ - public static String getJsonMessage(String message) { - SessionRecordLog sessionRecordLog = JSONObject.parseObject(message, SessionRecordLog.class); - String serverIp = sessionRecordLog.getServer_ip(); - String clientIp = sessionRecordLog.getClient_ip(); - try { - sessionRecordLog.setUid(SnowflakeId.generateId()); - sessionRecordLog.setServer_location(ipLookup.countryLookup(serverIp)); - sessionRecordLog.setClient_location(ipLookup.cityLookupDetail(clientIp)); - sessionRecordLog.setClient_asn(ipLookup.asnLookup(clientIp, true)); - sessionRecordLog.setServer_asn(ipLookup.asnLookup(serverIp, true)); - sessionRecordLog.setDomain(getTopDomain(sessionRecordLog.getSni(), sessionRecordLog.getHost())); - sessionRecordLog.setRecv_time(System.currentTimeMillis() / 1000); -// sessionRecordLog.setSubscribe_id(HbaseUtils.getAccount(clientIp)); - return JSONObject.toJSONString(sessionRecordLog); - } catch (Exception e) { - logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); - return ""; - } - } - /** * 解析日志，并补全 - * + * 补subscriber_id，不补domain * @param message radius原始日志 * @return 补全后的日志 */ public static String getRadiusMessage(String message) { - SessionRecordLog sessionRecordLog = JSONObject.parseObject(message, SessionRecordLog.class); - String serverIp = sessionRecordLog.getServer_ip(); - String clientIp = sessionRecordLog.getClient_ip(); + RadiusSessionRecordLog radiusSessionRecordLog = JSONObject.parseObject(message, RadiusSessionRecordLog.class); + String serverIp = radiusSessionRecordLog.getCommon_server_ip(); + String clientIp = radiusSessionRecordLog.getCommon_client_ip(); try { - sessionRecordLog.setUid(SnowflakeId.generateId()); - sessionRecordLog.setServer_location(ipLookup.countryLookup(serverIp)); - sessionRecordLog.setClient_location(ipLookup.cityLookupDetail(clientIp)); - sessionRecordLog.setClient_asn(ipLookup.asnLookup(clientIp, true)); - sessionRecordLog.setServer_asn(ipLookup.asnLookup(serverIp, true)); - sessionRecordLog.setRecv_time(System.currentTimeMillis() / 1000); - return JSONObject.toJSONString(sessionRecordLog); + radiusSessionRecordLog.setCommon_log_id(SnowflakeId.generateId()); + radiusSessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); + radiusSessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); + radiusSessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); + radiusSessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); + radiusSessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); + return JSONObject.toJSONString(radiusSessionRecordLog); } catch (Exception e) { logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); return ""; @@ -85,6 +56,96 @@ public class TransFormUtils { } + + + + + /** + * 解析日志，并补全 + * 补domain,补subscriber_id + * @param message Proxy原始日志 + * @return 补全后的日志 + */ + public static String getProxyMessage(String message) { + ProxySessionRecordLog proxySessionRecordLog = JSONObject.parseObject(message, ProxySessionRecordLog.class); + String serverIp = proxySessionRecordLog.getCommon_server_ip(); + String clientIp = proxySessionRecordLog.getCommon_client_ip(); + try { + proxySessionRecordLog.setCommon_log_id(SnowflakeId.generateId()); + proxySessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); + proxySessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); + proxySessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); + proxySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); + proxySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); + //TODO 集成AAA数据subscribe_id -数据端补全 + proxySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); +// proxySessionRecordLog.setCommon_subscriber_id("aaaaaaa"); + proxySessionRecordLog.setHttp_domain(getTopDomain(null, proxySessionRecordLog.getHttp_host())); + return JSONObject.toJSONString(proxySessionRecordLog); + } catch (Exception e) { + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); + return ""; + } + } + + + + /** + * 解析日志，并补全 + * 补domain,补subscriber_id + * @param message Security原始日志 + * @return 补全后的日志 + */ + public static String getSecurityMessage(String message) { + SecurityPolicyLog securitySessionRecordLog = JSONObject.parseObject(message, SecurityPolicyLog.class); + String serverIp = securitySessionRecordLog.getCommon_server_ip(); + String clientIp = securitySessionRecordLog.getCommon_client_ip(); + try { + securitySessionRecordLog.setCommon_log_id(SnowflakeId.generateId()); + securitySessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); + securitySessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); + securitySessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); + securitySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); + securitySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); + //TODO 集成AAA数据subscribe_id -数据端补全 + securitySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); +// securitySessionRecordLog.setCommon_subscriber_id("aaaaa"); + securitySessionRecordLog.setHttp_domain(getTopDomain(securitySessionRecordLog.getSsl_sni(), securitySessionRecordLog.getHttp_host())); + return JSONObject.toJSONString(securitySessionRecordLog); + } catch (Exception e) { + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); + return ""; + } + } + + /** + * 解析日志，并补全 + * 补domain,补subscriber_id + * @param message Security原始日志 + * @return 补全后的日志 + */ + public static String getCollectProtocolMessage(String message) { + CollectProtocolRecordLog securitySessionRecordLog = JSONObject.parseObject(message, CollectProtocolRecordLog.class); + String serverIp = securitySessionRecordLog.getCommon_server_ip(); + String clientIp = securitySessionRecordLog.getCommon_client_ip(); + try { + securitySessionRecordLog.setCommon_log_id(SnowflakeId.generateId()); + securitySessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); + securitySessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); + securitySessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); + securitySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); + securitySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); + //TODO 集成AAA数据subscribe_id -数据端补全 + securitySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); +// securitySessionRecordLog.setCommon_subscriber_id("aaaaa"); + securitySessionRecordLog.setHttp_domain(getTopDomain(securitySessionRecordLog.getSsl_sni(), securitySessionRecordLog.getHttp_host())); + return JSONObject.toJSONString(securitySessionRecordLog); + } catch (Exception e) { + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); + return ""; + } + } + /** * 有sni通过sni获取域名，有hots根据host获取域名 * @@ -102,24 +163,6 @@ public class TransFormUtils { } } - /** - * 获取用户名 - * - * @param key Sip - * @return SubscribeId - */ - private static String getSubscribeId(String key) { - String sub = ""; - try (Jedis jedis = RedisPollUtils.getJedis()) { - if (jedis != null) { - sub = jedis.get(key); - } - } catch (Exception e) { - logger.error("通过Redis获取用户名出现异常", e); - } - return sub; - } - /** * 根据url截取顶级域名 @@ -140,7 +183,7 @@ public class TransFormUtils { } public static void main(String[] args) { - String s = ipLookup.countryLookup("192.168.10.207"); + String s = ipLookup.cityLookupDetail("2001:470:19:790::24"); System.out.println(s); } diff --git a/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java index c7b05e3..b8fecda 100644 --- a/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java +++ b/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java @@ -18,26 +18,27 @@ import java.util.Map; /** * @author qidaijie */ +@SuppressWarnings("all") public class HbaseUtils { private final static Logger logger = Logger.getLogger(HbaseUtils.class); private static Map subIdMap = new HashMap<>(16); private static Connection connection; private static Long time; -// static { -// // 管理Hbase的配置信息 -// Configuration configuration = HBaseConfiguration.create(); -// // 设置zookeeper节点 -// configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); -// try { -// connection = ConnectionFactory.createConnection(configuration); -// time = System.currentTimeMillis(); -// getAll(); -// } catch (IOException e) { -// logger.error("获取HBase连接失败"); -// e.printStackTrace(); -// } -// } + static { + // 管理Hbase的配置信息 + Configuration configuration = HBaseConfiguration.create(); + // 设置zookeeper节点 + configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); + try { + connection = ConnectionFactory.createConnection(configuration); + time = System.currentTimeMillis(); + getAll(); + } catch (IOException e) { + logger.error("获取HBase连接失败"); + e.printStackTrace(); + } + } /** * 更新变量 diff --git a/src/main/java/cn/ac/iie/utils/redis/RedisClusterUtils.java b/src/main/java/cn/ac/iie/utils/redis/RedisClusterUtils.java deleted file mode 100644 index e7f67d9..0000000 --- a/src/main/java/cn/ac/iie/utils/redis/RedisClusterUtils.java +++ /dev/null @@ -1,79 +0,0 @@ -package cn.ac.iie.utils.redis; - -import cn.ac.iie.common.FlowWriteConfig; -import org.apache.log4j.Logger; -import redis.clients.jedis.HostAndPort; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.JedisPoolConfig; - -import java.io.IOException; -import java.util.LinkedHashSet; -import java.util.Properties; -import java.util.Set; - -/** - * 预用于对准IP对应的用户名的 Redis连接池 - * - * @author my - * @date 2018-07-04 - */ -public final class RedisClusterUtils { - private static final Logger logger = Logger.getLogger(RedisClusterUtils.class); - private static JedisCluster jedisCluster; - private static Properties props = new Properties(); - - static { - try { - String redisConfigFile = "redis_config.properties"; - props.load(RedisClusterUtils.class.getClassLoader().getResourceAsStream(redisConfigFile)); - } catch (IOException e) { - props = null; - logger.error("加载Redis配置文件失败！", e); - } - } - - /** - * 不允许通过new创建该类的实例 - */ - private RedisClusterUtils() { - } - - /** - * 初始化Redis连接池 - */ - private static JedisCluster getJedisCluster() { - if (jedisCluster == null) { - JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXACTIVE))); - poolConfig.setMaxIdle(Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXIDLE))); - poolConfig.setMaxWaitMillis(Long.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXWAIT))); - poolConfig.setTestOnReturn(Boolean.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_TESTONRETURN))); - poolConfig.setTestOnBorrow(Boolean.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_TESTONBORROW))); - Set nodes = new LinkedHashSet(); - for (String port : props.getProperty(FlowWriteConfig.REDIS_PORT).split(FlowWriteConfig.SEGMENTATION)) { - for (String ip : props.getProperty(FlowWriteConfig.REDIS_IP).split(FlowWriteConfig.SEGMENTATION)) { - nodes.add(new HostAndPort(ip, Integer.parseInt(port))); - } - } - jedisCluster = new JedisCluster(nodes, poolConfig); - } - return jedisCluster; - } - - /** - * 获取用户名 - * - * @param key service_ip - * @return Subscribe_id - */ - public static String get(String key) { - String s = key.split("\\.")[0]; - if (!FlowWriteConfig.CHECK_IP_SCOPE.contains(s)) { - jedisCluster = getJedisCluster(); - return jedisCluster.get(key); - } - return ""; - } - - -} diff --git a/src/main/java/cn/ac/iie/utils/redis/RedisPollUtils.java b/src/main/java/cn/ac/iie/utils/redis/RedisPollUtils.java deleted file mode 100644 index 0394378..0000000 --- a/src/main/java/cn/ac/iie/utils/redis/RedisPollUtils.java +++ /dev/null @@ -1,111 +0,0 @@ -package cn.ac.iie.utils.redis; - -import cn.ac.iie.common.FlowWriteConfig; -import com.zdjizhi.utils.StringUtil; -import org.apache.commons.lang3.RandomUtils; -import org.apache.log4j.Logger; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; - -import java.util.Properties; - -/** - * @author qidaijie - */ -public class RedisPollUtils { - private static final Logger logger = Logger.getLogger(RedisPollUtils.class); - private static JedisPool jedisPool = null; - private static Properties props = new Properties(); - - - private RedisPollUtils() { - } - - static { - initialPool(); - - } - - /** - * 初始化Redis连接池 - */ - private static void initialPool() { - try { - //加载连接池配置文件 - props.load(RedisPollUtils.class.getClassLoader().getResourceAsStream("redis_config.properties")); - // 创建jedis池配置实例 - JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXACTIVE))); - poolConfig.setMaxIdle(Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXIDLE))); - poolConfig.setMaxWaitMillis(Long.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_MAXWAIT))); - poolConfig.setTestOnReturn(Boolean.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_TESTONRETURN))); - poolConfig.setTestOnBorrow(Boolean.valueOf(props.getProperty(FlowWriteConfig.REDIS_POOL_TESTONBORROW))); - // 根据配置实例化jedis池 - jedisPool = new JedisPool(poolConfig, props.getProperty(FlowWriteConfig.REDIS_IP), - Integer.valueOf(props.getProperty(FlowWriteConfig.REDIS_PORT)), 3000, "123456"); - } catch (Exception e) { - logger.error("Redis连接池初始化错误", e); - } - } - - /** - * 获取Jedis实例 - * - * @return Jedis实例 - */ - public static Jedis getJedis() { - Jedis jedis = null; - try { - if (jedisPool == null) { - initialPool(); - } - jedis = jedisPool.getResource(); - } catch (Exception e) { - logger.error("Redis连接池错误,无法获取连接", e); - } - return jedis; - } - -// /** -// * @param key redis key -// * @return value -// */ -// public static Integer getWorkerId(String key) { -// int workId = 0; -// int maxId = 32; -// try (Jedis jedis = RedisPollUtils.getJedis()) { -// if (jedis != null) { -// String work = jedis.get(key); -// if (StringUtil.isBlank(work)) { -// jedis.set(key, "0"); -// } else { -// workId = Integer.parseInt(work); -// } -// if (workId < maxId) { -// jedis.set(key, String.valueOf(workId + 1)); -// } else { -// workId = 0; -// jedis.set(key, "1"); -// } -// } -// } catch (Exception e) { -// logger.error("通过Redis获取用户名出现异常", e); -// workId = RandomUtils.nextInt(0, 31); -// } -// return workId; -// } - - public static String getWorkerId(String key) { - String sub = ""; - try (Jedis jedis = RedisPollUtils.getJedis()) { - sub = jedis.get(key); - } catch (Exception e) { - logger.error("通过Redis获取用户名出现异常", e); - - } - return sub; - } - - -} From 360c06fe86fc03f0fb59f34d518758d5fc07e0f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Tue, 12 Nov 2019 14:36:00 +0800 Subject: [PATCH 03/27] =?UTF-8?q?V3=E6=97=A5=E5=BF=97=E8=A1=A5=E5=85=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/cn/ac/iie/common/FlowWriteConfig.java | 13 +------------ .../cn/ac/iie/topology/LogFlowWriteTopology.java | 3 ++- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java index 66e124d..97db894 100644 --- a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java +++ b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java @@ -49,17 +49,6 @@ public class FlowWriteConfig { public static final String IP_LIBRARY = FlowWriteConfigurations.getStringProperty(0, "ip.library"); - - /*** - * Redis - */ - public static final String REDIS_IP = "redis.ip"; - public static final String REDIS_PORT = "redis.port"; - public static final String REDIS_TIMEOUT = "redis.timeout"; - public static final String REDIS_POOL_MAXACTIVE = "redis.pool.maxActive"; - public static final String REDIS_POOL_MAXIDLE = "redis.pool.maxIdle"; - public static final String REDIS_POOL_MAXWAIT = "redis.pool.maxWait"; - public static final String REDIS_POOL_TESTONBORROW = "redis.pool.testOnBorrow"; - public static final String REDIS_POOL_TESTONRETURN = "redis.pool.testOnReturn"; + } \ No newline at end of file diff --git a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java index 80ecdb8..58859ae 100644 --- a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java +++ b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java @@ -1,6 +1,7 @@ package cn.ac.iie.topology; +import cn.ac.iie.bean.CollectProtocolRecordLog; import cn.ac.iie.bolt.NtcLogSendBolt; import cn.ac.iie.bolt.collectProtocol.CollectCompletedBolt; import cn.ac.iie.bolt.radius.RadiusCompletionBolt; @@ -76,7 +77,7 @@ public class LogFlowWriteTopology { builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt"); break; - case "COLLECT-CONNECTION-RECORD-LOG": + case "COLLECT-PROTOCOL-RECORD-LOG": builder.setBolt("CollectCompletedBolt", new CollectCompletedBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("CollectCompletedBolt"); break; From d83f648292f901f10349dcb6410d8ee9de926a5e Mon Sep 17 00:00:00 2001 From: qidaijie Date: Tue, 12 Nov 2019 16:02:50 +0800 Subject: [PATCH 04/27] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- properties/service_flow_config.properties | 16 ++++--- .../ac/iie/bean/CollectProtocolRecordLog.java | 15 ++++--- .../cn/ac/iie/bean/ProxySessionRecordLog.java | 8 +++- .../ac/iie/bean/PublicSessionRecordLog.java | 8 +++- .../ac/iie/bean/RadiusSessionRecordLog.java | 11 +++-- .../cn/ac/iie/bean/SecurityPolicyLog.java | 15 ++++--- .../java/cn/ac/iie/bolt/NtcLogSendBolt.java | 16 +++---- .../collectProtocol/CollectCompletedBolt.java | 13 +++--- .../iie/bolt/proxy/ProxyCompletionBolt.java | 11 ++--- .../iie/bolt/radius/RadiusCompletionBolt.java | 7 ++- .../bolt/security/SecurityCompletionBolt.java | 12 ++--- .../cn/ac/iie/common/FlowWriteConfig.java | 1 + .../ac/iie/topology/LogFlowWriteTopology.java | 24 +++------- .../ac/iie/utils/general/TransFormUtils.java | 44 ++++++++++--------- 14 files changed, 112 insertions(+), 89 deletions(-) diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index e07af38..fb65910 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -17,7 +17,7 @@ auto.offset.reset=latest kafka.topic=SECURITY-POLICY-LOG #读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； -group.id=session-completion-program +group.id=security-policy-191112 #输出topic results.output.topic=SECURITY-POLICY-COMPLETED-LOG @@ -35,16 +35,20 @@ datacenter.bolt.parallelism=1 kafka.bolt.parallelism=1 #定位库地址 -#ip.library=/home/ceiec/topology/dat/ -ip.library=D:/dat/ +ip.library=/home/ceiec/topology/dat/ +#ip.library=D:/dat/ + #kafka批量条数 -batch.insert.num=5000 +batch.insert.num=2000 #数据中心（UID） -data.center.id.num=10 +data.center.id.num=15 #tick时钟频率 -topology.tick.tuple.freq.secs=60 +topology.tick.tuple.freq.secs=5 + +#hbase 更新时间 +hbase.tick.tuple.freq.secs=60 #当bolt性能受限时，限制spout接收速度，理论看ack开启才有效 topology.config.max.spout.pending=150000 diff --git a/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java b/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java index 6131846..43b7d00 100644 --- a/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java +++ b/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java @@ -1,6 +1,11 @@ package cn.ac.iie.bean; -public class CollectProtocolRecordLog extends PublicSessionRecordLog{ +/** + * 除radius之外 + * + * @author qidaijie + */ +public class CollectProtocolRecordLog extends PublicSessionRecordLog { //TODO HTTP协议属性 21 @@ -77,23 +82,23 @@ public class CollectProtocolRecordLog extends PublicSessionRecordLog{ private String ssl_client_side_version; private String ssl_error; -//TODO FTP协议属性 2 + //TODO FTP协议属性 2 private String ftp_url; private String ftp_content; -//TODO BGP协议属性 3 + //TODO BGP协议属性 3 private int bgp_type; private String bgp_as_num; private String bgp_route; -//TODO VOIP协议属性 4 + //TODO VOIP协议属性 4 private String voip_calling_account; private String voip_called_account; private String voip_calling_number; private String voip_called_number; -//TODO STREAMING_MEDIA协议属性 2 + //TODO STREAMING_MEDIA协议属性 2 private String streaming_media_url; private String streaming_media_protocol; diff --git a/src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java b/src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java index 53b8bd7..441205d 100644 --- a/src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java +++ b/src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java @@ -1,6 +1,10 @@ package cn.ac.iie.bean; -import com.alibaba.fastjson.annotation.JSONField; -import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter; + + + +/** + * @author qidaijie + */ public class ProxySessionRecordLog extends PublicSessionRecordLog{ diff --git a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java index 7d853d4..d9e6ced 100644 --- a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java +++ b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java @@ -1,11 +1,15 @@ package cn.ac.iie.bean; -import com.alibaba.fastjson.annotation.JSONField; -import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter; +/** + * 公共类 + * + * @author qidaijie + */ public class PublicSessionRecordLog { //TODO 基础属性 40 int22 + private int common_service; private int common_direction; private long common_recv_time; diff --git a/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java index d467afc..a530da6 100644 --- a/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java +++ b/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java @@ -1,7 +1,12 @@ package cn.ac.iie.bean; -import com.alibaba.fastjson.annotation.JSONField; -import com.alibaba.fastjson.support.spring.annotation.FastJsonFilter; -public class RadiusSessionRecordLog extends PublicSessionRecordLog{ + + +/** + * Radius 日志 + * + * @author qidaijie + */ +public class RadiusSessionRecordLog extends PublicSessionRecordLog { //TODO RADIUS协议属性 4 diff --git a/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java b/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java index 775f1c7..998182f 100644 --- a/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java +++ b/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java @@ -1,6 +1,11 @@ package cn.ac.iie.bean; -public class SecurityPolicyLog extends PublicSessionRecordLog{ +/** + * 策略 + * + * @author qidaijie + */ +public class SecurityPolicyLog extends PublicSessionRecordLog { //TODO HTTP协议属性 21 @@ -77,23 +82,23 @@ public class SecurityPolicyLog extends PublicSessionRecordLog{ private String ssl_client_side_version; private String ssl_error; -//TODO FTP协议属性 2 + //TODO FTP协议属性 2 private String ftp_url; private String ftp_content; -//TODO BGP协议属性 3 + //TODO BGP协议属性 3 private int bgp_type; private String bgp_as_num; private String bgp_route; -//TODO VOIP协议属性 4 + //TODO VOIP协议属性 4 private String voip_calling_account; private String voip_called_account; private String voip_calling_number; private String voip_called_number; -//TODO STREAMING_MEDIA协议属性 2 + //TODO STREAMING_MEDIA协议属性 2 private String streaming_media_url; private String streaming_media_protocol; diff --git a/src/main/java/cn/ac/iie/bolt/NtcLogSendBolt.java b/src/main/java/cn/ac/iie/bolt/NtcLogSendBolt.java index e260447..da4786b 100644 --- a/src/main/java/cn/ac/iie/bolt/NtcLogSendBolt.java +++ b/src/main/java/cn/ac/iie/bolt/NtcLogSendBolt.java @@ -24,11 +24,11 @@ import java.util.Map; * @date 2018/8/14 */ public class NtcLogSendBolt extends BaseBasicBolt { - private static final long serialVersionUID = 3940515789830317517L; + private static final long serialVersionUID = -3663610927224396615L; private static Logger logger = Logger.getLogger(NtcLogSendBolt.class); private List list; private KafkaLogNtc kafkaLogNtc; - private static long successfulSum = 0; +// private static long successfulSum = 0; @Override @@ -43,11 +43,11 @@ public class NtcLogSendBolt extends BaseBasicBolt { if (TupleUtils.isTick(tuple)) { if (list.size() != 0) { kafkaLogNtc.sendMessage(list); - successfulSum += list.size(); +// successfulSum += list.size(); list.clear(); } - basicOutputCollector.emit(new Values(successfulSum)); - successfulSum = 0L; +// basicOutputCollector.emit(new Values(successfulSum)); +// successfulSum = 0L; } else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { @@ -55,12 +55,12 @@ public class NtcLogSendBolt extends BaseBasicBolt { } if (list.size() == FlowWriteConfig.BATCH_INSERT_NUM) { kafkaLogNtc.sendMessage(list); - successfulSum += list.size(); +// successfulSum += list.size(); list.clear(); } } } catch (Exception e) { - logger.error("日志发送Kafka过程出现异常"); + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志发送Kafka过程出现异常"); e.printStackTrace(); } } @@ -74,7 +74,7 @@ public class NtcLogSendBolt extends BaseBasicBolt { @Override public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { - outputFieldsDeclarer.declare(new Fields("suc")); +// outputFieldsDeclarer.declare(new Fields("suc")); } } diff --git a/src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java b/src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java index 1cd40fa..c619be1 100644 --- a/src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java +++ b/src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java @@ -1,4 +1,4 @@ -package cn.ac.iie.bolt.collectProtocol; +package cn.ac.iie.bolt.collectprotocol; import cn.ac.iie.common.FlowWriteConfig; import cn.ac.iie.utils.system.TupleUtils; @@ -23,10 +23,10 @@ import static cn.ac.iie.utils.hbase.HbaseUtils.change; * * @author qidaijie */ -@SuppressWarnings("all") + public class CollectCompletedBolt extends BaseBasicBolt { - private static final long serialVersionUID = -105915167013846589L; private final static Logger logger = Logger.getLogger(CollectCompletedBolt.class); + private static final long serialVersionUID = 4682827168247333522L; @Override public void prepare(Map stormConf, TopologyContext context) { @@ -38,14 +38,15 @@ public class CollectCompletedBolt extends BaseBasicBolt { try { if (TupleUtils.isTick(tuple)) { change(); - }else { + } else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { basicOutputCollector.emit(new Values(getCollectProtocolMessage(message))); } } } catch (Exception e) { - logger.error("接收解析过程出现异常", e); + logger.error(FlowWriteConfig.KAFKA_TOPIC + "接收/解析过程出现异常"); + e.printStackTrace(); } } @@ -53,7 +54,7 @@ public class CollectCompletedBolt extends BaseBasicBolt { public Map getComponentConfiguration() { Map conf = new HashMap(16); conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, - FlowWriteConfig.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + FlowWriteConfig.HBASE_TICK_TUPLE_FREQ_SECS); return conf; } diff --git a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java index 123095a..838702f 100644 --- a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java @@ -23,10 +23,10 @@ import static cn.ac.iie.utils.hbase.HbaseUtils.change; * * @author qidaijie */ -@SuppressWarnings("all") + public class ProxyCompletionBolt extends BaseBasicBolt { - private static final long serialVersionUID = -1059151670838465894L; private final static Logger logger = Logger.getLogger(ProxyCompletionBolt.class); + private static final long serialVersionUID = 6097654428594885032L; @Override public void prepare(Map stormConf, TopologyContext context) { @@ -38,14 +38,15 @@ public class ProxyCompletionBolt extends BaseBasicBolt { try { if (TupleUtils.isTick(tuple)) { change(); - }else { + } else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { basicOutputCollector.emit(new Values(getProxyMessage(message))); } } } catch (Exception e) { - logger.error("接收解析过程出现异常", e); + logger.error(FlowWriteConfig.KAFKA_TOPIC + "接收/解析过程出现异常"); + e.printStackTrace(); } } @@ -53,7 +54,7 @@ public class ProxyCompletionBolt extends BaseBasicBolt { public Map getComponentConfiguration() { Map conf = new HashMap(16); conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, - FlowWriteConfig.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + FlowWriteConfig.HBASE_TICK_TUPLE_FREQ_SECS); return conf; } diff --git a/src/main/java/cn/ac/iie/bolt/radius/RadiusCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/radius/RadiusCompletionBolt.java index 9b6ebde..7771e74 100644 --- a/src/main/java/cn/ac/iie/bolt/radius/RadiusCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/radius/RadiusCompletionBolt.java @@ -1,5 +1,6 @@ package cn.ac.iie.bolt.radius; +import cn.ac.iie.common.FlowWriteConfig; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; import org.apache.storm.task.TopologyContext; @@ -20,8 +21,9 @@ import static cn.ac.iie.utils.general.TransFormUtils.getRadiusMessage; * @author qidaijie */ public class RadiusCompletionBolt extends BaseBasicBolt { - private static final long serialVersionUID = -3657802387129063952L; + private final static Logger logger = Logger.getLogger(RadiusCompletionBolt.class); + private static final long serialVersionUID = -3657802387129063952L; @Override public void prepare(Map stormConf, TopologyContext context) { @@ -36,7 +38,8 @@ public class RadiusCompletionBolt extends BaseBasicBolt { basicOutputCollector.emit(new Values(getRadiusMessage(message))); } } catch (Exception e) { - logger.error("接收解析过程出现异常", e); + logger.error(FlowWriteConfig.KAFKA_TOPIC + "接收/解析过程出现异常"); + e.printStackTrace(); } } diff --git a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java index ebb33b7..4947ccd 100644 --- a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java @@ -23,10 +23,11 @@ import static cn.ac.iie.utils.general.TransFormUtils.getSecurityMessage; * * @author qidaijie */ -@SuppressWarnings("all") + public class SecurityCompletionBolt extends BaseBasicBolt { - private static final long serialVersionUID = -1059151770138464874L; + private final static Logger logger = Logger.getLogger(SecurityCompletionBolt.class); + private static final long serialVersionUID = -2380858260054733989L; @Override public void prepare(Map stormConf, TopologyContext context) { @@ -38,14 +39,15 @@ public class SecurityCompletionBolt extends BaseBasicBolt { try { if (TupleUtils.isTick(tuple)) { HbaseUtils.change(); - }else { + } else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { basicOutputCollector.emit(new Values(getSecurityMessage(message))); } } } catch (Exception e) { - logger.error("接收解析过程出现异常", e); + logger.error(FlowWriteConfig.KAFKA_TOPIC + "接收/解析过程出现异常"); + e.printStackTrace(); } } @@ -53,7 +55,7 @@ public class SecurityCompletionBolt extends BaseBasicBolt { public Map getComponentConfiguration() { Map conf = new HashMap(16); conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, - FlowWriteConfig.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + FlowWriteConfig.HBASE_TICK_TUPLE_FREQ_SECS); return conf; } diff --git a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java index 97db894..3313144 100644 --- a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java +++ b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java @@ -20,6 +20,7 @@ public class FlowWriteConfig { public static final Integer TOPOLOGY_WORKERS = FlowWriteConfigurations.getIntProperty(0, "topology.workers"); public static final Integer KAFKA_BOLT_PARALLELISM = FlowWriteConfigurations.getIntProperty(0, "kafka.bolt.parallelism"); public static final Integer TOPOLOGY_TICK_TUPLE_FREQ_SECS = FlowWriteConfigurations.getIntProperty(0, "topology.tick.tuple.freq.secs"); + public static final Integer HBASE_TICK_TUPLE_FREQ_SECS = FlowWriteConfigurations.getIntProperty(0, "hbase.tick.tuple.freq.secs"); public static final Integer TOPOLOGY_CONFIG_MAX_SPOUT_PENDING = FlowWriteConfigurations.getIntProperty(0, "topology.config.max.spout.pending"); public static final Integer TOPOLOGY_NUM_ACKS = FlowWriteConfigurations.getIntProperty(0, "topology.num.acks"); public static final Integer TOPOLOGY_SPOUT_SLEEP_TIME = FlowWriteConfigurations.getIntProperty(0, "topology.spout.sleep.time"); diff --git a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java index 58859ae..639a523 100644 --- a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java +++ b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java @@ -1,9 +1,8 @@ package cn.ac.iie.topology; -import cn.ac.iie.bean.CollectProtocolRecordLog; import cn.ac.iie.bolt.NtcLogSendBolt; -import cn.ac.iie.bolt.collectProtocol.CollectCompletedBolt; +import cn.ac.iie.bolt.collectprotocol.CollectCompletedBolt; import cn.ac.iie.bolt.radius.RadiusCompletionBolt; import cn.ac.iie.bolt.security.SecurityCompletionBolt; @@ -64,9 +63,7 @@ public class LogFlowWriteTopology { builder = new TopologyBuilder(); builder.setSpout("LogFlowWriteSpout", new CustomizedKafkaSpout(), FlowWriteConfig.SPOUT_PARALLELISM); - String topic_name = FlowWriteConfig.KAFKA_TOPIC; - - switch(topic_name){ + switch (FlowWriteConfig.KAFKA_TOPIC) { case "PROXY-POLICY-LOG": builder.setBolt("ProxyCompletionBolt", new ProxyCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ProxyCompletionBolt"); @@ -87,22 +84,11 @@ public class LogFlowWriteTopology { builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("SecurityCompletionBolt"); break; - - } - - /*if ("PROXY_POLICY_LOG".equals(FlowWriteConfig.KAFKA_TOPIC) || "COLLECT_HTTP_META_LOG".equals(FlowWriteConfig.KAFKA_TOPIC)) { - builder.setBolt("HttpCompletionBolt", new HttpCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); - builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("HttpCompletionBolt"); - } else if ("COLLECT_RADIUS_RECORD_LOG".equals(FlowWriteConfig.KAFKA_TOPIC)) { - builder.setBolt("RadiusCompletionBolt", new ConnCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); - builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt"); - } else { - builder.setBolt("ConnCompletionBolt", new ConnCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); - builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ConnCompletionBolt"); - }*/ -// builder.setBolt("SummaryBolt", new SummaryBolt(), 1).localOrShuffleGrouping("NtcLogSendBolt"); + default: } + } + public static void main(String[] args) throws Exception { LogFlowWriteTopology csst = null; boolean runLocally = true; diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index c515689..4801314 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -19,7 +19,7 @@ import java.util.regex.Pattern; * @author qidaijie * @create 2018-08-13 15:11 */ -@SuppressWarnings("all") + public class TransFormUtils { private static Logger logger = Logger.getLogger(TransFormUtils.class); private static Pattern WEB_PATTERN = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.cc|\\.me|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)"); @@ -34,6 +34,7 @@ public class TransFormUtils { /** * 解析日志，并补全 * 补subscriber_id，不补domain + * * @param message radius原始日志 * @return 补全后的日志 */ @@ -50,19 +51,17 @@ public class TransFormUtils { radiusSessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); return JSONObject.toJSONString(radiusSessionRecordLog); } catch (Exception e) { - logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常"); + e.printStackTrace(); return ""; } } - - - - /** * 解析日志，并补全 * 补domain,补subscriber_id + * * @param message Proxy原始日志 * @return 补全后的日志 */ @@ -83,16 +82,17 @@ public class TransFormUtils { proxySessionRecordLog.setHttp_domain(getTopDomain(null, proxySessionRecordLog.getHttp_host())); return JSONObject.toJSONString(proxySessionRecordLog); } catch (Exception e) { - logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常"); + e.printStackTrace(); return ""; } } - /** * 解析日志，并补全 * 补domain,补subscriber_id + * * @param message Security原始日志 * @return 补全后的日志 */ @@ -113,7 +113,8 @@ public class TransFormUtils { securitySessionRecordLog.setHttp_domain(getTopDomain(securitySessionRecordLog.getSsl_sni(), securitySessionRecordLog.getHttp_host())); return JSONObject.toJSONString(securitySessionRecordLog); } catch (Exception e) { - logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常"); + e.printStackTrace(); return ""; } } @@ -121,25 +122,26 @@ public class TransFormUtils { /** * 解析日志，并补全 * 补domain,补subscriber_id + * * @param message Security原始日志 * @return 补全后的日志 */ public static String getCollectProtocolMessage(String message) { - CollectProtocolRecordLog securitySessionRecordLog = JSONObject.parseObject(message, CollectProtocolRecordLog.class); - String serverIp = securitySessionRecordLog.getCommon_server_ip(); - String clientIp = securitySessionRecordLog.getCommon_client_ip(); + CollectProtocolRecordLog collectProtocolRecordLog = JSONObject.parseObject(message, CollectProtocolRecordLog.class); + String serverIp = collectProtocolRecordLog.getCommon_server_ip(); + String clientIp = collectProtocolRecordLog.getCommon_client_ip(); try { - securitySessionRecordLog.setCommon_log_id(SnowflakeId.generateId()); - securitySessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); - securitySessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); - securitySessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); - securitySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); - securitySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); + collectProtocolRecordLog.setCommon_log_id(SnowflakeId.generateId()); + collectProtocolRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); + collectProtocolRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); + collectProtocolRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); + collectProtocolRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); + collectProtocolRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); //TODO 集成AAA数据subscribe_id -数据端补全 - securitySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); + collectProtocolRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); // securitySessionRecordLog.setCommon_subscriber_id("aaaaa"); - securitySessionRecordLog.setHttp_domain(getTopDomain(securitySessionRecordLog.getSsl_sni(), securitySessionRecordLog.getHttp_host())); - return JSONObject.toJSONString(securitySessionRecordLog); + collectProtocolRecordLog.setHttp_domain(getTopDomain(collectProtocolRecordLog.getSsl_sni(), collectProtocolRecordLog.getHttp_host())); + return JSONObject.toJSONString(collectProtocolRecordLog); } catch (Exception e) { logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); return ""; From 56af3618cde058556da3e689bedc4c70ad69a9da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Tue, 12 Nov 2019 16:21:37 +0800 Subject: [PATCH 05/27] =?UTF-8?q?radius=E6=97=A5=E5=BF=97=E8=A1=A5?= =?UTF-8?q?=E5=85=A8=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/compiler.xml | 16 ++ .idea/encodings.xml | 6 + .idea/libraries/Maven__asm_asm_3_1.xml | 13 ++ .../Maven__com_101tec_zkclient_0_10.xml | 13 ++ .../Maven__com_alibaba_fastjson_1_2_47.xml | 13 ++ ...Maven__com_esotericsoftware_kryo_3_0_3.xml | 13 ++ ...ven__com_esotericsoftware_minlog_1_3_0.xml | 13 ++ ...com_esotericsoftware_reflectasm_1_10_1.xml | 13 ++ ...jackson_core_jackson_annotations_2_9_0.xml | 13 ++ ...terxml_jackson_core_jackson_core_2_9_1.xml | 13 ++ ...ml_jackson_core_jackson_databind_2_9_1.xml | 13 ++ ..._findbugs_findbugs_annotations_1_3_9_1.xml | 13 ++ ..._com_google_code_findbugs_jsr305_3_0_0.xml | 13 ++ ...Maven__com_google_code_gson_gson_2_2_4.xml | 13 ++ .../Maven__com_google_guava_guava_18_0.xml | 13 ++ ...om_google_protobuf_protobuf_java_2_5_0.xml | 13 ++ ...googlecode_json_simple_json_simple_1_1.xml | 13 ++ ...m_jamesmurty_utils_java_xmlbuilder_0_4.xml | 13 ++ .../Maven__com_jcraft_jsch_0_1_42.xml | 13 ++ .../Maven__com_lmax_disruptor_3_3_2.xml | 13 ++ ...Maven__com_maxmind_db_maxmind_db_1_2_2.xml | 13 ++ ...aven__com_maxmind_geoip2_geoip2_2_12_0.xml | 13 ++ ...ven__com_maxmind_geoip_geoip_api_1_3_1.xml | 13 ++ ...aven__com_squareup_okhttp_okhttp_2_4_0.xml | 13 ++ .../Maven__com_squareup_okio_okio_1_4_0.xml | 13 ++ ...__com_squareup_retrofit_retrofit_1_9_0.xml | 13 ++ .../Maven__com_sun_jersey_jersey_core_1_9.xml | 13 ++ .../Maven__com_sun_jersey_jersey_json_1_9.xml | 13 ++ ...aven__com_sun_jersey_jersey_server_1_9.xml | 13 ++ ...en__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml | 13 ++ ...m_thoughtworks_paranamer_paranamer_2_3.xml | 13 ++ ..._com_yammer_metrics_metrics_core_2_2_0.xml | 13 ++ .../Maven__com_zdjizhi_galaxy_1_0_1.xml | 13 ++ ...mons_beanutils_commons_beanutils_1_7_0.xml | 13 ++ ...beanutils_commons_beanutils_core_1_8_0.xml | 13 ++ .../Maven__commons_cli_commons_cli_1_2.xml | 13 ++ ...Maven__commons_codec_commons_codec_1_9.xml | 13 ++ ..._collections_commons_collections_3_2_2.xml | 13 ++ ...onfiguration_commons_configuration_1_6.xml | 13 ++ ...__commons_daemon_commons_daemon_1_0_13.xml | 13 ++ ..._commons_digester_commons_digester_1_8.xml | 13 ++ .../Maven__commons_el_commons_el_1_0.xml | 13 ++ ...mons_httpclient_commons_httpclient_3_1.xml | 13 ++ .../Maven__commons_io_commons_io_2_5.xml | 13 ++ .../Maven__commons_lang_commons_lang_2_5.xml | 13 ++ ...n__commons_logging_commons_logging_1_2.xml | 13 ++ .../Maven__commons_net_commons_net_3_1.xml | 13 ++ ..._dropwizard_metrics_metrics_core_3_1_2.xml | 13 ++ .../Maven__io_netty_netty_3_10_5_Final.xml | 13 ++ .../Maven__io_netty_netty_all_4_1_8_Final.xml | 13 ++ ...Maven__javax_activation_activation_1_1.xml | 13 ++ .../Maven__javax_servlet_jsp_jsp_api_2_1.xml | 13 ++ .../Maven__javax_servlet_servlet_api_2_5.xml | 13 ++ .../Maven__javax_xml_bind_jaxb_api_2_2_2.xml | 13 ++ ...Maven__javax_xml_stream_stax_api_1_0_2.xml | 13 ++ .idea/libraries/Maven__jline_jline_0_9_94.xml | 13 ++ .../Maven__joda_time_joda_time_2_10.xml | 13 ++ .idea/libraries/Maven__junit_junit_4_12.xml | 13 ++ .idea/libraries/Maven__log4j_log4j_1_2_14.xml | 13 ++ ...aven__net_java_dev_jets3t_jets3t_0_9_0.xml | 13 ++ ...__net_sf_jopt_simple_jopt_simple_5_0_4.xml | 13 ++ .../Maven__org_apache_avro_avro_1_7_4.xml | 13 ++ ..._apache_commons_commons_compress_1_4_1.xml | 13 ++ ...__org_apache_commons_commons_lang3_3_4.xml | 13 ++ ...org_apache_commons_commons_math3_3_1_1.xml | 13 ++ ...n__org_apache_commons_commons_math_2_2.xml | 13 ++ ...rg_apache_curator_curator_client_2_7_1.xml | 13 ++ ...pache_curator_curator_framework_2_10_0.xml | 13 ++ ...g_apache_curator_curator_recipes_2_7_1.xml | 13 ++ ...e_directory_api_api_asn1_api_1_0_0_M20.xml | 13 ++ ...pache_directory_api_api_util_1_0_0_M20.xml | 13 ++ ...rectory_server_apacheds_i18n_2_0_0_M15.xml | 13 ++ ...rver_apacheds_kerberos_codec_2_0_0_M15.xml | 13 ++ ...apache_hadoop_hadoop_annotations_2_7_1.xml | 13 ++ ...n__org_apache_hadoop_hadoop_auth_2_7_4.xml | 13 ++ ..._org_apache_hadoop_hadoop_client_2_7_1.xml | 13 ++ ..._org_apache_hadoop_hadoop_common_2_7_1.xml | 13 ++ ...n__org_apache_hadoop_hadoop_hdfs_2_7_1.xml | 13 ++ ...doop_hadoop_mapreduce_client_app_2_7_1.xml | 13 ++ ...p_hadoop_mapreduce_client_common_2_7_1.xml | 13 ++ ...oop_hadoop_mapreduce_client_core_2_7_4.xml | 13 ++ ...adoop_mapreduce_client_jobclient_2_7_1.xml | 13 ++ ..._hadoop_mapreduce_client_shuffle_2_7_1.xml | 13 ++ ...rg_apache_hadoop_hadoop_yarn_api_2_7_1.xml | 13 ++ ...apache_hadoop_hadoop_yarn_client_2_7_1.xml | 13 ++ ...apache_hadoop_hadoop_yarn_common_2_7_4.xml | 13 ++ ...hadoop_hadoop_yarn_server_common_2_7_1.xml | 13 ++ ...g_apache_hbase_hbase_annotations_1_4_9.xml | 13 ++ ...n__org_apache_hbase_hbase_client_1_4_9.xml | 13 ++ ...n__org_apache_hbase_hbase_common_1_4_9.xml | 13 ++ ..._apache_hbase_hbase_common_tests_1_4_9.xml | 13 ++ ...pache_hbase_hbase_hadoop2_compat_1_4_9.xml | 13 ++ ...apache_hbase_hbase_hadoop_compat_1_4_9.xml | 13 ++ ...__org_apache_hbase_hbase_metrics_1_4_9.xml | 13 ++ ...g_apache_hbase_hbase_metrics_api_1_4_9.xml | 13 ++ ...g_apache_hbase_hbase_prefix_tree_1_4_9.xml | 13 ++ ...org_apache_hbase_hbase_procedure_1_4_9.xml | 13 ++ ..._org_apache_hbase_hbase_protocol_1_4_9.xml | 13 ++ ...n__org_apache_hbase_hbase_server_1_4_9.xml | 13 ++ ...he_htrace_htrace_core_3_1_0_incubating.xml | 13 ++ ...apache_httpcomponents_httpclient_4_5_2.xml | 13 ++ ...g_apache_httpcomponents_httpcore_4_4_4.xml | 13 ++ ...ven__org_apache_kafka_kafka_2_11_1_0_0.xml | 13 ++ ...__org_apache_kafka_kafka_clients_1_0_0.xml | 13 ++ ...org_apache_logging_log4j_log4j_api_2_1.xml | 13 ++ ...rg_apache_logging_log4j_log4j_core_2_1.xml | 13 ++ ...che_logging_log4j_log4j_slf4j_impl_2_1.xml | 13 ++ ...ven__org_apache_storm_storm_core_1_0_2.xml | 13 ++ ...en__org_apache_storm_storm_kafka_1_0_2.xml | 13 ++ ...__org_apache_zookeeper_zookeeper_3_4_9.xml | 13 ++ .../Maven__org_clojure_clojure_1_7_0.xml | 13 ++ ...dehaus_jackson_jackson_core_asl_1_9_13.xml | 13 ++ ..._codehaus_jackson_jackson_jaxrs_1_9_13.xml | 13 ++ ...haus_jackson_jackson_mapper_asl_1_9_13.xml | 13 ++ ..._org_codehaus_jackson_jackson_xc_1_8_3.xml | 13 ++ ...en__org_codehaus_jettison_jettison_1_1.xml | 13 ++ ...sesource_leveldbjni_leveldbjni_all_1_8.xml | 13 ++ .../Maven__org_hamcrest_hamcrest_core_1_3.xml | 13 ++ .../Maven__org_influxdb_influxdb_java_2_1.xml | 13 ++ .../Maven__org_jamon_jamon_runtime_2_4_1.xml | 13 ++ ...ven__org_jruby_jcodings_jcodings_1_0_8.xml | 13 ++ .../Maven__org_jruby_joni_joni_2_1_2.xml | 13 ++ .../libraries/Maven__org_lz4_lz4_java_1_4.xml | 13 ++ .../Maven__org_mortbay_jetty_jetty_6_1_26.xml | 13 ++ ...g_mortbay_jetty_jetty_sslengine_6_1_26.xml | 13 ++ ...n__org_mortbay_jetty_jetty_util_6_1_26.xml | 13 ++ ...aven__org_mortbay_jetty_jsp_2_1_6_1_14.xml | 13 ++ ...__org_mortbay_jetty_jsp_api_2_1_6_1_14.xml | 13 ++ ...g_mortbay_jetty_servlet_api_2_5_6_1_14.xml | 13 ++ .../Maven__org_objenesis_objenesis_2_1.xml | 13 ++ .../Maven__org_ow2_asm_asm_5_0_3.xml | 13 ++ ...__org_scala_lang_scala_library_2_11_11.xml | 13 ++ .../Maven__org_slf4j_slf4j_api_1_7_7.xml | 13 ++ .idea/libraries/Maven__org_tukaani_xz_1_0.xml | 13 ++ ...n__org_xerial_snappy_snappy_java_1_1_4.xml | 13 ++ .../Maven__tomcat_jasper_compiler_5_5_23.xml | 13 ++ .../Maven__tomcat_jasper_runtime_5_5_23.xml | 13 ++ .../Maven__xerces_xercesImpl_2_9_1.xml | 13 ++ .../Maven__xml_apis_xml_apis_1_3_04.xml | 13 ++ .idea/libraries/Maven__xmlenc_xmlenc_0_52.xml | 13 ++ .idea/log-stream-completion.iml | 162 ++++++++++++++++++ .idea/misc.xml | 14 ++ .idea/modules.xml | 8 + .idea/uiDesigner.xml | 124 ++++++++++++++ .idea/vcs.xml | 6 + properties/core-site.xml | 64 ------- .../cn/ac/iie/common/FlowWriteConfig.java | 2 +- src/test/java/cn/ac/iie/test/TestThread.java | 4 +- src/test/java/cn/ac/iie/test/test.java | 12 +- 149 files changed, 2136 insertions(+), 76 deletions(-) create mode 100644 .idea/compiler.xml create mode 100644 .idea/encodings.xml create mode 100644 .idea/libraries/Maven__asm_asm_3_1.xml create mode 100644 .idea/libraries/Maven__com_101tec_zkclient_0_10.xml create mode 100644 .idea/libraries/Maven__com_alibaba_fastjson_1_2_47.xml create mode 100644 .idea/libraries/Maven__com_esotericsoftware_kryo_3_0_3.xml create mode 100644 .idea/libraries/Maven__com_esotericsoftware_minlog_1_3_0.xml create mode 100644 .idea/libraries/Maven__com_esotericsoftware_reflectasm_1_10_1.xml create mode 100644 .idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_0.xml create mode 100644 .idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_9_1.xml create mode 100644 .idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_9_1.xml create mode 100644 .idea/libraries/Maven__com_github_stephenc_findbugs_findbugs_annotations_1_3_9_1.xml create mode 100644 .idea/libraries/Maven__com_google_code_findbugs_jsr305_3_0_0.xml create mode 100644 .idea/libraries/Maven__com_google_code_gson_gson_2_2_4.xml create mode 100644 .idea/libraries/Maven__com_google_guava_guava_18_0.xml create mode 100644 .idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml create mode 100644 .idea/libraries/Maven__com_googlecode_json_simple_json_simple_1_1.xml create mode 100644 .idea/libraries/Maven__com_jamesmurty_utils_java_xmlbuilder_0_4.xml create mode 100644 .idea/libraries/Maven__com_jcraft_jsch_0_1_42.xml create mode 100644 .idea/libraries/Maven__com_lmax_disruptor_3_3_2.xml create mode 100644 .idea/libraries/Maven__com_maxmind_db_maxmind_db_1_2_2.xml create mode 100644 .idea/libraries/Maven__com_maxmind_geoip2_geoip2_2_12_0.xml create mode 100644 .idea/libraries/Maven__com_maxmind_geoip_geoip_api_1_3_1.xml create mode 100644 .idea/libraries/Maven__com_squareup_okhttp_okhttp_2_4_0.xml create mode 100644 .idea/libraries/Maven__com_squareup_okio_okio_1_4_0.xml create mode 100644 .idea/libraries/Maven__com_squareup_retrofit_retrofit_1_9_0.xml create mode 100644 .idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml create mode 100644 .idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml create mode 100644 .idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml create mode 100644 .idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml create mode 100644 .idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml create mode 100644 .idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml create mode 100644 .idea/libraries/Maven__com_zdjizhi_galaxy_1_0_1.xml create mode 100644 .idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml create mode 100644 .idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml create mode 100644 .idea/libraries/Maven__commons_cli_commons_cli_1_2.xml create mode 100644 .idea/libraries/Maven__commons_codec_commons_codec_1_9.xml create mode 100644 .idea/libraries/Maven__commons_collections_commons_collections_3_2_2.xml create mode 100644 .idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml create mode 100644 .idea/libraries/Maven__commons_daemon_commons_daemon_1_0_13.xml create mode 100644 .idea/libraries/Maven__commons_digester_commons_digester_1_8.xml create mode 100644 .idea/libraries/Maven__commons_el_commons_el_1_0.xml create mode 100644 .idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml create mode 100644 .idea/libraries/Maven__commons_io_commons_io_2_5.xml create mode 100644 .idea/libraries/Maven__commons_lang_commons_lang_2_5.xml create mode 100644 .idea/libraries/Maven__commons_logging_commons_logging_1_2.xml create mode 100644 .idea/libraries/Maven__commons_net_commons_net_3_1.xml create mode 100644 .idea/libraries/Maven__io_dropwizard_metrics_metrics_core_3_1_2.xml create mode 100644 .idea/libraries/Maven__io_netty_netty_3_10_5_Final.xml create mode 100644 .idea/libraries/Maven__io_netty_netty_all_4_1_8_Final.xml create mode 100644 .idea/libraries/Maven__javax_activation_activation_1_1.xml create mode 100644 .idea/libraries/Maven__javax_servlet_jsp_jsp_api_2_1.xml create mode 100644 .idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml create mode 100644 .idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml create mode 100644 .idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml create mode 100644 .idea/libraries/Maven__jline_jline_0_9_94.xml create mode 100644 .idea/libraries/Maven__joda_time_joda_time_2_10.xml create mode 100644 .idea/libraries/Maven__junit_junit_4_12.xml create mode 100644 .idea/libraries/Maven__log4j_log4j_1_2_14.xml create mode 100644 .idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_9_0.xml create mode 100644 .idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml create mode 100644 .idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml create mode 100644 .idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml create mode 100644 .idea/libraries/Maven__org_apache_commons_commons_lang3_3_4.xml create mode 100644 .idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml create mode 100644 .idea/libraries/Maven__org_apache_commons_commons_math_2_2.xml create mode 100644 .idea/libraries/Maven__org_apache_curator_curator_client_2_7_1.xml create mode 100644 .idea/libraries/Maven__org_apache_curator_curator_framework_2_10_0.xml create mode 100644 .idea/libraries/Maven__org_apache_curator_curator_recipes_2_7_1.xml create mode 100644 .idea/libraries/Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml create mode 100644 .idea/libraries/Maven__org_apache_directory_api_api_util_1_0_0_M20.xml create mode 100644 .idea/libraries/Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml create mode 100644 .idea/libraries/Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_7_1.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_7_4.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_client_2_7_1.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_7_1.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_7_1.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_7_1.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_7_1.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_7_4.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_7_1.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_7_1.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_7_1.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_7_1.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_7_4.xml create mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_7_1.xml create mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_annotations_1_4_9.xml create mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_client_1_4_9.xml create mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_common_1_4_9.xml create mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_common_tests_1_4_9.xml create mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_hadoop2_compat_1_4_9.xml create mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_hadoop_compat_1_4_9.xml create mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_metrics_1_4_9.xml create mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_metrics_api_1_4_9.xml create mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_prefix_tree_1_4_9.xml create mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_procedure_1_4_9.xml create mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_protocol_1_4_9.xml create mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_server_1_4_9.xml create mode 100644 .idea/libraries/Maven__org_apache_htrace_htrace_core_3_1_0_incubating.xml create mode 100644 .idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_5_2.xml create mode 100644 .idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_4_4.xml create mode 100644 .idea/libraries/Maven__org_apache_kafka_kafka_2_11_1_0_0.xml create mode 100644 .idea/libraries/Maven__org_apache_kafka_kafka_clients_1_0_0.xml create mode 100644 .idea/libraries/Maven__org_apache_logging_log4j_log4j_api_2_1.xml create mode 100644 .idea/libraries/Maven__org_apache_logging_log4j_log4j_core_2_1.xml create mode 100644 .idea/libraries/Maven__org_apache_logging_log4j_log4j_slf4j_impl_2_1.xml create mode 100644 .idea/libraries/Maven__org_apache_storm_storm_core_1_0_2.xml create mode 100644 .idea/libraries/Maven__org_apache_storm_storm_kafka_1_0_2.xml create mode 100644 .idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_9.xml create mode 100644 .idea/libraries/Maven__org_clojure_clojure_1_7_0.xml create mode 100644 .idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_9_13.xml create mode 100644 .idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_9_13.xml create mode 100644 .idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_13.xml create mode 100644 .idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml create mode 100644 .idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml create mode 100644 .idea/libraries/Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml create mode 100644 .idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml create mode 100644 .idea/libraries/Maven__org_influxdb_influxdb_java_2_1.xml create mode 100644 .idea/libraries/Maven__org_jamon_jamon_runtime_2_4_1.xml create mode 100644 .idea/libraries/Maven__org_jruby_jcodings_jcodings_1_0_8.xml create mode 100644 .idea/libraries/Maven__org_jruby_joni_joni_2_1_2.xml create mode 100644 .idea/libraries/Maven__org_lz4_lz4_java_1_4.xml create mode 100644 .idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml create mode 100644 .idea/libraries/Maven__org_mortbay_jetty_jetty_sslengine_6_1_26.xml create mode 100644 .idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml create mode 100644 .idea/libraries/Maven__org_mortbay_jetty_jsp_2_1_6_1_14.xml create mode 100644 .idea/libraries/Maven__org_mortbay_jetty_jsp_api_2_1_6_1_14.xml create mode 100644 .idea/libraries/Maven__org_mortbay_jetty_servlet_api_2_5_6_1_14.xml create mode 100644 .idea/libraries/Maven__org_objenesis_objenesis_2_1.xml create mode 100644 .idea/libraries/Maven__org_ow2_asm_asm_5_0_3.xml create mode 100644 .idea/libraries/Maven__org_scala_lang_scala_library_2_11_11.xml create mode 100644 .idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml create mode 100644 .idea/libraries/Maven__org_tukaani_xz_1_0.xml create mode 100644 .idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_4.xml create mode 100644 .idea/libraries/Maven__tomcat_jasper_compiler_5_5_23.xml create mode 100644 .idea/libraries/Maven__tomcat_jasper_runtime_5_5_23.xml create mode 100644 .idea/libraries/Maven__xerces_xercesImpl_2_9_1.xml create mode 100644 .idea/libraries/Maven__xml_apis_xml_apis_1_3_04.xml create mode 100644 .idea/libraries/Maven__xmlenc_xmlenc_0_52.xml create mode 100644 .idea/log-stream-completion.iml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/uiDesigner.xml create mode 100644 .idea/vcs.xml delete mode 100644 properties/core-site.xml diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..7603751 --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..e79da7e --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__asm_asm_3_1.xml b/.idea/libraries/Maven__asm_asm_3_1.xml new file mode 100644 index 0000000..3386f10 --- /dev/null +++ b/.idea/libraries/Maven__asm_asm_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_101tec_zkclient_0_10.xml b/.idea/libraries/Maven__com_101tec_zkclient_0_10.xml new file mode 100644 index 0000000..a2b08a2 --- /dev/null +++ b/.idea/libraries/Maven__com_101tec_zkclient_0_10.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_alibaba_fastjson_1_2_47.xml b/.idea/libraries/Maven__com_alibaba_fastjson_1_2_47.xml new file mode 100644 index 0000000..dcd6ee5 --- /dev/null +++ b/.idea/libraries/Maven__com_alibaba_fastjson_1_2_47.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_esotericsoftware_kryo_3_0_3.xml b/.idea/libraries/Maven__com_esotericsoftware_kryo_3_0_3.xml new file mode 100644 index 0000000..17a5c28 --- /dev/null +++ b/.idea/libraries/Maven__com_esotericsoftware_kryo_3_0_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_esotericsoftware_minlog_1_3_0.xml b/.idea/libraries/Maven__com_esotericsoftware_minlog_1_3_0.xml new file mode 100644 index 0000000..0f2bb63 --- /dev/null +++ b/.idea/libraries/Maven__com_esotericsoftware_minlog_1_3_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_esotericsoftware_reflectasm_1_10_1.xml b/.idea/libraries/Maven__com_esotericsoftware_reflectasm_1_10_1.xml new file mode 100644 index 0000000..5ea60d9 --- /dev/null +++ b/.idea/libraries/Maven__com_esotericsoftware_reflectasm_1_10_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_0.xml b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_0.xml new file mode 100644 index 0000000..06441f4 --- /dev/null +++ b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_9_1.xml b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_9_1.xml new file mode 100644 index 0000000..7719a19 --- /dev/null +++ b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_9_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_9_1.xml b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_9_1.xml new file mode 100644 index 0000000..c197771 --- /dev/null +++ b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_9_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_github_stephenc_findbugs_findbugs_annotations_1_3_9_1.xml b/.idea/libraries/Maven__com_github_stephenc_findbugs_findbugs_annotations_1_3_9_1.xml new file mode 100644 index 0000000..d1d9a7c --- /dev/null +++ b/.idea/libraries/Maven__com_github_stephenc_findbugs_findbugs_annotations_1_3_9_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_code_findbugs_jsr305_3_0_0.xml b/.idea/libraries/Maven__com_google_code_findbugs_jsr305_3_0_0.xml new file mode 100644 index 0000000..6644d12 --- /dev/null +++ b/.idea/libraries/Maven__com_google_code_findbugs_jsr305_3_0_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_code_gson_gson_2_2_4.xml b/.idea/libraries/Maven__com_google_code_gson_gson_2_2_4.xml new file mode 100644 index 0000000..4533c1b --- /dev/null +++ b/.idea/libraries/Maven__com_google_code_gson_gson_2_2_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_guava_guava_18_0.xml b/.idea/libraries/Maven__com_google_guava_guava_18_0.xml new file mode 100644 index 0000000..bbd71d7 --- /dev/null +++ b/.idea/libraries/Maven__com_google_guava_guava_18_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml b/.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml new file mode 100644 index 0000000..c06d3a1 --- /dev/null +++ b/.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_googlecode_json_simple_json_simple_1_1.xml b/.idea/libraries/Maven__com_googlecode_json_simple_json_simple_1_1.xml new file mode 100644 index 0000000..ea70fe1 --- /dev/null +++ b/.idea/libraries/Maven__com_googlecode_json_simple_json_simple_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_jamesmurty_utils_java_xmlbuilder_0_4.xml b/.idea/libraries/Maven__com_jamesmurty_utils_java_xmlbuilder_0_4.xml new file mode 100644 index 0000000..a2a0c04 --- /dev/null +++ b/.idea/libraries/Maven__com_jamesmurty_utils_java_xmlbuilder_0_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_jcraft_jsch_0_1_42.xml b/.idea/libraries/Maven__com_jcraft_jsch_0_1_42.xml new file mode 100644 index 0000000..880be15 --- /dev/null +++ b/.idea/libraries/Maven__com_jcraft_jsch_0_1_42.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_lmax_disruptor_3_3_2.xml b/.idea/libraries/Maven__com_lmax_disruptor_3_3_2.xml new file mode 100644 index 0000000..c22270a --- /dev/null +++ b/.idea/libraries/Maven__com_lmax_disruptor_3_3_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_maxmind_db_maxmind_db_1_2_2.xml b/.idea/libraries/Maven__com_maxmind_db_maxmind_db_1_2_2.xml new file mode 100644 index 0000000..1342872 --- /dev/null +++ b/.idea/libraries/Maven__com_maxmind_db_maxmind_db_1_2_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_maxmind_geoip2_geoip2_2_12_0.xml b/.idea/libraries/Maven__com_maxmind_geoip2_geoip2_2_12_0.xml new file mode 100644 index 0000000..1471229 --- /dev/null +++ b/.idea/libraries/Maven__com_maxmind_geoip2_geoip2_2_12_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_maxmind_geoip_geoip_api_1_3_1.xml b/.idea/libraries/Maven__com_maxmind_geoip_geoip_api_1_3_1.xml new file mode 100644 index 0000000..bd438ab --- /dev/null +++ b/.idea/libraries/Maven__com_maxmind_geoip_geoip_api_1_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_squareup_okhttp_okhttp_2_4_0.xml b/.idea/libraries/Maven__com_squareup_okhttp_okhttp_2_4_0.xml new file mode 100644 index 0000000..be09ac3 --- /dev/null +++ b/.idea/libraries/Maven__com_squareup_okhttp_okhttp_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_squareup_okio_okio_1_4_0.xml b/.idea/libraries/Maven__com_squareup_okio_okio_1_4_0.xml new file mode 100644 index 0000000..ddacf0f --- /dev/null +++ b/.idea/libraries/Maven__com_squareup_okio_okio_1_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_squareup_retrofit_retrofit_1_9_0.xml b/.idea/libraries/Maven__com_squareup_retrofit_retrofit_1_9_0.xml new file mode 100644 index 0000000..86047ed --- /dev/null +++ b/.idea/libraries/Maven__com_squareup_retrofit_retrofit_1_9_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml b/.idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml new file mode 100644 index 0000000..abd3e5d --- /dev/null +++ b/.idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml b/.idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml new file mode 100644 index 0000000..fff6aea --- /dev/null +++ b/.idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml b/.idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml new file mode 100644 index 0000000..717467f --- /dev/null +++ b/.idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml b/.idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml new file mode 100644 index 0000000..1165a3b --- /dev/null +++ b/.idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml b/.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml new file mode 100644 index 0000000..3807eb2 --- /dev/null +++ b/.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml b/.idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml new file mode 100644 index 0000000..fc159c5 --- /dev/null +++ b/.idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_zdjizhi_galaxy_1_0_1.xml b/.idea/libraries/Maven__com_zdjizhi_galaxy_1_0_1.xml new file mode 100644 index 0000000..5ec91ab --- /dev/null +++ b/.idea/libraries/Maven__com_zdjizhi_galaxy_1_0_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml b/.idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml new file mode 100644 index 0000000..1cb64ed --- /dev/null +++ b/.idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml b/.idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml new file mode 100644 index 0000000..3fda2c0 --- /dev/null +++ b/.idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml b/.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml new file mode 100644 index 0000000..cec2493 --- /dev/null +++ b/.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_codec_commons_codec_1_9.xml b/.idea/libraries/Maven__commons_codec_commons_codec_1_9.xml new file mode 100644 index 0000000..a66d039 --- /dev/null +++ b/.idea/libraries/Maven__commons_codec_commons_codec_1_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_collections_commons_collections_3_2_2.xml b/.idea/libraries/Maven__commons_collections_commons_collections_3_2_2.xml new file mode 100644 index 0000000..13afda2 --- /dev/null +++ b/.idea/libraries/Maven__commons_collections_commons_collections_3_2_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml b/.idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml new file mode 100644 index 0000000..ae96781 --- /dev/null +++ b/.idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_daemon_commons_daemon_1_0_13.xml b/.idea/libraries/Maven__commons_daemon_commons_daemon_1_0_13.xml new file mode 100644 index 0000000..d44fc67 --- /dev/null +++ b/.idea/libraries/Maven__commons_daemon_commons_daemon_1_0_13.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_digester_commons_digester_1_8.xml b/.idea/libraries/Maven__commons_digester_commons_digester_1_8.xml new file mode 100644 index 0000000..49f1def --- /dev/null +++ b/.idea/libraries/Maven__commons_digester_commons_digester_1_8.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_el_commons_el_1_0.xml b/.idea/libraries/Maven__commons_el_commons_el_1_0.xml new file mode 100644 index 0000000..598d1b0 --- /dev/null +++ b/.idea/libraries/Maven__commons_el_commons_el_1_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml b/.idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml new file mode 100644 index 0000000..66e6537 --- /dev/null +++ b/.idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_io_commons_io_2_5.xml b/.idea/libraries/Maven__commons_io_commons_io_2_5.xml new file mode 100644 index 0000000..67c2ad2 --- /dev/null +++ b/.idea/libraries/Maven__commons_io_commons_io_2_5.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_lang_commons_lang_2_5.xml b/.idea/libraries/Maven__commons_lang_commons_lang_2_5.xml new file mode 100644 index 0000000..055afe5 --- /dev/null +++ b/.idea/libraries/Maven__commons_lang_commons_lang_2_5.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_logging_commons_logging_1_2.xml b/.idea/libraries/Maven__commons_logging_commons_logging_1_2.xml new file mode 100644 index 0000000..eab40b3 --- /dev/null +++ b/.idea/libraries/Maven__commons_logging_commons_logging_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_net_commons_net_3_1.xml b/.idea/libraries/Maven__commons_net_commons_net_3_1.xml new file mode 100644 index 0000000..dc40661 --- /dev/null +++ b/.idea/libraries/Maven__commons_net_commons_net_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__io_dropwizard_metrics_metrics_core_3_1_2.xml b/.idea/libraries/Maven__io_dropwizard_metrics_metrics_core_3_1_2.xml new file mode 100644 index 0000000..e25f69c --- /dev/null +++ b/.idea/libraries/Maven__io_dropwizard_metrics_metrics_core_3_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__io_netty_netty_3_10_5_Final.xml b/.idea/libraries/Maven__io_netty_netty_3_10_5_Final.xml new file mode 100644 index 0000000..d1cfe80 --- /dev/null +++ b/.idea/libraries/Maven__io_netty_netty_3_10_5_Final.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__io_netty_netty_all_4_1_8_Final.xml b/.idea/libraries/Maven__io_netty_netty_all_4_1_8_Final.xml new file mode 100644 index 0000000..7f19a44 --- /dev/null +++ b/.idea/libraries/Maven__io_netty_netty_all_4_1_8_Final.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_activation_activation_1_1.xml b/.idea/libraries/Maven__javax_activation_activation_1_1.xml new file mode 100644 index 0000000..180d587 --- /dev/null +++ b/.idea/libraries/Maven__javax_activation_activation_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_servlet_jsp_jsp_api_2_1.xml b/.idea/libraries/Maven__javax_servlet_jsp_jsp_api_2_1.xml new file mode 100644 index 0000000..4b5f67f --- /dev/null +++ b/.idea/libraries/Maven__javax_servlet_jsp_jsp_api_2_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml b/.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml new file mode 100644 index 0000000..679e09a --- /dev/null +++ b/.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml b/.idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml new file mode 100644 index 0000000..a3054d2 --- /dev/null +++ b/.idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml b/.idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml new file mode 100644 index 0000000..2a4dd7a --- /dev/null +++ b/.idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__jline_jline_0_9_94.xml b/.idea/libraries/Maven__jline_jline_0_9_94.xml new file mode 100644 index 0000000..903c339 --- /dev/null +++ b/.idea/libraries/Maven__jline_jline_0_9_94.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__joda_time_joda_time_2_10.xml b/.idea/libraries/Maven__joda_time_joda_time_2_10.xml new file mode 100644 index 0000000..9274557 --- /dev/null +++ b/.idea/libraries/Maven__joda_time_joda_time_2_10.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__junit_junit_4_12.xml b/.idea/libraries/Maven__junit_junit_4_12.xml new file mode 100644 index 0000000..d411041 --- /dev/null +++ b/.idea/libraries/Maven__junit_junit_4_12.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__log4j_log4j_1_2_14.xml b/.idea/libraries/Maven__log4j_log4j_1_2_14.xml new file mode 100644 index 0000000..2825a67 --- /dev/null +++ b/.idea/libraries/Maven__log4j_log4j_1_2_14.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_9_0.xml b/.idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_9_0.xml new file mode 100644 index 0000000..f4ff312 --- /dev/null +++ b/.idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_9_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml b/.idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml new file mode 100644 index 0000000..1ef5173 --- /dev/null +++ b/.idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml b/.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml new file mode 100644 index 0000000..cc03056 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml b/.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml new file mode 100644 index 0000000..190209e --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_4.xml b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_4.xml new file mode 100644 index 0000000..78cfcd3 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml b/.idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml new file mode 100644 index 0000000..5627f0c --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_math_2_2.xml b/.idea/libraries/Maven__org_apache_commons_commons_math_2_2.xml new file mode 100644 index 0000000..8f3e315 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_math_2_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_curator_curator_client_2_7_1.xml b/.idea/libraries/Maven__org_apache_curator_curator_client_2_7_1.xml new file mode 100644 index 0000000..18c48a6 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_curator_curator_client_2_7_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_curator_curator_framework_2_10_0.xml b/.idea/libraries/Maven__org_apache_curator_curator_framework_2_10_0.xml new file mode 100644 index 0000000..e7280b8 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_curator_curator_framework_2_10_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_curator_curator_recipes_2_7_1.xml b/.idea/libraries/Maven__org_apache_curator_curator_recipes_2_7_1.xml new file mode 100644 index 0000000..3f3e79a --- /dev/null +++ b/.idea/libraries/Maven__org_apache_curator_curator_recipes_2_7_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml b/.idea/libraries/Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml new file mode 100644 index 0000000..bd5ac37 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_directory_api_api_util_1_0_0_M20.xml b/.idea/libraries/Maven__org_apache_directory_api_api_util_1_0_0_M20.xml new file mode 100644 index 0000000..b3d86af --- /dev/null +++ b/.idea/libraries/Maven__org_apache_directory_api_api_util_1_0_0_M20.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml b/.idea/libraries/Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml new file mode 100644 index 0000000..7a73f45 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml b/.idea/libraries/Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml new file mode 100644 index 0000000..5378216 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_7_1.xml new file mode 100644 index 0000000..106c941 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_7_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_7_4.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_7_4.xml new file mode 100644 index 0000000..a424dd6 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_7_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_client_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_client_2_7_1.xml new file mode 100644 index 0000000..780c61b --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_client_2_7_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_7_1.xml new file mode 100644 index 0000000..0d6c002 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_7_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_7_1.xml new file mode 100644 index 0000000..80663c5 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_7_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_7_1.xml new file mode 100644 index 0000000..17526fe --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_7_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_7_1.xml new file mode 100644 index 0000000..d5185d1 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_7_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_7_4.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_7_4.xml new file mode 100644 index 0000000..81d159d --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_7_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_7_1.xml new file mode 100644 index 0000000..e57a978 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_7_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_7_1.xml new file mode 100644 index 0000000..8a29199 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_7_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_7_1.xml new file mode 100644 index 0000000..9ac03d9 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_7_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_7_1.xml new file mode 100644 index 0000000..0f9e524 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_7_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_7_4.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_7_4.xml new file mode 100644 index 0000000..6d4f20c --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_7_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_7_1.xml new file mode 100644 index 0000000..07bbc3c --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_7_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_annotations_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_annotations_1_4_9.xml new file mode 100644 index 0000000..1ddeff6 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hbase_hbase_annotations_1_4_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_client_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_client_1_4_9.xml new file mode 100644 index 0000000..6e08e98 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hbase_hbase_client_1_4_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_common_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_common_1_4_9.xml new file mode 100644 index 0000000..8fa4d1a --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hbase_hbase_common_1_4_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_common_tests_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_common_tests_1_4_9.xml new file mode 100644 index 0000000..d390357 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hbase_hbase_common_tests_1_4_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_hadoop2_compat_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_hadoop2_compat_1_4_9.xml new file mode 100644 index 0000000..6555370 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hbase_hbase_hadoop2_compat_1_4_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_hadoop_compat_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_hadoop_compat_1_4_9.xml new file mode 100644 index 0000000..bc8c736 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hbase_hbase_hadoop_compat_1_4_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_metrics_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_metrics_1_4_9.xml new file mode 100644 index 0000000..c53e03c --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hbase_hbase_metrics_1_4_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_metrics_api_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_metrics_api_1_4_9.xml new file mode 100644 index 0000000..c1578ad --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hbase_hbase_metrics_api_1_4_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_prefix_tree_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_prefix_tree_1_4_9.xml new file mode 100644 index 0000000..0c2803e --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hbase_hbase_prefix_tree_1_4_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_procedure_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_procedure_1_4_9.xml new file mode 100644 index 0000000..56da90c --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hbase_hbase_procedure_1_4_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_protocol_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_protocol_1_4_9.xml new file mode 100644 index 0000000..e53de29 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hbase_hbase_protocol_1_4_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_server_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_server_1_4_9.xml new file mode 100644 index 0000000..53856e3 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_hbase_hbase_server_1_4_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_htrace_htrace_core_3_1_0_incubating.xml b/.idea/libraries/Maven__org_apache_htrace_htrace_core_3_1_0_incubating.xml new file mode 100644 index 0000000..08aeeed --- /dev/null +++ b/.idea/libraries/Maven__org_apache_htrace_htrace_core_3_1_0_incubating.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_5_2.xml b/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_5_2.xml new file mode 100644 index 0000000..fdb7ead --- /dev/null +++ b/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_5_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_4_4.xml b/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_4_4.xml new file mode 100644 index 0000000..3a5aa19 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_4_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_kafka_kafka_2_11_1_0_0.xml b/.idea/libraries/Maven__org_apache_kafka_kafka_2_11_1_0_0.xml new file mode 100644 index 0000000..cad83b3 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_kafka_kafka_2_11_1_0_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_kafka_kafka_clients_1_0_0.xml b/.idea/libraries/Maven__org_apache_kafka_kafka_clients_1_0_0.xml new file mode 100644 index 0000000..25e3efc --- /dev/null +++ b/.idea/libraries/Maven__org_apache_kafka_kafka_clients_1_0_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_logging_log4j_log4j_api_2_1.xml b/.idea/libraries/Maven__org_apache_logging_log4j_log4j_api_2_1.xml new file mode 100644 index 0000000..e0f672a --- /dev/null +++ b/.idea/libraries/Maven__org_apache_logging_log4j_log4j_api_2_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_logging_log4j_log4j_core_2_1.xml b/.idea/libraries/Maven__org_apache_logging_log4j_log4j_core_2_1.xml new file mode 100644 index 0000000..28ef981 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_logging_log4j_log4j_core_2_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_logging_log4j_log4j_slf4j_impl_2_1.xml b/.idea/libraries/Maven__org_apache_logging_log4j_log4j_slf4j_impl_2_1.xml new file mode 100644 index 0000000..f29d4cf --- /dev/null +++ b/.idea/libraries/Maven__org_apache_logging_log4j_log4j_slf4j_impl_2_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_storm_storm_core_1_0_2.xml b/.idea/libraries/Maven__org_apache_storm_storm_core_1_0_2.xml new file mode 100644 index 0000000..ac5ed57 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_storm_storm_core_1_0_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_storm_storm_kafka_1_0_2.xml b/.idea/libraries/Maven__org_apache_storm_storm_kafka_1_0_2.xml new file mode 100644 index 0000000..9617e32 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_storm_storm_kafka_1_0_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_9.xml b/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_9.xml new file mode 100644 index 0000000..24de43f --- /dev/null +++ b/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_clojure_clojure_1_7_0.xml b/.idea/libraries/Maven__org_clojure_clojure_1_7_0.xml new file mode 100644 index 0000000..3e944f3 --- /dev/null +++ b/.idea/libraries/Maven__org_clojure_clojure_1_7_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_9_13.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_9_13.xml new file mode 100644 index 0000000..98eb549 --- /dev/null +++ b/.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_9_13.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_9_13.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_9_13.xml new file mode 100644 index 0000000..43fc6ed --- /dev/null +++ b/.idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_9_13.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_13.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_13.xml new file mode 100644 index 0000000..77f3bad --- /dev/null +++ b/.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_13.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml new file mode 100644 index 0000000..9fc3894 --- /dev/null +++ b/.idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml b/.idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml new file mode 100644 index 0000000..d62802e --- /dev/null +++ b/.idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml b/.idea/libraries/Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml new file mode 100644 index 0000000..4a829d2 --- /dev/null +++ b/.idea/libraries/Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml b/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml new file mode 100644 index 0000000..f58bbc1 --- /dev/null +++ b/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_influxdb_influxdb_java_2_1.xml b/.idea/libraries/Maven__org_influxdb_influxdb_java_2_1.xml new file mode 100644 index 0000000..a6c5b54 --- /dev/null +++ b/.idea/libraries/Maven__org_influxdb_influxdb_java_2_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_jamon_jamon_runtime_2_4_1.xml b/.idea/libraries/Maven__org_jamon_jamon_runtime_2_4_1.xml new file mode 100644 index 0000000..8858b4f --- /dev/null +++ b/.idea/libraries/Maven__org_jamon_jamon_runtime_2_4_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_jruby_jcodings_jcodings_1_0_8.xml b/.idea/libraries/Maven__org_jruby_jcodings_jcodings_1_0_8.xml new file mode 100644 index 0000000..7e032d7 --- /dev/null +++ b/.idea/libraries/Maven__org_jruby_jcodings_jcodings_1_0_8.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_jruby_joni_joni_2_1_2.xml b/.idea/libraries/Maven__org_jruby_joni_joni_2_1_2.xml new file mode 100644 index 0000000..a557ab1 --- /dev/null +++ b/.idea/libraries/Maven__org_jruby_joni_joni_2_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_lz4_lz4_java_1_4.xml b/.idea/libraries/Maven__org_lz4_lz4_java_1_4.xml new file mode 100644 index 0000000..8cf9aff --- /dev/null +++ b/.idea/libraries/Maven__org_lz4_lz4_java_1_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml b/.idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml new file mode 100644 index 0000000..0f5d0b8 --- /dev/null +++ b/.idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jetty_sslengine_6_1_26.xml b/.idea/libraries/Maven__org_mortbay_jetty_jetty_sslengine_6_1_26.xml new file mode 100644 index 0000000..4720600 --- /dev/null +++ b/.idea/libraries/Maven__org_mortbay_jetty_jetty_sslengine_6_1_26.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml b/.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml new file mode 100644 index 0000000..f6ec2b9 --- /dev/null +++ b/.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jsp_2_1_6_1_14.xml b/.idea/libraries/Maven__org_mortbay_jetty_jsp_2_1_6_1_14.xml new file mode 100644 index 0000000..cc33d9f --- /dev/null +++ b/.idea/libraries/Maven__org_mortbay_jetty_jsp_2_1_6_1_14.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jsp_api_2_1_6_1_14.xml b/.idea/libraries/Maven__org_mortbay_jetty_jsp_api_2_1_6_1_14.xml new file mode 100644 index 0000000..590bc87 --- /dev/null +++ b/.idea/libraries/Maven__org_mortbay_jetty_jsp_api_2_1_6_1_14.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_servlet_api_2_5_6_1_14.xml b/.idea/libraries/Maven__org_mortbay_jetty_servlet_api_2_5_6_1_14.xml new file mode 100644 index 0000000..5536a9c --- /dev/null +++ b/.idea/libraries/Maven__org_mortbay_jetty_servlet_api_2_5_6_1_14.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_objenesis_objenesis_2_1.xml b/.idea/libraries/Maven__org_objenesis_objenesis_2_1.xml new file mode 100644 index 0000000..7ab319b --- /dev/null +++ b/.idea/libraries/Maven__org_objenesis_objenesis_2_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_ow2_asm_asm_5_0_3.xml b/.idea/libraries/Maven__org_ow2_asm_asm_5_0_3.xml new file mode 100644 index 0000000..da69f6c --- /dev/null +++ b/.idea/libraries/Maven__org_ow2_asm_asm_5_0_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_11.xml b/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_11.xml new file mode 100644 index 0000000..1553078 --- /dev/null +++ b/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_11.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml new file mode 100644 index 0000000..1e67260 --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_tukaani_xz_1_0.xml b/.idea/libraries/Maven__org_tukaani_xz_1_0.xml new file mode 100644 index 0000000..8b8042f --- /dev/null +++ b/.idea/libraries/Maven__org_tukaani_xz_1_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_4.xml b/.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_4.xml new file mode 100644 index 0000000..33526ab --- /dev/null +++ b/.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__tomcat_jasper_compiler_5_5_23.xml b/.idea/libraries/Maven__tomcat_jasper_compiler_5_5_23.xml new file mode 100644 index 0000000..4610fa2 --- /dev/null +++ b/.idea/libraries/Maven__tomcat_jasper_compiler_5_5_23.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__tomcat_jasper_runtime_5_5_23.xml b/.idea/libraries/Maven__tomcat_jasper_runtime_5_5_23.xml new file mode 100644 index 0000000..e80db89 --- /dev/null +++ b/.idea/libraries/Maven__tomcat_jasper_runtime_5_5_23.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__xerces_xercesImpl_2_9_1.xml b/.idea/libraries/Maven__xerces_xercesImpl_2_9_1.xml new file mode 100644 index 0000000..19d39db --- /dev/null +++ b/.idea/libraries/Maven__xerces_xercesImpl_2_9_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__xml_apis_xml_apis_1_3_04.xml b/.idea/libraries/Maven__xml_apis_xml_apis_1_3_04.xml new file mode 100644 index 0000000..891caa2 --- /dev/null +++ b/.idea/libraries/Maven__xml_apis_xml_apis_1_3_04.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__xmlenc_xmlenc_0_52.xml b/.idea/libraries/Maven__xmlenc_xmlenc_0_52.xml new file mode 100644 index 0000000..59d176d --- /dev/null +++ b/.idea/libraries/Maven__xmlenc_xmlenc_0_52.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/log-stream-completion.iml b/.idea/log-stream-completion.iml new file mode 100644 index 0000000..52db4e7 --- /dev/null +++ b/.idea/log-stream-completion.iml @@ -0,0 +1,162 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..fa89799 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,14 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..9d425bb --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml new file mode 100644 index 0000000..e96534f --- /dev/null +++ b/.idea/uiDesigner.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/properties/core-site.xml b/properties/core-site.xml deleted file mode 100644 index 66742f0..0000000 --- a/properties/core-site.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - - - - - - fs.defaultFS - hdfs://ns1 - - - hadoop.tmp.dir - file:/opt/hadoop/tmp - - - io.file.buffer.size - 131702 - - - hadoop.proxyuser.root.hosts - * - - - hadoop.proxyuser.root.groups - * - - - hadoop.logfile.size - 10000000 - The max size of each log file - - - - hadoop.logfile.count - 1 - The max number of log files - - - ha.zookeeper.quorum - 192.168.40.202:2181,192.168.40.203:2181,192.168.40.206:2181 - - -io.compression.codecs -com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec - - -io.compression.codec.lzo.class -com.hadoop.compression.lzo.LzoCodec - - - diff --git a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java index 97db894..fedf7cf 100644 --- a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java +++ b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java @@ -49,6 +49,6 @@ public class FlowWriteConfig { public static final String IP_LIBRARY = FlowWriteConfigurations.getStringProperty(0, "ip.library"); - + } \ No newline at end of file diff --git a/src/test/java/cn/ac/iie/test/TestThread.java b/src/test/java/cn/ac/iie/test/TestThread.java index 5938b8a..89aa655 100644 --- a/src/test/java/cn/ac/iie/test/TestThread.java +++ b/src/test/java/cn/ac/iie/test/TestThread.java @@ -1,3 +1,4 @@ +/* package cn.ac.iie.test; @@ -10,7 +11,7 @@ class RunnableDemo implements Runnable { private static ZookeeperUtils zookeeperUtils = new ZookeeperUtils(); static { - zookeeperUtils.connectZookeeper("192.168.40.207:2181"); + zookeeperUtils.connectZookeeper(); } @Override @@ -47,3 +48,4 @@ public class TestThread { } } +*/ diff --git a/src/test/java/cn/ac/iie/test/test.java b/src/test/java/cn/ac/iie/test/test.java index 705a0f7..f65cd5f 100644 --- a/src/test/java/cn/ac/iie/test/test.java +++ b/src/test/java/cn/ac/iie/test/test.java @@ -1,24 +1,16 @@ package cn.ac.iie.test; -import cn.ac.iie.bean.SessionRecordLog; -import cn.ac.iie.common.FlowWriteConfig; -import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; -import com.alibaba.fastjson.serializer.SerializerFeature; -import com.zdjizhi.utils.IpLookup; import org.junit.Test; -import javax.servlet.http.HttpServletRequest; -import java.math.BigInteger; -import java.util.Arrays; import java.util.HashMap; import java.util.Map; public class test { public static void main(String[] args) { String message = "{\"str_ea_m-t-r-a-ceid\":\"JSON\",\"uid\":\"0\"}"; - SessionRecordLog sessionRecordLog = JSONObject.parseObject(message, SessionRecordLog.class); - System.out.println(JSONObject.toJSONString(sessionRecordLog)); +// SessionRecordLog sessionRecordLog = JSONObject.parseObject(message, SessionRecordLog.class); +// System.out.println(JSONObject.toJSONString(sessionRecordLog)); } @Test From 0464e9cde54c420c1e1d7d479813c4c18a461b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Tue, 12 Nov 2019 17:39:20 +0800 Subject: [PATCH 06/27] =?UTF-8?q?=E6=97=A5=E5=BF=97=E8=A1=A5=E5=85=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/ac/iie/bean/PublicSessionRecordLog.java | 2 +- .../cn/ac/iie/bean/RadiusSessionRecordLog.java | 14 ++++++-------- .../CollectCompletedBolt.java | 2 +- .../cn/ac/iie/topology/LogFlowWriteTopology.java | 2 +- .../cn/ac/iie/utils/general/TransFormUtils.java | 5 +---- 5 files changed, 10 insertions(+), 15 deletions(-) rename src/main/java/cn/ac/iie/bolt/{collectProtocol => collect}/CollectCompletedBolt.java (98%) diff --git a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java index d9e6ced..6b852cd 100644 --- a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java +++ b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java @@ -8,7 +8,7 @@ package cn.ac.iie.bean; */ public class PublicSessionRecordLog { -//TODO 基础属性 40 int22 +//TODO 基础属性 41 int22 private int common_service; private int common_direction; diff --git a/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java index a530da6..d4f7679 100644 --- a/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java +++ b/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java @@ -9,12 +9,17 @@ package cn.ac.iie.bean; public class RadiusSessionRecordLog extends PublicSessionRecordLog { -//TODO RADIUS协议属性 4 +//TODO RADIUS协议属性 8 private int radius_session_timeout; private int radius_idle_timeout; private int radius_acct_status_type; private int radius_acct_terminate_cause; + private int radius_packet_type; + private String radius_nas_ip; + private String radius_framed_ip; + private String radius_account; + public int getRadius_session_timeout() { return radius_session_timeout; @@ -48,13 +53,6 @@ public class RadiusSessionRecordLog extends PublicSessionRecordLog { this.radius_acct_terminate_cause = radius_acct_terminate_cause; } - private int radius_packet_type; - - private String radius_nas_ip; - private String radius_framed_ip; - private String radius_account; - - public int getRadius_packet_type() { return radius_packet_type; } diff --git a/src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java b/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java similarity index 98% rename from src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java rename to src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java index c619be1..2e08d2e 100644 --- a/src/main/java/cn/ac/iie/bolt/collectProtocol/CollectCompletedBolt.java +++ b/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java @@ -1,4 +1,4 @@ -package cn.ac.iie.bolt.collectprotocol; +package cn.ac.iie.bolt.collect; import cn.ac.iie.common.FlowWriteConfig; import cn.ac.iie.utils.system.TupleUtils; diff --git a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java index 639a523..9f3a54d 100644 --- a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java +++ b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java @@ -2,7 +2,7 @@ package cn.ac.iie.topology; import cn.ac.iie.bolt.NtcLogSendBolt; -import cn.ac.iie.bolt.collectprotocol.CollectCompletedBolt; +import cn.ac.iie.bolt.collect.CollectCompletedBolt; import cn.ac.iie.bolt.radius.RadiusCompletionBolt; import cn.ac.iie.bolt.security.SecurityCompletionBolt; diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index 4801314..66f5c49 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -78,7 +78,6 @@ public class TransFormUtils { proxySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); //TODO 集成AAA数据subscribe_id -数据端补全 proxySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); -// proxySessionRecordLog.setCommon_subscriber_id("aaaaaaa"); proxySessionRecordLog.setHttp_domain(getTopDomain(null, proxySessionRecordLog.getHttp_host())); return JSONObject.toJSONString(proxySessionRecordLog); } catch (Exception e) { @@ -109,7 +108,6 @@ public class TransFormUtils { securitySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); //TODO 集成AAA数据subscribe_id -数据端补全 securitySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); -// securitySessionRecordLog.setCommon_subscriber_id("aaaaa"); securitySessionRecordLog.setHttp_domain(getTopDomain(securitySessionRecordLog.getSsl_sni(), securitySessionRecordLog.getHttp_host())); return JSONObject.toJSONString(securitySessionRecordLog); } catch (Exception e) { @@ -123,7 +121,7 @@ public class TransFormUtils { * 解析日志，并补全 * 补domain,补subscriber_id * - * @param message Security原始日志 + * @param message CollectProtocol原始日志 * @return 补全后的日志 */ public static String getCollectProtocolMessage(String message) { @@ -139,7 +137,6 @@ public class TransFormUtils { collectProtocolRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); //TODO 集成AAA数据subscribe_id -数据端补全 collectProtocolRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); -// securitySessionRecordLog.setCommon_subscriber_id("aaaaa"); collectProtocolRecordLog.setHttp_domain(getTopDomain(collectProtocolRecordLog.getSsl_sni(), collectProtocolRecordLog.getHttp_host())); return JSONObject.toJSONString(collectProtocolRecordLog); } catch (Exception e) { From 89cf61d0a924394d5a742e1a90e9a9222e8e7bc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Tue, 12 Nov 2019 17:42:11 +0800 Subject: [PATCH 07/27] Deleted .idea/libraries/Maven__asm_asm_3_1.xml, .idea/libraries/Maven__com_101tec_zkclient_0_10.xml, .idea/libraries/Maven__com_alibaba_fastjson_1_2_47.xml, .idea/libraries/Maven__com_esotericsoftware_kryo_3_0_3.xml, .idea/libraries/Maven__com_esotericsoftware_minlog_1_3_0.xml, .idea/libraries/Maven__com_esotericsoftware_reflectasm_1_10_1.xml, .idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_0.xml, .idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_9_1.xml, .idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_9_1.xml, .idea/libraries/Maven__com_github_stephenc_findbugs_findbugs_annotations_1_3_9_1.xml, .idea/libraries/Maven__com_google_code_findbugs_jsr305_3_0_0.xml, .idea/libraries/Maven__com_google_code_gson_gson_2_2_4.xml, .idea/libraries/Maven__com_google_guava_guava_18_0.xml, .idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml, .idea/libraries/Maven__com_googlecode_json_simple_json_simple_1_1.xml, .idea/libraries/Maven__com_jamesmurty_utils_java_xmlbuilder_0_4.xml, .idea/libraries/Maven__com_jcraft_jsch_0_1_42.xml, .idea/libraries/Maven__com_lmax_disruptor_3_3_2.xml, .idea/libraries/Maven__com_maxmind_db_maxmind_db_1_2_2.xml, .idea/libraries/Maven__com_maxmind_geoip2_geoip2_2_12_0.xml, .idea/libraries/Maven__com_maxmind_geoip_geoip_api_1_3_1.xml, .idea/libraries/Maven__com_squareup_okhttp_okhttp_2_4_0.xml, .idea/libraries/Maven__com_squareup_okio_okio_1_4_0.xml, .idea/libraries/Maven__com_squareup_retrofit_retrofit_1_9_0.xml, .idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml, .idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml, .idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml, .idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml, .idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml, .idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml, .idea/libraries/Maven__com_zdjizhi_galaxy_1_0_1.xml, .idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml, .idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml, .idea/libraries/Maven__commons_cli_commons_cli_1_2.xml, .idea/libraries/Maven__commons_codec_commons_codec_1_9.xml, .idea/libraries/Maven__commons_collections_commons_collections_3_2_2.xml, .idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml, .idea/libraries/Maven__commons_daemon_commons_daemon_1_0_13.xml, .idea/libraries/Maven__commons_digester_commons_digester_1_8.xml, .idea/libraries/Maven__commons_el_commons_el_1_0.xml, .idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml, .idea/libraries/Maven__commons_io_commons_io_2_5.xml, .idea/libraries/Maven__commons_lang_commons_lang_2_5.xml, .idea/libraries/Maven__commons_logging_commons_logging_1_2.xml, .idea/libraries/Maven__commons_net_commons_net_3_1.xml, .idea/libraries/Maven__io_dropwizard_metrics_metrics_core_3_1_2.xml, .idea/libraries/Maven__io_netty_netty_3_10_5_Final.xml, .idea/libraries/Maven__io_netty_netty_all_4_1_8_Final.xml, .idea/libraries/Maven__javax_activation_activation_1_1.xml, .idea/libraries/Maven__javax_servlet_jsp_jsp_api_2_1.xml, .idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml, .idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml, .idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml, .idea/libraries/Maven__jline_jline_0_9_94.xml, .idea/libraries/Maven__joda_time_joda_time_2_10.xml, .idea/libraries/Maven__junit_junit_4_12.xml, .idea/libraries/Maven__log4j_log4j_1_2_14.xml, .idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_9_0.xml, .idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml, .idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml, .idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml, .idea/libraries/Maven__org_apache_commons_commons_lang3_3_4.xml, .idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml, .idea/libraries/Maven__org_apache_commons_commons_math_2_2.xml, .idea/libraries/Maven__org_apache_curator_curator_client_2_7_1.xml, .idea/libraries/Maven__org_apache_curator_curator_framework_2_10_0.xml, .idea/libraries/Maven__org_apache_curator_curator_recipes_2_7_1.xml, .idea/libraries/Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml, .idea/libraries/Maven__org_apache_directory_api_api_util_1_0_0_M20.xml, .idea/libraries/Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml, .idea/libraries/Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_7_1.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_7_4.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_client_2_7_1.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_7_1.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_7_1.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_7_1.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_7_1.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_7_4.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_7_1.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_7_1.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_7_1.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_7_1.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_7_4.xml, .idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_7_1.xml, .idea/libraries/Maven__org_apache_hbase_hbase_annotations_1_4_9.xml, .idea/libraries/Maven__org_apache_hbase_hbase_client_1_4_9.xml, .idea/libraries/Maven__org_apache_hbase_hbase_common_1_4_9.xml, .idea/libraries/Maven__org_apache_hbase_hbase_common_tests_1_4_9.xml, .idea/libraries/Maven__org_apache_hbase_hbase_hadoop2_compat_1_4_9.xml, .idea/libraries/Maven__org_apache_hbase_hbase_hadoop_compat_1_4_9.xml, .idea/libraries/Maven__org_apache_hbase_hbase_metrics_1_4_9.xml, .idea/libraries/Maven__org_apache_hbase_hbase_metrics_api_1_4_9.xml, .idea/libraries/Maven__org_apache_hbase_hbase_prefix_tree_1_4_9.xml, .idea/libraries/Maven__org_apache_hbase_hbase_procedure_1_4_9.xml, .idea/libraries/Maven__org_apache_hbase_hbase_protocol_1_4_9.xml, .idea/libraries/Maven__org_apache_hbase_hbase_server_1_4_9.xml, .idea/libraries/Maven__org_apache_htrace_htrace_core_3_1_0_incubating.xml, .idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_5_2.xml, .idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_4_4.xml, .idea/libraries/Maven__org_apache_kafka_kafka_2_11_1_0_0.xml, .idea/libraries/Maven__org_apache_kafka_kafka_clients_1_0_0.xml, .idea/libraries/Maven__org_apache_logging_log4j_log4j_api_2_1.xml, .idea/libraries/Maven__org_apache_logging_log4j_log4j_core_2_1.xml, .idea/libraries/Maven__org_apache_logging_log4j_log4j_slf4j_impl_2_1.xml, .idea/libraries/Maven__org_apache_storm_storm_core_1_0_2.xml, .idea/libraries/Maven__org_apache_storm_storm_kafka_1_0_2.xml, .idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_9.xml, .idea/libraries/Maven__org_clojure_clojure_1_7_0.xml, .idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_9_13.xml, .idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_9_13.xml, .idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_13.xml, .idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml, .idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml, .idea/libraries/Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml, .idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml, .idea/libraries/Maven__org_influxdb_influxdb_java_2_1.xml, .idea/libraries/Maven__org_jamon_jamon_runtime_2_4_1.xml, .idea/libraries/Maven__org_jruby_jcodings_jcodings_1_0_8.xml, .idea/libraries/Maven__org_jruby_joni_joni_2_1_2.xml, .idea/libraries/Maven__org_lz4_lz4_java_1_4.xml, .idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml, .idea/libraries/Maven__org_mortbay_jetty_jetty_sslengine_6_1_26.xml, .idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml, .idea/libraries/Maven__org_mortbay_jetty_jsp_2_1_6_1_14.xml, .idea/libraries/Maven__org_mortbay_jetty_jsp_api_2_1_6_1_14.xml, .idea/libraries/Maven__org_mortbay_jetty_servlet_api_2_5_6_1_14.xml, .idea/libraries/Maven__org_objenesis_objenesis_2_1.xml, .idea/libraries/Maven__org_ow2_asm_asm_5_0_3.xml, .idea/libraries/Maven__org_scala_lang_scala_library_2_11_11.xml, .idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml, .idea/libraries/Maven__org_tukaani_xz_1_0.xml, .idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_4.xml, .idea/libraries/Maven__tomcat_jasper_compiler_5_5_23.xml, .idea/libraries/Maven__tomcat_jasper_runtime_5_5_23.xml, .idea/libraries/Maven__xerces_xercesImpl_2_9_1.xml, .idea/libraries/Maven__xml_apis_xml_apis_1_3_04.xml, .idea/libraries/Maven__xmlenc_xmlenc_0_52.xml, .idea/libraries, .idea/compiler.xml, .idea/encodings.xml, .idea/log-stream-completion.iml, .idea/misc.xml, .idea/modules.xml, .idea/uiDesigner.xml, .idea/vcs.xml, .idea files --- .idea/compiler.xml | 16 -- .idea/encodings.xml | 6 - .idea/libraries/Maven__asm_asm_3_1.xml | 13 -- .../Maven__com_101tec_zkclient_0_10.xml | 13 -- .../Maven__com_alibaba_fastjson_1_2_47.xml | 13 -- ...Maven__com_esotericsoftware_kryo_3_0_3.xml | 13 -- ...ven__com_esotericsoftware_minlog_1_3_0.xml | 13 -- ...com_esotericsoftware_reflectasm_1_10_1.xml | 13 -- ...jackson_core_jackson_annotations_2_9_0.xml | 13 -- ...terxml_jackson_core_jackson_core_2_9_1.xml | 13 -- ...ml_jackson_core_jackson_databind_2_9_1.xml | 13 -- ..._findbugs_findbugs_annotations_1_3_9_1.xml | 13 -- ..._com_google_code_findbugs_jsr305_3_0_0.xml | 13 -- ...Maven__com_google_code_gson_gson_2_2_4.xml | 13 -- .../Maven__com_google_guava_guava_18_0.xml | 13 -- ...om_google_protobuf_protobuf_java_2_5_0.xml | 13 -- ...googlecode_json_simple_json_simple_1_1.xml | 13 -- ...m_jamesmurty_utils_java_xmlbuilder_0_4.xml | 13 -- .../Maven__com_jcraft_jsch_0_1_42.xml | 13 -- .../Maven__com_lmax_disruptor_3_3_2.xml | 13 -- ...Maven__com_maxmind_db_maxmind_db_1_2_2.xml | 13 -- ...aven__com_maxmind_geoip2_geoip2_2_12_0.xml | 13 -- ...ven__com_maxmind_geoip_geoip_api_1_3_1.xml | 13 -- ...aven__com_squareup_okhttp_okhttp_2_4_0.xml | 13 -- .../Maven__com_squareup_okio_okio_1_4_0.xml | 13 -- ...__com_squareup_retrofit_retrofit_1_9_0.xml | 13 -- .../Maven__com_sun_jersey_jersey_core_1_9.xml | 13 -- .../Maven__com_sun_jersey_jersey_json_1_9.xml | 13 -- ...aven__com_sun_jersey_jersey_server_1_9.xml | 13 -- ...en__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml | 13 -- ...m_thoughtworks_paranamer_paranamer_2_3.xml | 13 -- ..._com_yammer_metrics_metrics_core_2_2_0.xml | 13 -- .../Maven__com_zdjizhi_galaxy_1_0_1.xml | 13 -- ...mons_beanutils_commons_beanutils_1_7_0.xml | 13 -- ...beanutils_commons_beanutils_core_1_8_0.xml | 13 -- .../Maven__commons_cli_commons_cli_1_2.xml | 13 -- ...Maven__commons_codec_commons_codec_1_9.xml | 13 -- ..._collections_commons_collections_3_2_2.xml | 13 -- ...onfiguration_commons_configuration_1_6.xml | 13 -- ...__commons_daemon_commons_daemon_1_0_13.xml | 13 -- ..._commons_digester_commons_digester_1_8.xml | 13 -- .../Maven__commons_el_commons_el_1_0.xml | 13 -- ...mons_httpclient_commons_httpclient_3_1.xml | 13 -- .../Maven__commons_io_commons_io_2_5.xml | 13 -- .../Maven__commons_lang_commons_lang_2_5.xml | 13 -- ...n__commons_logging_commons_logging_1_2.xml | 13 -- .../Maven__commons_net_commons_net_3_1.xml | 13 -- ..._dropwizard_metrics_metrics_core_3_1_2.xml | 13 -- .../Maven__io_netty_netty_3_10_5_Final.xml | 13 -- .../Maven__io_netty_netty_all_4_1_8_Final.xml | 13 -- ...Maven__javax_activation_activation_1_1.xml | 13 -- .../Maven__javax_servlet_jsp_jsp_api_2_1.xml | 13 -- .../Maven__javax_servlet_servlet_api_2_5.xml | 13 -- .../Maven__javax_xml_bind_jaxb_api_2_2_2.xml | 13 -- ...Maven__javax_xml_stream_stax_api_1_0_2.xml | 13 -- .idea/libraries/Maven__jline_jline_0_9_94.xml | 13 -- .../Maven__joda_time_joda_time_2_10.xml | 13 -- .idea/libraries/Maven__junit_junit_4_12.xml | 13 -- .idea/libraries/Maven__log4j_log4j_1_2_14.xml | 13 -- ...aven__net_java_dev_jets3t_jets3t_0_9_0.xml | 13 -- ...__net_sf_jopt_simple_jopt_simple_5_0_4.xml | 13 -- .../Maven__org_apache_avro_avro_1_7_4.xml | 13 -- ..._apache_commons_commons_compress_1_4_1.xml | 13 -- ...__org_apache_commons_commons_lang3_3_4.xml | 13 -- ...org_apache_commons_commons_math3_3_1_1.xml | 13 -- ...n__org_apache_commons_commons_math_2_2.xml | 13 -- ...rg_apache_curator_curator_client_2_7_1.xml | 13 -- ...pache_curator_curator_framework_2_10_0.xml | 13 -- ...g_apache_curator_curator_recipes_2_7_1.xml | 13 -- ...e_directory_api_api_asn1_api_1_0_0_M20.xml | 13 -- ...pache_directory_api_api_util_1_0_0_M20.xml | 13 -- ...rectory_server_apacheds_i18n_2_0_0_M15.xml | 13 -- ...rver_apacheds_kerberos_codec_2_0_0_M15.xml | 13 -- ...apache_hadoop_hadoop_annotations_2_7_1.xml | 13 -- ...n__org_apache_hadoop_hadoop_auth_2_7_4.xml | 13 -- ..._org_apache_hadoop_hadoop_client_2_7_1.xml | 13 -- ..._org_apache_hadoop_hadoop_common_2_7_1.xml | 13 -- ...n__org_apache_hadoop_hadoop_hdfs_2_7_1.xml | 13 -- ...doop_hadoop_mapreduce_client_app_2_7_1.xml | 13 -- ...p_hadoop_mapreduce_client_common_2_7_1.xml | 13 -- ...oop_hadoop_mapreduce_client_core_2_7_4.xml | 13 -- ...adoop_mapreduce_client_jobclient_2_7_1.xml | 13 -- ..._hadoop_mapreduce_client_shuffle_2_7_1.xml | 13 -- ...rg_apache_hadoop_hadoop_yarn_api_2_7_1.xml | 13 -- ...apache_hadoop_hadoop_yarn_client_2_7_1.xml | 13 -- ...apache_hadoop_hadoop_yarn_common_2_7_4.xml | 13 -- ...hadoop_hadoop_yarn_server_common_2_7_1.xml | 13 -- ...g_apache_hbase_hbase_annotations_1_4_9.xml | 13 -- ...n__org_apache_hbase_hbase_client_1_4_9.xml | 13 -- ...n__org_apache_hbase_hbase_common_1_4_9.xml | 13 -- ..._apache_hbase_hbase_common_tests_1_4_9.xml | 13 -- ...pache_hbase_hbase_hadoop2_compat_1_4_9.xml | 13 -- ...apache_hbase_hbase_hadoop_compat_1_4_9.xml | 13 -- ...__org_apache_hbase_hbase_metrics_1_4_9.xml | 13 -- ...g_apache_hbase_hbase_metrics_api_1_4_9.xml | 13 -- ...g_apache_hbase_hbase_prefix_tree_1_4_9.xml | 13 -- ...org_apache_hbase_hbase_procedure_1_4_9.xml | 13 -- ..._org_apache_hbase_hbase_protocol_1_4_9.xml | 13 -- ...n__org_apache_hbase_hbase_server_1_4_9.xml | 13 -- ...he_htrace_htrace_core_3_1_0_incubating.xml | 13 -- ...apache_httpcomponents_httpclient_4_5_2.xml | 13 -- ...g_apache_httpcomponents_httpcore_4_4_4.xml | 13 -- ...ven__org_apache_kafka_kafka_2_11_1_0_0.xml | 13 -- ...__org_apache_kafka_kafka_clients_1_0_0.xml | 13 -- ...org_apache_logging_log4j_log4j_api_2_1.xml | 13 -- ...rg_apache_logging_log4j_log4j_core_2_1.xml | 13 -- ...che_logging_log4j_log4j_slf4j_impl_2_1.xml | 13 -- ...ven__org_apache_storm_storm_core_1_0_2.xml | 13 -- ...en__org_apache_storm_storm_kafka_1_0_2.xml | 13 -- ...__org_apache_zookeeper_zookeeper_3_4_9.xml | 13 -- .../Maven__org_clojure_clojure_1_7_0.xml | 13 -- ...dehaus_jackson_jackson_core_asl_1_9_13.xml | 13 -- ..._codehaus_jackson_jackson_jaxrs_1_9_13.xml | 13 -- ...haus_jackson_jackson_mapper_asl_1_9_13.xml | 13 -- ..._org_codehaus_jackson_jackson_xc_1_8_3.xml | 13 -- ...en__org_codehaus_jettison_jettison_1_1.xml | 13 -- ...sesource_leveldbjni_leveldbjni_all_1_8.xml | 13 -- .../Maven__org_hamcrest_hamcrest_core_1_3.xml | 13 -- .../Maven__org_influxdb_influxdb_java_2_1.xml | 13 -- .../Maven__org_jamon_jamon_runtime_2_4_1.xml | 13 -- ...ven__org_jruby_jcodings_jcodings_1_0_8.xml | 13 -- .../Maven__org_jruby_joni_joni_2_1_2.xml | 13 -- .../libraries/Maven__org_lz4_lz4_java_1_4.xml | 13 -- .../Maven__org_mortbay_jetty_jetty_6_1_26.xml | 13 -- ...g_mortbay_jetty_jetty_sslengine_6_1_26.xml | 13 -- ...n__org_mortbay_jetty_jetty_util_6_1_26.xml | 13 -- ...aven__org_mortbay_jetty_jsp_2_1_6_1_14.xml | 13 -- ...__org_mortbay_jetty_jsp_api_2_1_6_1_14.xml | 13 -- ...g_mortbay_jetty_servlet_api_2_5_6_1_14.xml | 13 -- .../Maven__org_objenesis_objenesis_2_1.xml | 13 -- .../Maven__org_ow2_asm_asm_5_0_3.xml | 13 -- ...__org_scala_lang_scala_library_2_11_11.xml | 13 -- .../Maven__org_slf4j_slf4j_api_1_7_7.xml | 13 -- .idea/libraries/Maven__org_tukaani_xz_1_0.xml | 13 -- ...n__org_xerial_snappy_snappy_java_1_1_4.xml | 13 -- .../Maven__tomcat_jasper_compiler_5_5_23.xml | 13 -- .../Maven__tomcat_jasper_runtime_5_5_23.xml | 13 -- .../Maven__xerces_xercesImpl_2_9_1.xml | 13 -- .../Maven__xml_apis_xml_apis_1_3_04.xml | 13 -- .idea/libraries/Maven__xmlenc_xmlenc_0_52.xml | 13 -- .idea/log-stream-completion.iml | 162 ------------------ .idea/misc.xml | 14 -- .idea/modules.xml | 8 - .idea/uiDesigner.xml | 124 -------------- .idea/vcs.xml | 6 - 145 files changed, 2130 deletions(-) delete mode 100644 .idea/compiler.xml delete mode 100644 .idea/encodings.xml delete mode 100644 .idea/libraries/Maven__asm_asm_3_1.xml delete mode 100644 .idea/libraries/Maven__com_101tec_zkclient_0_10.xml delete mode 100644 .idea/libraries/Maven__com_alibaba_fastjson_1_2_47.xml delete mode 100644 .idea/libraries/Maven__com_esotericsoftware_kryo_3_0_3.xml delete mode 100644 .idea/libraries/Maven__com_esotericsoftware_minlog_1_3_0.xml delete mode 100644 .idea/libraries/Maven__com_esotericsoftware_reflectasm_1_10_1.xml delete mode 100644 .idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_0.xml delete mode 100644 .idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_9_1.xml delete mode 100644 .idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_9_1.xml delete mode 100644 .idea/libraries/Maven__com_github_stephenc_findbugs_findbugs_annotations_1_3_9_1.xml delete mode 100644 .idea/libraries/Maven__com_google_code_findbugs_jsr305_3_0_0.xml delete mode 100644 .idea/libraries/Maven__com_google_code_gson_gson_2_2_4.xml delete mode 100644 .idea/libraries/Maven__com_google_guava_guava_18_0.xml delete mode 100644 .idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml delete mode 100644 .idea/libraries/Maven__com_googlecode_json_simple_json_simple_1_1.xml delete mode 100644 .idea/libraries/Maven__com_jamesmurty_utils_java_xmlbuilder_0_4.xml delete mode 100644 .idea/libraries/Maven__com_jcraft_jsch_0_1_42.xml delete mode 100644 .idea/libraries/Maven__com_lmax_disruptor_3_3_2.xml delete mode 100644 .idea/libraries/Maven__com_maxmind_db_maxmind_db_1_2_2.xml delete mode 100644 .idea/libraries/Maven__com_maxmind_geoip2_geoip2_2_12_0.xml delete mode 100644 .idea/libraries/Maven__com_maxmind_geoip_geoip_api_1_3_1.xml delete mode 100644 .idea/libraries/Maven__com_squareup_okhttp_okhttp_2_4_0.xml delete mode 100644 .idea/libraries/Maven__com_squareup_okio_okio_1_4_0.xml delete mode 100644 .idea/libraries/Maven__com_squareup_retrofit_retrofit_1_9_0.xml delete mode 100644 .idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml delete mode 100644 .idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml delete mode 100644 .idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml delete mode 100644 .idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml delete mode 100644 .idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml delete mode 100644 .idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml delete mode 100644 .idea/libraries/Maven__com_zdjizhi_galaxy_1_0_1.xml delete mode 100644 .idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml delete mode 100644 .idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml delete mode 100644 .idea/libraries/Maven__commons_cli_commons_cli_1_2.xml delete mode 100644 .idea/libraries/Maven__commons_codec_commons_codec_1_9.xml delete mode 100644 .idea/libraries/Maven__commons_collections_commons_collections_3_2_2.xml delete mode 100644 .idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml delete mode 100644 .idea/libraries/Maven__commons_daemon_commons_daemon_1_0_13.xml delete mode 100644 .idea/libraries/Maven__commons_digester_commons_digester_1_8.xml delete mode 100644 .idea/libraries/Maven__commons_el_commons_el_1_0.xml delete mode 100644 .idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml delete mode 100644 .idea/libraries/Maven__commons_io_commons_io_2_5.xml delete mode 100644 .idea/libraries/Maven__commons_lang_commons_lang_2_5.xml delete mode 100644 .idea/libraries/Maven__commons_logging_commons_logging_1_2.xml delete mode 100644 .idea/libraries/Maven__commons_net_commons_net_3_1.xml delete mode 100644 .idea/libraries/Maven__io_dropwizard_metrics_metrics_core_3_1_2.xml delete mode 100644 .idea/libraries/Maven__io_netty_netty_3_10_5_Final.xml delete mode 100644 .idea/libraries/Maven__io_netty_netty_all_4_1_8_Final.xml delete mode 100644 .idea/libraries/Maven__javax_activation_activation_1_1.xml delete mode 100644 .idea/libraries/Maven__javax_servlet_jsp_jsp_api_2_1.xml delete mode 100644 .idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml delete mode 100644 .idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml delete mode 100644 .idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml delete mode 100644 .idea/libraries/Maven__jline_jline_0_9_94.xml delete mode 100644 .idea/libraries/Maven__joda_time_joda_time_2_10.xml delete mode 100644 .idea/libraries/Maven__junit_junit_4_12.xml delete mode 100644 .idea/libraries/Maven__log4j_log4j_1_2_14.xml delete mode 100644 .idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_9_0.xml delete mode 100644 .idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml delete mode 100644 .idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml delete mode 100644 .idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_commons_commons_lang3_3_4.xml delete mode 100644 .idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_commons_commons_math_2_2.xml delete mode 100644 .idea/libraries/Maven__org_apache_curator_curator_client_2_7_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_curator_curator_framework_2_10_0.xml delete mode 100644 .idea/libraries/Maven__org_apache_curator_curator_recipes_2_7_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml delete mode 100644 .idea/libraries/Maven__org_apache_directory_api_api_util_1_0_0_M20.xml delete mode 100644 .idea/libraries/Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml delete mode 100644 .idea/libraries/Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_7_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_7_4.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_client_2_7_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_7_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_7_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_7_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_7_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_7_4.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_7_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_7_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_7_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_7_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_7_4.xml delete mode 100644 .idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_7_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_annotations_1_4_9.xml delete mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_client_1_4_9.xml delete mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_common_1_4_9.xml delete mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_common_tests_1_4_9.xml delete mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_hadoop2_compat_1_4_9.xml delete mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_hadoop_compat_1_4_9.xml delete mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_metrics_1_4_9.xml delete mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_metrics_api_1_4_9.xml delete mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_prefix_tree_1_4_9.xml delete mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_procedure_1_4_9.xml delete mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_protocol_1_4_9.xml delete mode 100644 .idea/libraries/Maven__org_apache_hbase_hbase_server_1_4_9.xml delete mode 100644 .idea/libraries/Maven__org_apache_htrace_htrace_core_3_1_0_incubating.xml delete mode 100644 .idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_5_2.xml delete mode 100644 .idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_4_4.xml delete mode 100644 .idea/libraries/Maven__org_apache_kafka_kafka_2_11_1_0_0.xml delete mode 100644 .idea/libraries/Maven__org_apache_kafka_kafka_clients_1_0_0.xml delete mode 100644 .idea/libraries/Maven__org_apache_logging_log4j_log4j_api_2_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_logging_log4j_log4j_core_2_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_logging_log4j_log4j_slf4j_impl_2_1.xml delete mode 100644 .idea/libraries/Maven__org_apache_storm_storm_core_1_0_2.xml delete mode 100644 .idea/libraries/Maven__org_apache_storm_storm_kafka_1_0_2.xml delete mode 100644 .idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_9.xml delete mode 100644 .idea/libraries/Maven__org_clojure_clojure_1_7_0.xml delete mode 100644 .idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_9_13.xml delete mode 100644 .idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_9_13.xml delete mode 100644 .idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_13.xml delete mode 100644 .idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml delete mode 100644 .idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml delete mode 100644 .idea/libraries/Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml delete mode 100644 .idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml delete mode 100644 .idea/libraries/Maven__org_influxdb_influxdb_java_2_1.xml delete mode 100644 .idea/libraries/Maven__org_jamon_jamon_runtime_2_4_1.xml delete mode 100644 .idea/libraries/Maven__org_jruby_jcodings_jcodings_1_0_8.xml delete mode 100644 .idea/libraries/Maven__org_jruby_joni_joni_2_1_2.xml delete mode 100644 .idea/libraries/Maven__org_lz4_lz4_java_1_4.xml delete mode 100644 .idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml delete mode 100644 .idea/libraries/Maven__org_mortbay_jetty_jetty_sslengine_6_1_26.xml delete mode 100644 .idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml delete mode 100644 .idea/libraries/Maven__org_mortbay_jetty_jsp_2_1_6_1_14.xml delete mode 100644 .idea/libraries/Maven__org_mortbay_jetty_jsp_api_2_1_6_1_14.xml delete mode 100644 .idea/libraries/Maven__org_mortbay_jetty_servlet_api_2_5_6_1_14.xml delete mode 100644 .idea/libraries/Maven__org_objenesis_objenesis_2_1.xml delete mode 100644 .idea/libraries/Maven__org_ow2_asm_asm_5_0_3.xml delete mode 100644 .idea/libraries/Maven__org_scala_lang_scala_library_2_11_11.xml delete mode 100644 .idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml delete mode 100644 .idea/libraries/Maven__org_tukaani_xz_1_0.xml delete mode 100644 .idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_4.xml delete mode 100644 .idea/libraries/Maven__tomcat_jasper_compiler_5_5_23.xml delete mode 100644 .idea/libraries/Maven__tomcat_jasper_runtime_5_5_23.xml delete mode 100644 .idea/libraries/Maven__xerces_xercesImpl_2_9_1.xml delete mode 100644 .idea/libraries/Maven__xml_apis_xml_apis_1_3_04.xml delete mode 100644 .idea/libraries/Maven__xmlenc_xmlenc_0_52.xml delete mode 100644 .idea/log-stream-completion.iml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/uiDesigner.xml delete mode 100644 .idea/vcs.xml diff --git a/.idea/compiler.xml b/.idea/compiler.xml deleted file mode 100644 index 7603751..0000000 --- a/.idea/compiler.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml deleted file mode 100644 index e79da7e..0000000 --- a/.idea/encodings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__asm_asm_3_1.xml b/.idea/libraries/Maven__asm_asm_3_1.xml deleted file mode 100644 index 3386f10..0000000 --- a/.idea/libraries/Maven__asm_asm_3_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_101tec_zkclient_0_10.xml b/.idea/libraries/Maven__com_101tec_zkclient_0_10.xml deleted file mode 100644 index a2b08a2..0000000 --- a/.idea/libraries/Maven__com_101tec_zkclient_0_10.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_alibaba_fastjson_1_2_47.xml b/.idea/libraries/Maven__com_alibaba_fastjson_1_2_47.xml deleted file mode 100644 index dcd6ee5..0000000 --- a/.idea/libraries/Maven__com_alibaba_fastjson_1_2_47.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_esotericsoftware_kryo_3_0_3.xml b/.idea/libraries/Maven__com_esotericsoftware_kryo_3_0_3.xml deleted file mode 100644 index 17a5c28..0000000 --- a/.idea/libraries/Maven__com_esotericsoftware_kryo_3_0_3.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_esotericsoftware_minlog_1_3_0.xml b/.idea/libraries/Maven__com_esotericsoftware_minlog_1_3_0.xml deleted file mode 100644 index 0f2bb63..0000000 --- a/.idea/libraries/Maven__com_esotericsoftware_minlog_1_3_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_esotericsoftware_reflectasm_1_10_1.xml b/.idea/libraries/Maven__com_esotericsoftware_reflectasm_1_10_1.xml deleted file mode 100644 index 5ea60d9..0000000 --- a/.idea/libraries/Maven__com_esotericsoftware_reflectasm_1_10_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_0.xml b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_0.xml deleted file mode 100644 index 06441f4..0000000 --- a/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_9_1.xml b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_9_1.xml deleted file mode 100644 index 7719a19..0000000 --- a/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_9_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_9_1.xml b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_9_1.xml deleted file mode 100644 index c197771..0000000 --- a/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_9_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_github_stephenc_findbugs_findbugs_annotations_1_3_9_1.xml b/.idea/libraries/Maven__com_github_stephenc_findbugs_findbugs_annotations_1_3_9_1.xml deleted file mode 100644 index d1d9a7c..0000000 --- a/.idea/libraries/Maven__com_github_stephenc_findbugs_findbugs_annotations_1_3_9_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_code_findbugs_jsr305_3_0_0.xml b/.idea/libraries/Maven__com_google_code_findbugs_jsr305_3_0_0.xml deleted file mode 100644 index 6644d12..0000000 --- a/.idea/libraries/Maven__com_google_code_findbugs_jsr305_3_0_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_code_gson_gson_2_2_4.xml b/.idea/libraries/Maven__com_google_code_gson_gson_2_2_4.xml deleted file mode 100644 index 4533c1b..0000000 --- a/.idea/libraries/Maven__com_google_code_gson_gson_2_2_4.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_guava_guava_18_0.xml b/.idea/libraries/Maven__com_google_guava_guava_18_0.xml deleted file mode 100644 index bbd71d7..0000000 --- a/.idea/libraries/Maven__com_google_guava_guava_18_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml b/.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml deleted file mode 100644 index c06d3a1..0000000 --- a/.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_googlecode_json_simple_json_simple_1_1.xml b/.idea/libraries/Maven__com_googlecode_json_simple_json_simple_1_1.xml deleted file mode 100644 index ea70fe1..0000000 --- a/.idea/libraries/Maven__com_googlecode_json_simple_json_simple_1_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_jamesmurty_utils_java_xmlbuilder_0_4.xml b/.idea/libraries/Maven__com_jamesmurty_utils_java_xmlbuilder_0_4.xml deleted file mode 100644 index a2a0c04..0000000 --- a/.idea/libraries/Maven__com_jamesmurty_utils_java_xmlbuilder_0_4.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_jcraft_jsch_0_1_42.xml b/.idea/libraries/Maven__com_jcraft_jsch_0_1_42.xml deleted file mode 100644 index 880be15..0000000 --- a/.idea/libraries/Maven__com_jcraft_jsch_0_1_42.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_lmax_disruptor_3_3_2.xml b/.idea/libraries/Maven__com_lmax_disruptor_3_3_2.xml deleted file mode 100644 index c22270a..0000000 --- a/.idea/libraries/Maven__com_lmax_disruptor_3_3_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_maxmind_db_maxmind_db_1_2_2.xml b/.idea/libraries/Maven__com_maxmind_db_maxmind_db_1_2_2.xml deleted file mode 100644 index 1342872..0000000 --- a/.idea/libraries/Maven__com_maxmind_db_maxmind_db_1_2_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_maxmind_geoip2_geoip2_2_12_0.xml b/.idea/libraries/Maven__com_maxmind_geoip2_geoip2_2_12_0.xml deleted file mode 100644 index 1471229..0000000 --- a/.idea/libraries/Maven__com_maxmind_geoip2_geoip2_2_12_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_maxmind_geoip_geoip_api_1_3_1.xml b/.idea/libraries/Maven__com_maxmind_geoip_geoip_api_1_3_1.xml deleted file mode 100644 index bd438ab..0000000 --- a/.idea/libraries/Maven__com_maxmind_geoip_geoip_api_1_3_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_squareup_okhttp_okhttp_2_4_0.xml b/.idea/libraries/Maven__com_squareup_okhttp_okhttp_2_4_0.xml deleted file mode 100644 index be09ac3..0000000 --- a/.idea/libraries/Maven__com_squareup_okhttp_okhttp_2_4_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_squareup_okio_okio_1_4_0.xml b/.idea/libraries/Maven__com_squareup_okio_okio_1_4_0.xml deleted file mode 100644 index ddacf0f..0000000 --- a/.idea/libraries/Maven__com_squareup_okio_okio_1_4_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_squareup_retrofit_retrofit_1_9_0.xml b/.idea/libraries/Maven__com_squareup_retrofit_retrofit_1_9_0.xml deleted file mode 100644 index 86047ed..0000000 --- a/.idea/libraries/Maven__com_squareup_retrofit_retrofit_1_9_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml b/.idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml deleted file mode 100644 index abd3e5d..0000000 --- a/.idea/libraries/Maven__com_sun_jersey_jersey_core_1_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml b/.idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml deleted file mode 100644 index fff6aea..0000000 --- a/.idea/libraries/Maven__com_sun_jersey_jersey_json_1_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml b/.idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml deleted file mode 100644 index 717467f..0000000 --- a/.idea/libraries/Maven__com_sun_jersey_jersey_server_1_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml b/.idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml deleted file mode 100644 index 1165a3b..0000000 --- a/.idea/libraries/Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml b/.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml deleted file mode 100644 index 3807eb2..0000000 --- a/.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml b/.idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml deleted file mode 100644 index fc159c5..0000000 --- a/.idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__com_zdjizhi_galaxy_1_0_1.xml b/.idea/libraries/Maven__com_zdjizhi_galaxy_1_0_1.xml deleted file mode 100644 index 5ec91ab..0000000 --- a/.idea/libraries/Maven__com_zdjizhi_galaxy_1_0_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml b/.idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml deleted file mode 100644 index 1cb64ed..0000000 --- a/.idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml b/.idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml deleted file mode 100644 index 3fda2c0..0000000 --- a/.idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml b/.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml deleted file mode 100644 index cec2493..0000000 --- a/.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_codec_commons_codec_1_9.xml b/.idea/libraries/Maven__commons_codec_commons_codec_1_9.xml deleted file mode 100644 index a66d039..0000000 --- a/.idea/libraries/Maven__commons_codec_commons_codec_1_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_collections_commons_collections_3_2_2.xml b/.idea/libraries/Maven__commons_collections_commons_collections_3_2_2.xml deleted file mode 100644 index 13afda2..0000000 --- a/.idea/libraries/Maven__commons_collections_commons_collections_3_2_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml b/.idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml deleted file mode 100644 index ae96781..0000000 --- a/.idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_daemon_commons_daemon_1_0_13.xml b/.idea/libraries/Maven__commons_daemon_commons_daemon_1_0_13.xml deleted file mode 100644 index d44fc67..0000000 --- a/.idea/libraries/Maven__commons_daemon_commons_daemon_1_0_13.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_digester_commons_digester_1_8.xml b/.idea/libraries/Maven__commons_digester_commons_digester_1_8.xml deleted file mode 100644 index 49f1def..0000000 --- a/.idea/libraries/Maven__commons_digester_commons_digester_1_8.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_el_commons_el_1_0.xml b/.idea/libraries/Maven__commons_el_commons_el_1_0.xml deleted file mode 100644 index 598d1b0..0000000 --- a/.idea/libraries/Maven__commons_el_commons_el_1_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml b/.idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml deleted file mode 100644 index 66e6537..0000000 --- a/.idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_io_commons_io_2_5.xml b/.idea/libraries/Maven__commons_io_commons_io_2_5.xml deleted file mode 100644 index 67c2ad2..0000000 --- a/.idea/libraries/Maven__commons_io_commons_io_2_5.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_lang_commons_lang_2_5.xml b/.idea/libraries/Maven__commons_lang_commons_lang_2_5.xml deleted file mode 100644 index 055afe5..0000000 --- a/.idea/libraries/Maven__commons_lang_commons_lang_2_5.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_logging_commons_logging_1_2.xml b/.idea/libraries/Maven__commons_logging_commons_logging_1_2.xml deleted file mode 100644 index eab40b3..0000000 --- a/.idea/libraries/Maven__commons_logging_commons_logging_1_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_net_commons_net_3_1.xml b/.idea/libraries/Maven__commons_net_commons_net_3_1.xml deleted file mode 100644 index dc40661..0000000 --- a/.idea/libraries/Maven__commons_net_commons_net_3_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__io_dropwizard_metrics_metrics_core_3_1_2.xml b/.idea/libraries/Maven__io_dropwizard_metrics_metrics_core_3_1_2.xml deleted file mode 100644 index e25f69c..0000000 --- a/.idea/libraries/Maven__io_dropwizard_metrics_metrics_core_3_1_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__io_netty_netty_3_10_5_Final.xml b/.idea/libraries/Maven__io_netty_netty_3_10_5_Final.xml deleted file mode 100644 index d1cfe80..0000000 --- a/.idea/libraries/Maven__io_netty_netty_3_10_5_Final.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__io_netty_netty_all_4_1_8_Final.xml b/.idea/libraries/Maven__io_netty_netty_all_4_1_8_Final.xml deleted file mode 100644 index 7f19a44..0000000 --- a/.idea/libraries/Maven__io_netty_netty_all_4_1_8_Final.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_activation_activation_1_1.xml b/.idea/libraries/Maven__javax_activation_activation_1_1.xml deleted file mode 100644 index 180d587..0000000 --- a/.idea/libraries/Maven__javax_activation_activation_1_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_servlet_jsp_jsp_api_2_1.xml b/.idea/libraries/Maven__javax_servlet_jsp_jsp_api_2_1.xml deleted file mode 100644 index 4b5f67f..0000000 --- a/.idea/libraries/Maven__javax_servlet_jsp_jsp_api_2_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml b/.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml deleted file mode 100644 index 679e09a..0000000 --- a/.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml b/.idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml deleted file mode 100644 index a3054d2..0000000 --- a/.idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml b/.idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml deleted file mode 100644 index 2a4dd7a..0000000 --- a/.idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__jline_jline_0_9_94.xml b/.idea/libraries/Maven__jline_jline_0_9_94.xml deleted file mode 100644 index 903c339..0000000 --- a/.idea/libraries/Maven__jline_jline_0_9_94.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__joda_time_joda_time_2_10.xml b/.idea/libraries/Maven__joda_time_joda_time_2_10.xml deleted file mode 100644 index 9274557..0000000 --- a/.idea/libraries/Maven__joda_time_joda_time_2_10.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__junit_junit_4_12.xml b/.idea/libraries/Maven__junit_junit_4_12.xml deleted file mode 100644 index d411041..0000000 --- a/.idea/libraries/Maven__junit_junit_4_12.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__log4j_log4j_1_2_14.xml b/.idea/libraries/Maven__log4j_log4j_1_2_14.xml deleted file mode 100644 index 2825a67..0000000 --- a/.idea/libraries/Maven__log4j_log4j_1_2_14.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_9_0.xml b/.idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_9_0.xml deleted file mode 100644 index f4ff312..0000000 --- a/.idea/libraries/Maven__net_java_dev_jets3t_jets3t_0_9_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml b/.idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml deleted file mode 100644 index 1ef5173..0000000 --- a/.idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml b/.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml deleted file mode 100644 index cc03056..0000000 --- a/.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml b/.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml deleted file mode 100644 index 190209e..0000000 --- a/.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_4.xml b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_4.xml deleted file mode 100644 index 78cfcd3..0000000 --- a/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_4.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml b/.idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml deleted file mode 100644 index 5627f0c..0000000 --- a/.idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_math_2_2.xml b/.idea/libraries/Maven__org_apache_commons_commons_math_2_2.xml deleted file mode 100644 index 8f3e315..0000000 --- a/.idea/libraries/Maven__org_apache_commons_commons_math_2_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_curator_curator_client_2_7_1.xml b/.idea/libraries/Maven__org_apache_curator_curator_client_2_7_1.xml deleted file mode 100644 index 18c48a6..0000000 --- a/.idea/libraries/Maven__org_apache_curator_curator_client_2_7_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_curator_curator_framework_2_10_0.xml b/.idea/libraries/Maven__org_apache_curator_curator_framework_2_10_0.xml deleted file mode 100644 index e7280b8..0000000 --- a/.idea/libraries/Maven__org_apache_curator_curator_framework_2_10_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_curator_curator_recipes_2_7_1.xml b/.idea/libraries/Maven__org_apache_curator_curator_recipes_2_7_1.xml deleted file mode 100644 index 3f3e79a..0000000 --- a/.idea/libraries/Maven__org_apache_curator_curator_recipes_2_7_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml b/.idea/libraries/Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml deleted file mode 100644 index bd5ac37..0000000 --- a/.idea/libraries/Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_directory_api_api_util_1_0_0_M20.xml b/.idea/libraries/Maven__org_apache_directory_api_api_util_1_0_0_M20.xml deleted file mode 100644 index b3d86af..0000000 --- a/.idea/libraries/Maven__org_apache_directory_api_api_util_1_0_0_M20.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml b/.idea/libraries/Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml deleted file mode 100644 index 7a73f45..0000000 --- a/.idea/libraries/Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml b/.idea/libraries/Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml deleted file mode 100644 index 5378216..0000000 --- a/.idea/libraries/Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_7_1.xml deleted file mode 100644 index 106c941..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_7_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_7_4.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_7_4.xml deleted file mode 100644 index a424dd6..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_7_4.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_client_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_client_2_7_1.xml deleted file mode 100644 index 780c61b..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_client_2_7_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_7_1.xml deleted file mode 100644 index 0d6c002..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_7_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_7_1.xml deleted file mode 100644 index 80663c5..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_hdfs_2_7_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_7_1.xml deleted file mode 100644 index 17526fe..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_7_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_7_1.xml deleted file mode 100644 index d5185d1..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_7_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_7_4.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_7_4.xml deleted file mode 100644 index 81d159d..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_7_4.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_7_1.xml deleted file mode 100644 index e57a978..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_7_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_7_1.xml deleted file mode 100644 index 8a29199..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_7_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_7_1.xml deleted file mode 100644 index 9ac03d9..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_7_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_7_1.xml deleted file mode 100644 index 0f9e524..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_client_2_7_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_7_4.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_7_4.xml deleted file mode 100644 index 6d4f20c..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_7_4.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_7_1.xml b/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_7_1.xml deleted file mode 100644 index 07bbc3c..0000000 --- a/.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_server_common_2_7_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_annotations_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_annotations_1_4_9.xml deleted file mode 100644 index 1ddeff6..0000000 --- a/.idea/libraries/Maven__org_apache_hbase_hbase_annotations_1_4_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_client_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_client_1_4_9.xml deleted file mode 100644 index 6e08e98..0000000 --- a/.idea/libraries/Maven__org_apache_hbase_hbase_client_1_4_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_common_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_common_1_4_9.xml deleted file mode 100644 index 8fa4d1a..0000000 --- a/.idea/libraries/Maven__org_apache_hbase_hbase_common_1_4_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_common_tests_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_common_tests_1_4_9.xml deleted file mode 100644 index d390357..0000000 --- a/.idea/libraries/Maven__org_apache_hbase_hbase_common_tests_1_4_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_hadoop2_compat_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_hadoop2_compat_1_4_9.xml deleted file mode 100644 index 6555370..0000000 --- a/.idea/libraries/Maven__org_apache_hbase_hbase_hadoop2_compat_1_4_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_hadoop_compat_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_hadoop_compat_1_4_9.xml deleted file mode 100644 index bc8c736..0000000 --- a/.idea/libraries/Maven__org_apache_hbase_hbase_hadoop_compat_1_4_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_metrics_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_metrics_1_4_9.xml deleted file mode 100644 index c53e03c..0000000 --- a/.idea/libraries/Maven__org_apache_hbase_hbase_metrics_1_4_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_metrics_api_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_metrics_api_1_4_9.xml deleted file mode 100644 index c1578ad..0000000 --- a/.idea/libraries/Maven__org_apache_hbase_hbase_metrics_api_1_4_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_prefix_tree_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_prefix_tree_1_4_9.xml deleted file mode 100644 index 0c2803e..0000000 --- a/.idea/libraries/Maven__org_apache_hbase_hbase_prefix_tree_1_4_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_procedure_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_procedure_1_4_9.xml deleted file mode 100644 index 56da90c..0000000 --- a/.idea/libraries/Maven__org_apache_hbase_hbase_procedure_1_4_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_protocol_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_protocol_1_4_9.xml deleted file mode 100644 index e53de29..0000000 --- a/.idea/libraries/Maven__org_apache_hbase_hbase_protocol_1_4_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_hbase_hbase_server_1_4_9.xml b/.idea/libraries/Maven__org_apache_hbase_hbase_server_1_4_9.xml deleted file mode 100644 index 53856e3..0000000 --- a/.idea/libraries/Maven__org_apache_hbase_hbase_server_1_4_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_htrace_htrace_core_3_1_0_incubating.xml b/.idea/libraries/Maven__org_apache_htrace_htrace_core_3_1_0_incubating.xml deleted file mode 100644 index 08aeeed..0000000 --- a/.idea/libraries/Maven__org_apache_htrace_htrace_core_3_1_0_incubating.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_5_2.xml b/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_5_2.xml deleted file mode 100644 index fdb7ead..0000000 --- a/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_5_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_4_4.xml b/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_4_4.xml deleted file mode 100644 index 3a5aa19..0000000 --- a/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_4_4.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_kafka_kafka_2_11_1_0_0.xml b/.idea/libraries/Maven__org_apache_kafka_kafka_2_11_1_0_0.xml deleted file mode 100644 index cad83b3..0000000 --- a/.idea/libraries/Maven__org_apache_kafka_kafka_2_11_1_0_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_kafka_kafka_clients_1_0_0.xml b/.idea/libraries/Maven__org_apache_kafka_kafka_clients_1_0_0.xml deleted file mode 100644 index 25e3efc..0000000 --- a/.idea/libraries/Maven__org_apache_kafka_kafka_clients_1_0_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_logging_log4j_log4j_api_2_1.xml b/.idea/libraries/Maven__org_apache_logging_log4j_log4j_api_2_1.xml deleted file mode 100644 index e0f672a..0000000 --- a/.idea/libraries/Maven__org_apache_logging_log4j_log4j_api_2_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_logging_log4j_log4j_core_2_1.xml b/.idea/libraries/Maven__org_apache_logging_log4j_log4j_core_2_1.xml deleted file mode 100644 index 28ef981..0000000 --- a/.idea/libraries/Maven__org_apache_logging_log4j_log4j_core_2_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_logging_log4j_log4j_slf4j_impl_2_1.xml b/.idea/libraries/Maven__org_apache_logging_log4j_log4j_slf4j_impl_2_1.xml deleted file mode 100644 index f29d4cf..0000000 --- a/.idea/libraries/Maven__org_apache_logging_log4j_log4j_slf4j_impl_2_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_storm_storm_core_1_0_2.xml b/.idea/libraries/Maven__org_apache_storm_storm_core_1_0_2.xml deleted file mode 100644 index ac5ed57..0000000 --- a/.idea/libraries/Maven__org_apache_storm_storm_core_1_0_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_storm_storm_kafka_1_0_2.xml b/.idea/libraries/Maven__org_apache_storm_storm_kafka_1_0_2.xml deleted file mode 100644 index 9617e32..0000000 --- a/.idea/libraries/Maven__org_apache_storm_storm_kafka_1_0_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_9.xml b/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_9.xml deleted file mode 100644 index 24de43f..0000000 --- a/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_9.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_clojure_clojure_1_7_0.xml b/.idea/libraries/Maven__org_clojure_clojure_1_7_0.xml deleted file mode 100644 index 3e944f3..0000000 --- a/.idea/libraries/Maven__org_clojure_clojure_1_7_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_9_13.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_9_13.xml deleted file mode 100644 index 98eb549..0000000 --- a/.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_9_13.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_9_13.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_9_13.xml deleted file mode 100644 index 43fc6ed..0000000 --- a/.idea/libraries/Maven__org_codehaus_jackson_jackson_jaxrs_1_9_13.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_13.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_13.xml deleted file mode 100644 index 77f3bad..0000000 --- a/.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_13.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml deleted file mode 100644 index 9fc3894..0000000 --- a/.idea/libraries/Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml b/.idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml deleted file mode 100644 index d62802e..0000000 --- a/.idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml b/.idea/libraries/Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml deleted file mode 100644 index 4a829d2..0000000 --- a/.idea/libraries/Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml b/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml deleted file mode 100644 index f58bbc1..0000000 --- a/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_influxdb_influxdb_java_2_1.xml b/.idea/libraries/Maven__org_influxdb_influxdb_java_2_1.xml deleted file mode 100644 index a6c5b54..0000000 --- a/.idea/libraries/Maven__org_influxdb_influxdb_java_2_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_jamon_jamon_runtime_2_4_1.xml b/.idea/libraries/Maven__org_jamon_jamon_runtime_2_4_1.xml deleted file mode 100644 index 8858b4f..0000000 --- a/.idea/libraries/Maven__org_jamon_jamon_runtime_2_4_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_jruby_jcodings_jcodings_1_0_8.xml b/.idea/libraries/Maven__org_jruby_jcodings_jcodings_1_0_8.xml deleted file mode 100644 index 7e032d7..0000000 --- a/.idea/libraries/Maven__org_jruby_jcodings_jcodings_1_0_8.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_jruby_joni_joni_2_1_2.xml b/.idea/libraries/Maven__org_jruby_joni_joni_2_1_2.xml deleted file mode 100644 index a557ab1..0000000 --- a/.idea/libraries/Maven__org_jruby_joni_joni_2_1_2.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_lz4_lz4_java_1_4.xml b/.idea/libraries/Maven__org_lz4_lz4_java_1_4.xml deleted file mode 100644 index 8cf9aff..0000000 --- a/.idea/libraries/Maven__org_lz4_lz4_java_1_4.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml b/.idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml deleted file mode 100644 index 0f5d0b8..0000000 --- a/.idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jetty_sslengine_6_1_26.xml b/.idea/libraries/Maven__org_mortbay_jetty_jetty_sslengine_6_1_26.xml deleted file mode 100644 index 4720600..0000000 --- a/.idea/libraries/Maven__org_mortbay_jetty_jetty_sslengine_6_1_26.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml b/.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml deleted file mode 100644 index f6ec2b9..0000000 --- a/.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jsp_2_1_6_1_14.xml b/.idea/libraries/Maven__org_mortbay_jetty_jsp_2_1_6_1_14.xml deleted file mode 100644 index cc33d9f..0000000 --- a/.idea/libraries/Maven__org_mortbay_jetty_jsp_2_1_6_1_14.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jsp_api_2_1_6_1_14.xml b/.idea/libraries/Maven__org_mortbay_jetty_jsp_api_2_1_6_1_14.xml deleted file mode 100644 index 590bc87..0000000 --- a/.idea/libraries/Maven__org_mortbay_jetty_jsp_api_2_1_6_1_14.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_servlet_api_2_5_6_1_14.xml b/.idea/libraries/Maven__org_mortbay_jetty_servlet_api_2_5_6_1_14.xml deleted file mode 100644 index 5536a9c..0000000 --- a/.idea/libraries/Maven__org_mortbay_jetty_servlet_api_2_5_6_1_14.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_objenesis_objenesis_2_1.xml b/.idea/libraries/Maven__org_objenesis_objenesis_2_1.xml deleted file mode 100644 index 7ab319b..0000000 --- a/.idea/libraries/Maven__org_objenesis_objenesis_2_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_ow2_asm_asm_5_0_3.xml b/.idea/libraries/Maven__org_ow2_asm_asm_5_0_3.xml deleted file mode 100644 index da69f6c..0000000 --- a/.idea/libraries/Maven__org_ow2_asm_asm_5_0_3.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_11.xml b/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_11.xml deleted file mode 100644 index 1553078..0000000 --- a/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_11.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml deleted file mode 100644 index 1e67260..0000000 --- a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_tukaani_xz_1_0.xml b/.idea/libraries/Maven__org_tukaani_xz_1_0.xml deleted file mode 100644 index 8b8042f..0000000 --- a/.idea/libraries/Maven__org_tukaani_xz_1_0.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_4.xml b/.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_4.xml deleted file mode 100644 index 33526ab..0000000 --- a/.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_4.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__tomcat_jasper_compiler_5_5_23.xml b/.idea/libraries/Maven__tomcat_jasper_compiler_5_5_23.xml deleted file mode 100644 index 4610fa2..0000000 --- a/.idea/libraries/Maven__tomcat_jasper_compiler_5_5_23.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__tomcat_jasper_runtime_5_5_23.xml b/.idea/libraries/Maven__tomcat_jasper_runtime_5_5_23.xml deleted file mode 100644 index e80db89..0000000 --- a/.idea/libraries/Maven__tomcat_jasper_runtime_5_5_23.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__xerces_xercesImpl_2_9_1.xml b/.idea/libraries/Maven__xerces_xercesImpl_2_9_1.xml deleted file mode 100644 index 19d39db..0000000 --- a/.idea/libraries/Maven__xerces_xercesImpl_2_9_1.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__xml_apis_xml_apis_1_3_04.xml b/.idea/libraries/Maven__xml_apis_xml_apis_1_3_04.xml deleted file mode 100644 index 891caa2..0000000 --- a/.idea/libraries/Maven__xml_apis_xml_apis_1_3_04.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/Maven__xmlenc_xmlenc_0_52.xml b/.idea/libraries/Maven__xmlenc_xmlenc_0_52.xml deleted file mode 100644 index 59d176d..0000000 --- a/.idea/libraries/Maven__xmlenc_xmlenc_0_52.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/log-stream-completion.iml b/.idea/log-stream-completion.iml deleted file mode 100644 index 52db4e7..0000000 --- a/.idea/log-stream-completion.iml +++ /dev/null @@ -1,162 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index fa89799..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 9d425bb..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml deleted file mode 100644 index e96534f..0000000 --- a/.idea/uiDesigner.xml +++ /dev/null @@ -1,124 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 97148a3ad033bb20acf0960bf279ec19945b2a3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Wed, 13 Nov 2019 16:51:50 +0800 Subject: [PATCH 08/27] =?UTF-8?q?=E6=97=A5=E5=BF=97=E8=A1=A5=E5=85=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/cn/ac/iie/bean/CollectProtocolRecordLog.java | 2 +- src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java | 2 +- .../java/cn/ac/iie/topology/LogFlowWriteTopology.java | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java b/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java index 43b7d00..a4d82e8 100644 --- a/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java +++ b/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java @@ -1,7 +1,7 @@ package cn.ac.iie.bean; /** - * 除radius之外 + * 除radius之外，通联日志类 * * @author qidaijie */ diff --git a/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java b/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java index 998182f..799a8bd 100644 --- a/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java +++ b/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java @@ -1,7 +1,7 @@ package cn.ac.iie.bean; /** - * 策略 + * 安全策略 * * @author qidaijie */ diff --git a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java index 9f3a54d..c5c5090 100644 --- a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java +++ b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java @@ -64,22 +64,22 @@ public class LogFlowWriteTopology { builder.setSpout("LogFlowWriteSpout", new CustomizedKafkaSpout(), FlowWriteConfig.SPOUT_PARALLELISM); switch (FlowWriteConfig.KAFKA_TOPIC) { - case "PROXY-POLICY-LOG": + case "PROXY-EVENT-LOG": builder.setBolt("ProxyCompletionBolt", new ProxyCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ProxyCompletionBolt"); break; - case "COLLECT-RADIUS-RECORD-LOG": + case "RADIUS-RECORD-LOG": builder.setBolt("RadiusCompletionBolt", new RadiusCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt"); break; - case "COLLECT-PROTOCOL-RECORD-LOG": + case "CONNECTION-RECORD-LOG": builder.setBolt("CollectCompletedBolt", new CollectCompletedBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("CollectCompletedBolt"); break; - case "SECURITY-POLICY-LOG": + case "SECURITY-EVENT-LOG": builder.setBolt("SecurityCompletionBolt", new SecurityCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("SecurityCompletionBolt"); break; From 8f59751da9d37b6e2ee929d34199abd3e44bc37e Mon Sep 17 00:00:00 2001 From: qidaijie Date: Fri, 15 Nov 2019 09:49:51 +0800 Subject: [PATCH 09/27] =?UTF-8?q?=E6=9B=B4=E6=96=B0hbase=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- properties/core-site.xml | 71 ++ properties/hbase-site.xml | 34 +- properties/hdfs-site.xml | 24 +- properties/service_flow_config.properties | 20 +- .../bean/connection/ConnectionRecordLog.java | 684 ++++++++++++++++++ .../iie/bean/proxy/ProxySessionRecordLog.java | 203 ++++++ .../bean/radius/RadiusSessionRecordLog.java | 89 +++ .../iie/bean/security/SecurityPolicyLog.java | 684 ++++++++++++++++++ .../bolt/collect/CollectCompletedBolt.java | 5 +- .../iie/bolt/proxy/ProxyCompletionBolt.java | 2 +- .../bolt/security/SecurityCompletionBolt.java | 4 +- .../ac/iie/utils/general/TransFormUtils.java | 22 +- .../cn/ac/iie/utils/hbase/HbaseUtils.java | 36 +- src/main/java/log4j.properties | 2 +- .../java/cn/ac/iie/test/hbase/HBaseTest.java | 140 ++++ 16 files changed, 1962 insertions(+), 60 deletions(-) create mode 100644 properties/core-site.xml create mode 100644 src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java create mode 100644 src/main/java/cn/ac/iie/bean/proxy/ProxySessionRecordLog.java create mode 100644 src/main/java/cn/ac/iie/bean/radius/RadiusSessionRecordLog.java create mode 100644 src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java create mode 100644 src/test/java/cn/ac/iie/test/hbase/HBaseTest.java diff --git a/pom.xml b/pom.xml index 88624d0..def6d01 100644 --- a/pom.xml +++ b/pom.xml @@ -142,7 +142,7 @@ com.alibaba fastjson - 1.2.47 + 1.2.59 diff --git a/properties/core-site.xml b/properties/core-site.xml new file mode 100644 index 0000000..93dfb1d --- /dev/null +++ b/properties/core-site.xml @@ -0,0 +1,71 @@ + + + + + + + + + fs.defaultFS + hdfs://ns1 + + + hadoop.tmp.dir + file:/opt/hadoop/tmp + + + io.file.buffer.size + 131702 + + + hadoop.proxyuser.root.hosts + * + + + hadoop.proxyuser.root.groups + * + + + hadoop.logfile.size + 10000000 + The max size of each log file + + + + hadoop.logfile.count + 1 + The max number of log files + + + ha.zookeeper.quorum + master:2181,slave1:2181,slave2:2181 + + +    +     fs.hdfs.impl   +     org.apache.hadoop.hdfs.DistributedFileSystem   +     The FileSystem for hdfs: uris.   + + + +io.compression.codecs +com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec + + +io.compression.codec.lzo.class +com.hadoop.compression.lzo.LzoCodec + + + diff --git a/properties/hbase-site.xml b/properties/hbase-site.xml index d26f7df..54554e4 100644 --- a/properties/hbase-site.xml +++ b/properties/hbase-site.xml @@ -23,7 +23,7 @@ hbase.rootdir - hdfs://ns1/hbase/hbase-1.4.9 + hdfs://ns1/hbase-1.4.9 hbase.cluster.distributed @@ -31,13 +31,13 @@ hbase.zookeeper.quorum - master,slave1,slave2 + 192.168.40.119,192.168.40.122,192.168.40.123 hbase.master.info.port 60010 -#开启schema支持 对应hbase的namespace + phoenix.schema.isNamespaceMappingEnabled true @@ -46,4 +46,32 @@ phoenix.schema.mapSystemTablesToNamespace true + + hbase.client.keyvalue.maxsize + 99428800 + + + hbase.server.keyvalue.maxsize + 99428800 + + + hbase.regionserver.wal.codec + org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec + + + phoenix.query.timeoutMs + 1800000 + + + hbase.rpc.timeout + 1200000 + + + hbase.client.scanner.caching + 1000 + + + hbase.client.scanner.timeout.period + 1200000 + diff --git a/properties/hdfs-site.xml b/properties/hdfs-site.xml index 845b195..1e148b7 100644 --- a/properties/hdfs-site.xml +++ b/properties/hdfs-site.xml @@ -17,13 +17,13 @@ - + dfs.namenode.name.dir - file:/home/bigdata/hadoop/dfs/name + file:/home/ceiec/hadoop/dfs/name dfs.datanode.data.dir - file:/home/bigdata/hadoop/dfs/data + file:/home/ceiec/hadoop/dfs/data dfs.replication @@ -31,11 +31,11 @@ dfs.namenode.secondary.http-address - master:9001 + 192.168.40.119:9001 - dfs.webhdfs.enabled - true + dfs.webhdfs.enabled + true dfs.permissions @@ -60,32 +60,32 @@ dfs.namenode.rpc-address.ns1.nn1 - master:9000 + 192.168.40.119:8020 dfs.namenode.http-address.ns1.nn1 - master:50070 + 192.168.40.119:50070 dfs.namenode.rpc-address.ns1.nn2 - slave1:9000 + 192.168.40.122:8020 dfs.namenode.http-address.ns1.nn2 - slave1:50070 + 192.168.40.122:50070 dfs.namenode.shared.edits.dir - qjournal://slave1:8485;slave2:8485;master:8485/ns1 + qjournal://192.168.40.119:8485;192.168.40.122:8485;192.168.40.123:8485/ns1 dfs.journalnode.edits.dir - /home/bigdata/hadoop/journal + /home/ceiec/hadoop/journal diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index fb65910..8646d78 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -1,11 +1,11 @@ #管理kafka地址 -bootstrap.servers=192.168.40.152:9092 +bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092 #zookeeper 地址 -zookeeper.servers=192.168.40.152:2181 +zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 #hbase zookeeper地址 -hbase.zookeeper.servers=192.168.40.203:2181,192.168.40.206:2181 +hbase.zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 #hbase tablename hbase.table.name=subcriber_info @@ -14,25 +14,25 @@ hbase.table.name=subcriber_info auto.offset.reset=latest #kafka broker下的topic名称 -kafka.topic=SECURITY-POLICY-LOG +kafka.topic=SECURITY-EVENT-LOG #读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； group.id=security-policy-191112 #输出topic -results.output.topic=SECURITY-POLICY-COMPLETED-LOG +results.output.topic=SECURITY-EVENT-COMPLETED-LOG #storm topology workers -topology.workers=1 +topology.workers=2 #spout并行度 建议与kafka分区数相同 -spout.parallelism=1 +spout.parallelism=3 #处理补全操作的bolt并行度-worker的倍数 -datacenter.bolt.parallelism=1 +datacenter.bolt.parallelism=10 -#写入kafka的并行度 -kafka.bolt.parallelism=1 +#写入kafka的并行度10 +kafka.bolt.parallelism=10 #定位库地址 ip.library=/home/ceiec/topology/dat/ diff --git a/src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java b/src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java new file mode 100644 index 0000000..e73c2d2 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java @@ -0,0 +1,684 @@ +package cn.ac.iie.bean.connection; + +import cn.ac.iie.bean.PublicSessionRecordLog; + +/** + * 除radius之外 + * + * @author qidaijie + */ +public class ConnectionRecordLog extends PublicSessionRecordLog { + + +//TODO HTTP协议属性 21 + + private String http_url; + private String http_host; + private String http_domain; + private String http_request_line; + private String http_response_line; + private String http_request_header; + private String http_response_header; + private String http_request_body; + private String http_response_body; + private String http_request_body_key; + private String http_response_body_key; + private int http_proxy_flag; + private int http_sequence; + private String http_snapshot; + private String http_cookie; + private String http_referer; + private String http_user_agent; + private String http_content_length; + private String http_content_type; + private String http_set_cookie; + private String http_version; + +//TODO MAIL协议属性 9 + + private String mail_protocol_type; + private String mail_sender; + private String mail_receiver; + private String mail_subject; + private String mail_content; + private String mail_attachment_name; + private String mail_attachment_content; + private String mail_eml_file; + private String mail_snapshot; + +//TODO 3DNS协议属性 18 + + private int dns_message_id; + private int dns_qr; + private int dns_opcode; + private int dns_aa; + private int dns_tc; + private int dns_rd; + private int dns_ra; + private int dns_rcode; + private int dns_qdcount; + private int dns_ancount; + private int dns_nscount; + private int dns_arcount; + private String dns_qname; + private int dns_qtype; + private int dns_qclass; + private String dns_cname; + private int dns_sub; + private String dns_rr; + +//TODO SSL协议属性 13 + + + private int ssl_pinningst; + private int ssl_intercept_state; + private int ssl_server_side_latency; + private int ssl_client_side_latency; + private int ssl_cert_verify; + private int ssl_con_latency_ms; + private String ssl_version; + private String ssl_sni; + private String ssl_san; + private String ssl_cn; + private String ssl_server_side_version; + private String ssl_client_side_version; + private String ssl_error; + + //TODO FTP协议属性 2 + private String ftp_url; + private String ftp_content; + + + //TODO BGP协议属性 3 + private int bgp_type; + private String bgp_as_num; + private String bgp_route; + + //TODO VOIP协议属性 4 + private String voip_calling_account; + private String voip_called_account; + private String voip_calling_number; + private String voip_called_number; + + //TODO STREAMING_MEDIA协议属性 2 + private String streaming_media_url; + private String streaming_media_protocol; + + + public String getHttp_url() { + return http_url; + } + + public void setHttp_url(String http_url) { + this.http_url = http_url; + } + + public String getHttp_host() { + return http_host; + } + + public void setHttp_host(String http_host) { + this.http_host = http_host; + } + + public String getHttp_domain() { + return http_domain; + } + + public void setHttp_domain(String http_domain) { + this.http_domain = http_domain; + } + + public String getHttp_request_line() { + return http_request_line; + } + + public void setHttp_request_line(String http_request_line) { + this.http_request_line = http_request_line; + } + + public String getHttp_response_line() { + return http_response_line; + } + + public void setHttp_response_line(String http_response_line) { + this.http_response_line = http_response_line; + } + + public String getHttp_request_header() { + return http_request_header; + } + + public void setHttp_request_header(String http_request_header) { + this.http_request_header = http_request_header; + } + + public String getHttp_response_header() { + return http_response_header; + } + + public void setHttp_response_header(String http_response_header) { + this.http_response_header = http_response_header; + } + + public String getHttp_request_body() { + return http_request_body; + } + + public void setHttp_request_body(String http_request_body) { + this.http_request_body = http_request_body; + } + + public String getHttp_response_body() { + return http_response_body; + } + + public void setHttp_response_body(String http_response_body) { + this.http_response_body = http_response_body; + } + + public String getHttp_request_body_key() { + return http_request_body_key; + } + + public void setHttp_request_body_key(String http_request_body_key) { + this.http_request_body_key = http_request_body_key; + } + + public String getHttp_response_body_key() { + return http_response_body_key; + } + + public void setHttp_response_body_key(String http_response_body_key) { + this.http_response_body_key = http_response_body_key; + } + + public int getHttp_proxy_flag() { + return http_proxy_flag; + } + + public void setHttp_proxy_flag(int http_proxy_flag) { + this.http_proxy_flag = http_proxy_flag; + } + + public int getHttp_sequence() { + return http_sequence; + } + + public void setHttp_sequence(int http_sequence) { + this.http_sequence = http_sequence; + } + + public String getHttp_snapshot() { + return http_snapshot; + } + + public void setHttp_snapshot(String http_snapshot) { + this.http_snapshot = http_snapshot; + } + + public String getHttp_cookie() { + return http_cookie; + } + + public void setHttp_cookie(String http_cookie) { + this.http_cookie = http_cookie; + } + + public String getHttp_referer() { + return http_referer; + } + + public void setHttp_referer(String http_referer) { + this.http_referer = http_referer; + } + + public String getHttp_user_agent() { + return http_user_agent; + } + + public void setHttp_user_agent(String http_user_agent) { + this.http_user_agent = http_user_agent; + } + + public String getHttp_content_length() { + return http_content_length; + } + + public void setHttp_content_length(String http_content_length) { + this.http_content_length = http_content_length; + } + + public String getHttp_content_type() { + return http_content_type; + } + + public void setHttp_content_type(String http_content_type) { + this.http_content_type = http_content_type; + } + + public String getHttp_set_cookie() { + return http_set_cookie; + } + + public void setHttp_set_cookie(String http_set_cookie) { + this.http_set_cookie = http_set_cookie; + } + + public String getHttp_version() { + return http_version; + } + + public void setHttp_version(String http_version) { + this.http_version = http_version; + } + + public String getMail_protocol_type() { + return mail_protocol_type; + } + + public void setMail_protocol_type(String mail_protocol_type) { + this.mail_protocol_type = mail_protocol_type; + } + + public String getMail_sender() { + return mail_sender; + } + + public void setMail_sender(String mail_sender) { + this.mail_sender = mail_sender; + } + + public String getMail_receiver() { + return mail_receiver; + } + + public void setMail_receiver(String mail_receiver) { + this.mail_receiver = mail_receiver; + } + + public String getMail_subject() { + return mail_subject; + } + + public void setMail_subject(String mail_subject) { + this.mail_subject = mail_subject; + } + + public String getMail_content() { + return mail_content; + } + + public void setMail_content(String mail_content) { + this.mail_content = mail_content; + } + + public String getMail_attachment_name() { + return mail_attachment_name; + } + + public void setMail_attachment_name(String mail_attachment_name) { + this.mail_attachment_name = mail_attachment_name; + } + + public String getMail_attachment_content() { + return mail_attachment_content; + } + + public void setMail_attachment_content(String mail_attachment_content) { + this.mail_attachment_content = mail_attachment_content; + } + + public String getMail_eml_file() { + return mail_eml_file; + } + + public void setMail_eml_file(String mail_eml_file) { + this.mail_eml_file = mail_eml_file; + } + + public String getMail_snapshot() { + return mail_snapshot; + } + + public void setMail_snapshot(String mail_snapshot) { + this.mail_snapshot = mail_snapshot; + } + + public int getDns_message_id() { + return dns_message_id; + } + + public void setDns_message_id(int dns_message_id) { + this.dns_message_id = dns_message_id; + } + + public int getDns_qr() { + return dns_qr; + } + + public void setDns_qr(int dns_qr) { + this.dns_qr = dns_qr; + } + + public int getDns_opcode() { + return dns_opcode; + } + + public void setDns_opcode(int dns_opcode) { + this.dns_opcode = dns_opcode; + } + + public int getDns_aa() { + return dns_aa; + } + + public void setDns_aa(int dns_aa) { + this.dns_aa = dns_aa; + } + + public int getDns_tc() { + return dns_tc; + } + + public void setDns_tc(int dns_tc) { + this.dns_tc = dns_tc; + } + + public int getDns_rd() { + return dns_rd; + } + + public void setDns_rd(int dns_rd) { + this.dns_rd = dns_rd; + } + + public int getDns_ra() { + return dns_ra; + } + + public void setDns_ra(int dns_ra) { + this.dns_ra = dns_ra; + } + + public int getDns_rcode() { + return dns_rcode; + } + + public void setDns_rcode(int dns_rcode) { + this.dns_rcode = dns_rcode; + } + + public int getDns_qdcount() { + return dns_qdcount; + } + + public void setDns_qdcount(int dns_qdcount) { + this.dns_qdcount = dns_qdcount; + } + + public int getDns_ancount() { + return dns_ancount; + } + + public void setDns_ancount(int dns_ancount) { + this.dns_ancount = dns_ancount; + } + + public int getDns_nscount() { + return dns_nscount; + } + + public void setDns_nscount(int dns_nscount) { + this.dns_nscount = dns_nscount; + } + + public int getDns_arcount() { + return dns_arcount; + } + + public void setDns_arcount(int dns_arcount) { + this.dns_arcount = dns_arcount; + } + + public String getDns_qname() { + return dns_qname; + } + + public void setDns_qname(String dns_qname) { + this.dns_qname = dns_qname; + } + + public int getDns_qtype() { + return dns_qtype; + } + + public void setDns_qtype(int dns_qtype) { + this.dns_qtype = dns_qtype; + } + + public int getDns_qclass() { + return dns_qclass; + } + + public void setDns_qclass(int dns_qclass) { + this.dns_qclass = dns_qclass; + } + + public String getDns_cname() { + return dns_cname; + } + + public void setDns_cname(String dns_cname) { + this.dns_cname = dns_cname; + } + + public int getDns_sub() { + return dns_sub; + } + + public void setDns_sub(int dns_sub) { + this.dns_sub = dns_sub; + } + + public String getDns_rr() { + return dns_rr; + } + + public void setDns_rr(String dns_rr) { + this.dns_rr = dns_rr; + } + + public int getSsl_pinningst() { + return ssl_pinningst; + } + + public void setSsl_pinningst(int ssl_pinningst) { + this.ssl_pinningst = ssl_pinningst; + } + + public int getSsl_intercept_state() { + return ssl_intercept_state; + } + + public void setSsl_intercept_state(int ssl_intercept_state) { + this.ssl_intercept_state = ssl_intercept_state; + } + + public int getSsl_server_side_latency() { + return ssl_server_side_latency; + } + + public void setSsl_server_side_latency(int ssl_server_side_latency) { + this.ssl_server_side_latency = ssl_server_side_latency; + } + + public int getSsl_client_side_latency() { + return ssl_client_side_latency; + } + + public void setSsl_client_side_latency(int ssl_client_side_latency) { + this.ssl_client_side_latency = ssl_client_side_latency; + } + + public int getSsl_cert_verify() { + return ssl_cert_verify; + } + + public void setSsl_cert_verify(int ssl_cert_verify) { + this.ssl_cert_verify = ssl_cert_verify; + } + + public int getSsl_con_latency_ms() { + return ssl_con_latency_ms; + } + + public void setSsl_con_latency_ms(int ssl_con_latency_ms) { + this.ssl_con_latency_ms = ssl_con_latency_ms; + } + + public String getSsl_version() { + return ssl_version; + } + + public void setSsl_version(String ssl_version) { + this.ssl_version = ssl_version; + } + + public String getSsl_sni() { + return ssl_sni; + } + + public void setSsl_sni(String ssl_sni) { + this.ssl_sni = ssl_sni; + } + + public String getSsl_san() { + return ssl_san; + } + + public void setSsl_san(String ssl_san) { + this.ssl_san = ssl_san; + } + + public String getSsl_cn() { + return ssl_cn; + } + + public void setSsl_cn(String ssl_cn) { + this.ssl_cn = ssl_cn; + } + + public String getSsl_server_side_version() { + return ssl_server_side_version; + } + + public void setSsl_server_side_version(String ssl_server_side_version) { + this.ssl_server_side_version = ssl_server_side_version; + } + + public String getSsl_client_side_version() { + return ssl_client_side_version; + } + + public void setSsl_client_side_version(String ssl_client_side_version) { + this.ssl_client_side_version = ssl_client_side_version; + } + + public String getSsl_error() { + return ssl_error; + } + + public void setSsl_error(String ssl_error) { + this.ssl_error = ssl_error; + } + + public String getFtp_url() { + return ftp_url; + } + + public void setFtp_url(String ftp_url) { + this.ftp_url = ftp_url; + } + + public String getFtp_content() { + return ftp_content; + } + + public void setFtp_content(String ftp_content) { + this.ftp_content = ftp_content; + } + + public int getBgp_type() { + return bgp_type; + } + + public void setBgp_type(int bgp_type) { + this.bgp_type = bgp_type; + } + + public String getBgp_as_num() { + return bgp_as_num; + } + + public void setBgp_as_num(String bgp_as_num) { + this.bgp_as_num = bgp_as_num; + } + + public String getBgp_route() { + return bgp_route; + } + + public void setBgp_route(String bgp_route) { + this.bgp_route = bgp_route; + } + + public String getVoip_calling_account() { + return voip_calling_account; + } + + public void setVoip_calling_account(String voip_calling_account) { + this.voip_calling_account = voip_calling_account; + } + + public String getVoip_called_account() { + return voip_called_account; + } + + public void setVoip_called_account(String voip_called_account) { + this.voip_called_account = voip_called_account; + } + + public String getVoip_calling_number() { + return voip_calling_number; + } + + public void setVoip_calling_number(String voip_calling_number) { + this.voip_calling_number = voip_calling_number; + } + + public String getVoip_called_number() { + return voip_called_number; + } + + public void setVoip_called_number(String voip_called_number) { + this.voip_called_number = voip_called_number; + } + + + public String getStreaming_media_url() { + return streaming_media_url; + } + + public void setStreaming_media_url(String streaming_media_url) { + this.streaming_media_url = streaming_media_url; + } + + public String getStreaming_media_protocol() { + return streaming_media_protocol; + } + + public void setStreaming_media_protocol(String streaming_media_protocol) { + this.streaming_media_protocol = streaming_media_protocol; + } +} diff --git a/src/main/java/cn/ac/iie/bean/proxy/ProxySessionRecordLog.java b/src/main/java/cn/ac/iie/bean/proxy/ProxySessionRecordLog.java new file mode 100644 index 0000000..ba5dc85 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/proxy/ProxySessionRecordLog.java @@ -0,0 +1,203 @@ +package cn.ac.iie.bean.proxy; + + +import cn.ac.iie.bean.PublicSessionRecordLog; + +/** + * @author qidaijie + */ +public class ProxySessionRecordLog extends PublicSessionRecordLog { + + +//TODO HTTP协议属性 21 + + private String http_url; + private String http_host; + private String http_domain; + private String http_request_line; + private String http_response_line; + private String http_request_header; + private String http_response_header; + private String http_request_body; + private String http_response_body; + private String http_request_body_key; + private String http_response_body_key; + private int http_proxy_flag; + private int http_sequence; + private String http_snapshot; + private String http_cookie; + private String http_referer; + private String http_user_agent; + private String http_content_length; + private String http_content_type; + private String http_set_cookie; + private String http_version; + + public String getHttp_url() { + return http_url; + } + + public void setHttp_url(String http_url) { + this.http_url = http_url; + } + + public String getHttp_host() { + return http_host; + } + + public void setHttp_host(String http_host) { + this.http_host = http_host; + } + + public String getHttp_domain() { + return http_domain; + } + + public void setHttp_domain(String http_domain) { + this.http_domain = http_domain; + } + + public String getHttp_request_line() { + return http_request_line; + } + + public void setHttp_request_line(String http_request_line) { + this.http_request_line = http_request_line; + } + + public String getHttp_response_line() { + return http_response_line; + } + + public void setHttp_response_line(String http_response_line) { + this.http_response_line = http_response_line; + } + + public String getHttp_request_header() { + return http_request_header; + } + + public void setHttp_request_header(String http_request_header) { + this.http_request_header = http_request_header; + } + + public String getHttp_response_header() { + return http_response_header; + } + + public void setHttp_response_header(String http_response_header) { + this.http_response_header = http_response_header; + } + + public String getHttp_request_body() { + return http_request_body; + } + + public void setHttp_request_body(String http_request_body) { + this.http_request_body = http_request_body; + } + + public String getHttp_response_body() { + return http_response_body; + } + + public void setHttp_response_body(String http_response_body) { + this.http_response_body = http_response_body; + } + + public String getHttp_request_body_key() { + return http_request_body_key; + } + + public void setHttp_request_body_key(String http_request_body_key) { + this.http_request_body_key = http_request_body_key; + } + + public String getHttp_response_body_key() { + return http_response_body_key; + } + + public void setHttp_response_body_key(String http_response_body_key) { + this.http_response_body_key = http_response_body_key; + } + + public int getHttp_proxy_flag() { + return http_proxy_flag; + } + + public void setHttp_proxy_flag(int http_proxy_flag) { + this.http_proxy_flag = http_proxy_flag; + } + + public int getHttp_sequence() { + return http_sequence; + } + + public void setHttp_sequence(int http_sequence) { + this.http_sequence = http_sequence; + } + + public String getHttp_snapshot() { + return http_snapshot; + } + + public void setHttp_snapshot(String http_snapshot) { + this.http_snapshot = http_snapshot; + } + + public String getHttp_cookie() { + return http_cookie; + } + + public void setHttp_cookie(String http_cookie) { + this.http_cookie = http_cookie; + } + + public String getHttp_referer() { + return http_referer; + } + + public void setHttp_referer(String http_referer) { + this.http_referer = http_referer; + } + + public String getHttp_user_agent() { + return http_user_agent; + } + + public void setHttp_user_agent(String http_user_agent) { + this.http_user_agent = http_user_agent; + } + + public String getHttp_content_length() { + return http_content_length; + } + + public void setHttp_content_length(String http_content_length) { + this.http_content_length = http_content_length; + } + + public String getHttp_content_type() { + return http_content_type; + } + + public void setHttp_content_type(String http_content_type) { + this.http_content_type = http_content_type; + } + + public String getHttp_set_cookie() { + return http_set_cookie; + } + + public void setHttp_set_cookie(String http_set_cookie) { + this.http_set_cookie = http_set_cookie; + } + + public String getHttp_version() { + return http_version; + } + + public void setHttp_version(String http_version) { + this.http_version = http_version; + } +} diff --git a/src/main/java/cn/ac/iie/bean/radius/RadiusSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/radius/RadiusSessionRecordLog.java new file mode 100644 index 0000000..4dd11a9 --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/radius/RadiusSessionRecordLog.java @@ -0,0 +1,89 @@ +package cn.ac.iie.bean.radius; + + +import cn.ac.iie.bean.PublicSessionRecordLog; + +/** + * Radius 日志 + * + * @author qidaijie + */ +public class RadiusSessionRecordLog extends PublicSessionRecordLog { + + +//TODO RADIUS协议属性 8 + + private int radius_session_timeout; + private int radius_idle_timeout; + private int radius_acct_status_type; + private int radius_acct_terminate_cause; + private int radius_packet_type; + private String radius_nas_ip; + private String radius_framed_ip; + private String radius_account; + + + public int getRadius_session_timeout() { + return radius_session_timeout; + } + + public void setRadius_session_timeout(int radius_session_timeout) { + this.radius_session_timeout = radius_session_timeout; + } + + public int getRadius_idle_timeout() { + return radius_idle_timeout; + } + + public void setRadius_idle_timeout(int radius_idle_timeout) { + this.radius_idle_timeout = radius_idle_timeout; + } + + public int getRadius_acct_status_type() { + return radius_acct_status_type; + } + + public void setRadius_acct_status_type(int radius_acct_status_type) { + this.radius_acct_status_type = radius_acct_status_type; + } + + public int getRadius_acct_terminate_cause() { + return radius_acct_terminate_cause; + } + + public void setRadius_acct_terminate_cause(int radius_acct_terminate_cause) { + this.radius_acct_terminate_cause = radius_acct_terminate_cause; + } + + public int getRadius_packet_type() { + return radius_packet_type; + } + + public void setRadius_packet_type(int radius_packet_type) { + this.radius_packet_type = radius_packet_type; + } + + public String getRadius_nas_ip() { + return radius_nas_ip; + } + + public void setRadius_nas_ip(String radius_nas_ip) { + this.radius_nas_ip = radius_nas_ip; + } + + public String getRadius_framed_ip() { + return radius_framed_ip; + } + + public void setRadius_framed_ip(String radius_framed_ip) { + this.radius_framed_ip = radius_framed_ip; + } + + public String getRadius_account() { + return radius_account; + } + + public void setRadius_account(String radius_account) { + this.radius_account = radius_account; + } +} diff --git a/src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java b/src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java new file mode 100644 index 0000000..e266b5e --- /dev/null +++ b/src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java @@ -0,0 +1,684 @@ +package cn.ac.iie.bean.security; + +import cn.ac.iie.bean.PublicSessionRecordLog; + +/** + * 策略 + * + * @author qidaijie + */ +public class SecurityPolicyLog extends PublicSessionRecordLog { + + +//TODO HTTP协议属性 21 + + private String http_url; + private String http_host; + private String http_domain; + private String http_request_line; + private String http_response_line; + private String http_request_header; + private String http_response_header; + private String http_request_body; + private String http_response_body; + private String http_request_body_key; + private String http_response_body_key; + private int http_proxy_flag; + private int http_sequence; + private String http_snapshot; + private String http_cookie; + private String http_referer; + private String http_user_agent; + private String http_content_length; + private String http_content_type; + private String http_set_cookie; + private String http_version; + +//TODO MAIL协议属性 9 + + private String mail_protocol_type; + private String mail_sender; + private String mail_receiver; + private String mail_subject; + private String mail_content; + private String mail_attachment_name; + private String mail_attachment_content; + private String mail_eml_file; + private String mail_snapshot; + +//TODO 3DNS协议属性 18 + + private int dns_message_id; + private int dns_qr; + private int dns_opcode; + private int dns_aa; + private int dns_tc; + private int dns_rd; + private int dns_ra; + private int dns_rcode; + private int dns_qdcount; + private int dns_ancount; + private int dns_nscount; + private int dns_arcount; + private String dns_qname; + private int dns_qtype; + private int dns_qclass; + private String dns_cname; + private int dns_sub; + private String dns_rr; + +//TODO SSL协议属性 13 + + + private int ssl_pinningst; + private int ssl_intercept_state; + private int ssl_server_side_latency; + private int ssl_client_side_latency; + private int ssl_cert_verify; + private int ssl_con_latency_ms; + private String ssl_version; + private String ssl_sni; + private String ssl_san; + private String ssl_cn; + private String ssl_server_side_version; + private String ssl_client_side_version; + private String ssl_error; + + //TODO FTP协议属性 2 + private String ftp_url; + private String ftp_content; + + + //TODO BGP协议属性 3 + private int bgp_type; + private String bgp_as_num; + private String bgp_route; + + //TODO VOIP协议属性 4 + private String voip_calling_account; + private String voip_called_account; + private String voip_calling_number; + private String voip_called_number; + + //TODO STREAMING_MEDIA协议属性 2 + private String streaming_media_url; + private String streaming_media_protocol; + + + public String getHttp_url() { + return http_url; + } + + public void setHttp_url(String http_url) { + this.http_url = http_url; + } + + public String getHttp_host() { + return http_host; + } + + public void setHttp_host(String http_host) { + this.http_host = http_host; + } + + public String getHttp_domain() { + return http_domain; + } + + public void setHttp_domain(String http_domain) { + this.http_domain = http_domain; + } + + public String getHttp_request_line() { + return http_request_line; + } + + public void setHttp_request_line(String http_request_line) { + this.http_request_line = http_request_line; + } + + public String getHttp_response_line() { + return http_response_line; + } + + public void setHttp_response_line(String http_response_line) { + this.http_response_line = http_response_line; + } + + public String getHttp_request_header() { + return http_request_header; + } + + public void setHttp_request_header(String http_request_header) { + this.http_request_header = http_request_header; + } + + public String getHttp_response_header() { + return http_response_header; + } + + public void setHttp_response_header(String http_response_header) { + this.http_response_header = http_response_header; + } + + public String getHttp_request_body() { + return http_request_body; + } + + public void setHttp_request_body(String http_request_body) { + this.http_request_body = http_request_body; + } + + public String getHttp_response_body() { + return http_response_body; + } + + public void setHttp_response_body(String http_response_body) { + this.http_response_body = http_response_body; + } + + public String getHttp_request_body_key() { + return http_request_body_key; + } + + public void setHttp_request_body_key(String http_request_body_key) { + this.http_request_body_key = http_request_body_key; + } + + public String getHttp_response_body_key() { + return http_response_body_key; + } + + public void setHttp_response_body_key(String http_response_body_key) { + this.http_response_body_key = http_response_body_key; + } + + public int getHttp_proxy_flag() { + return http_proxy_flag; + } + + public void setHttp_proxy_flag(int http_proxy_flag) { + this.http_proxy_flag = http_proxy_flag; + } + + public int getHttp_sequence() { + return http_sequence; + } + + public void setHttp_sequence(int http_sequence) { + this.http_sequence = http_sequence; + } + + public String getHttp_snapshot() { + return http_snapshot; + } + + public void setHttp_snapshot(String http_snapshot) { + this.http_snapshot = http_snapshot; + } + + public String getHttp_cookie() { + return http_cookie; + } + + public void setHttp_cookie(String http_cookie) { + this.http_cookie = http_cookie; + } + + public String getHttp_referer() { + return http_referer; + } + + public void setHttp_referer(String http_referer) { + this.http_referer = http_referer; + } + + public String getHttp_user_agent() { + return http_user_agent; + } + + public void setHttp_user_agent(String http_user_agent) { + this.http_user_agent = http_user_agent; + } + + public String getHttp_content_length() { + return http_content_length; + } + + public void setHttp_content_length(String http_content_length) { + this.http_content_length = http_content_length; + } + + public String getHttp_content_type() { + return http_content_type; + } + + public void setHttp_content_type(String http_content_type) { + this.http_content_type = http_content_type; + } + + public String getHttp_set_cookie() { + return http_set_cookie; + } + + public void setHttp_set_cookie(String http_set_cookie) { + this.http_set_cookie = http_set_cookie; + } + + public String getHttp_version() { + return http_version; + } + + public void setHttp_version(String http_version) { + this.http_version = http_version; + } + + public String getMail_protocol_type() { + return mail_protocol_type; + } + + public void setMail_protocol_type(String mail_protocol_type) { + this.mail_protocol_type = mail_protocol_type; + } + + public String getMail_sender() { + return mail_sender; + } + + public void setMail_sender(String mail_sender) { + this.mail_sender = mail_sender; + } + + public String getMail_receiver() { + return mail_receiver; + } + + public void setMail_receiver(String mail_receiver) { + this.mail_receiver = mail_receiver; + } + + public String getMail_subject() { + return mail_subject; + } + + public void setMail_subject(String mail_subject) { + this.mail_subject = mail_subject; + } + + public String getMail_content() { + return mail_content; + } + + public void setMail_content(String mail_content) { + this.mail_content = mail_content; + } + + public String getMail_attachment_name() { + return mail_attachment_name; + } + + public void setMail_attachment_name(String mail_attachment_name) { + this.mail_attachment_name = mail_attachment_name; + } + + public String getMail_attachment_content() { + return mail_attachment_content; + } + + public void setMail_attachment_content(String mail_attachment_content) { + this.mail_attachment_content = mail_attachment_content; + } + + public String getMail_eml_file() { + return mail_eml_file; + } + + public void setMail_eml_file(String mail_eml_file) { + this.mail_eml_file = mail_eml_file; + } + + public String getMail_snapshot() { + return mail_snapshot; + } + + public void setMail_snapshot(String mail_snapshot) { + this.mail_snapshot = mail_snapshot; + } + + public int getDns_message_id() { + return dns_message_id; + } + + public void setDns_message_id(int dns_message_id) { + this.dns_message_id = dns_message_id; + } + + public int getDns_qr() { + return dns_qr; + } + + public void setDns_qr(int dns_qr) { + this.dns_qr = dns_qr; + } + + public int getDns_opcode() { + return dns_opcode; + } + + public void setDns_opcode(int dns_opcode) { + this.dns_opcode = dns_opcode; + } + + public int getDns_aa() { + return dns_aa; + } + + public void setDns_aa(int dns_aa) { + this.dns_aa = dns_aa; + } + + public int getDns_tc() { + return dns_tc; + } + + public void setDns_tc(int dns_tc) { + this.dns_tc = dns_tc; + } + + public int getDns_rd() { + return dns_rd; + } + + public void setDns_rd(int dns_rd) { + this.dns_rd = dns_rd; + } + + public int getDns_ra() { + return dns_ra; + } + + public void setDns_ra(int dns_ra) { + this.dns_ra = dns_ra; + } + + public int getDns_rcode() { + return dns_rcode; + } + + public void setDns_rcode(int dns_rcode) { + this.dns_rcode = dns_rcode; + } + + public int getDns_qdcount() { + return dns_qdcount; + } + + public void setDns_qdcount(int dns_qdcount) { + this.dns_qdcount = dns_qdcount; + } + + public int getDns_ancount() { + return dns_ancount; + } + + public void setDns_ancount(int dns_ancount) { + this.dns_ancount = dns_ancount; + } + + public int getDns_nscount() { + return dns_nscount; + } + + public void setDns_nscount(int dns_nscount) { + this.dns_nscount = dns_nscount; + } + + public int getDns_arcount() { + return dns_arcount; + } + + public void setDns_arcount(int dns_arcount) { + this.dns_arcount = dns_arcount; + } + + public String getDns_qname() { + return dns_qname; + } + + public void setDns_qname(String dns_qname) { + this.dns_qname = dns_qname; + } + + public int getDns_qtype() { + return dns_qtype; + } + + public void setDns_qtype(int dns_qtype) { + this.dns_qtype = dns_qtype; + } + + public int getDns_qclass() { + return dns_qclass; + } + + public void setDns_qclass(int dns_qclass) { + this.dns_qclass = dns_qclass; + } + + public String getDns_cname() { + return dns_cname; + } + + public void setDns_cname(String dns_cname) { + this.dns_cname = dns_cname; + } + + public int getDns_sub() { + return dns_sub; + } + + public void setDns_sub(int dns_sub) { + this.dns_sub = dns_sub; + } + + public String getDns_rr() { + return dns_rr; + } + + public void setDns_rr(String dns_rr) { + this.dns_rr = dns_rr; + } + + public int getSsl_pinningst() { + return ssl_pinningst; + } + + public void setSsl_pinningst(int ssl_pinningst) { + this.ssl_pinningst = ssl_pinningst; + } + + public int getSsl_intercept_state() { + return ssl_intercept_state; + } + + public void setSsl_intercept_state(int ssl_intercept_state) { + this.ssl_intercept_state = ssl_intercept_state; + } + + public int getSsl_server_side_latency() { + return ssl_server_side_latency; + } + + public void setSsl_server_side_latency(int ssl_server_side_latency) { + this.ssl_server_side_latency = ssl_server_side_latency; + } + + public int getSsl_client_side_latency() { + return ssl_client_side_latency; + } + + public void setSsl_client_side_latency(int ssl_client_side_latency) { + this.ssl_client_side_latency = ssl_client_side_latency; + } + + public int getSsl_cert_verify() { + return ssl_cert_verify; + } + + public void setSsl_cert_verify(int ssl_cert_verify) { + this.ssl_cert_verify = ssl_cert_verify; + } + + public int getSsl_con_latency_ms() { + return ssl_con_latency_ms; + } + + public void setSsl_con_latency_ms(int ssl_con_latency_ms) { + this.ssl_con_latency_ms = ssl_con_latency_ms; + } + + public String getSsl_version() { + return ssl_version; + } + + public void setSsl_version(String ssl_version) { + this.ssl_version = ssl_version; + } + + public String getSsl_sni() { + return ssl_sni; + } + + public void setSsl_sni(String ssl_sni) { + this.ssl_sni = ssl_sni; + } + + public String getSsl_san() { + return ssl_san; + } + + public void setSsl_san(String ssl_san) { + this.ssl_san = ssl_san; + } + + public String getSsl_cn() { + return ssl_cn; + } + + public void setSsl_cn(String ssl_cn) { + this.ssl_cn = ssl_cn; + } + + public String getSsl_server_side_version() { + return ssl_server_side_version; + } + + public void setSsl_server_side_version(String ssl_server_side_version) { + this.ssl_server_side_version = ssl_server_side_version; + } + + public String getSsl_client_side_version() { + return ssl_client_side_version; + } + + public void setSsl_client_side_version(String ssl_client_side_version) { + this.ssl_client_side_version = ssl_client_side_version; + } + + public String getSsl_error() { + return ssl_error; + } + + public void setSsl_error(String ssl_error) { + this.ssl_error = ssl_error; + } + + public String getFtp_url() { + return ftp_url; + } + + public void setFtp_url(String ftp_url) { + this.ftp_url = ftp_url; + } + + public String getFtp_content() { + return ftp_content; + } + + public void setFtp_content(String ftp_content) { + this.ftp_content = ftp_content; + } + + public int getBgp_type() { + return bgp_type; + } + + public void setBgp_type(int bgp_type) { + this.bgp_type = bgp_type; + } + + public String getBgp_as_num() { + return bgp_as_num; + } + + public void setBgp_as_num(String bgp_as_num) { + this.bgp_as_num = bgp_as_num; + } + + public String getBgp_route() { + return bgp_route; + } + + public void setBgp_route(String bgp_route) { + this.bgp_route = bgp_route; + } + + public String getVoip_calling_account() { + return voip_calling_account; + } + + public void setVoip_calling_account(String voip_calling_account) { + this.voip_calling_account = voip_calling_account; + } + + public String getVoip_called_account() { + return voip_called_account; + } + + public void setVoip_called_account(String voip_called_account) { + this.voip_called_account = voip_called_account; + } + + public String getVoip_calling_number() { + return voip_calling_number; + } + + public void setVoip_calling_number(String voip_calling_number) { + this.voip_calling_number = voip_calling_number; + } + + public String getVoip_called_number() { + return voip_called_number; + } + + public void setVoip_called_number(String voip_called_number) { + this.voip_called_number = voip_called_number; + } + + + public String getStreaming_media_url() { + return streaming_media_url; + } + + public void setStreaming_media_url(String streaming_media_url) { + this.streaming_media_url = streaming_media_url; + } + + public String getStreaming_media_protocol() { + return streaming_media_protocol; + } + + public void setStreaming_media_protocol(String streaming_media_protocol) { + this.streaming_media_protocol = streaming_media_protocol; + } +} diff --git a/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java b/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java index 2e08d2e..6f923d5 100644 --- a/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java +++ b/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java @@ -1,6 +1,7 @@ package cn.ac.iie.bolt.collect; import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.system.TupleUtils; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; @@ -16,21 +17,19 @@ import java.util.HashMap; import java.util.Map; import static cn.ac.iie.utils.general.TransFormUtils.getCollectProtocolMessage; -import static cn.ac.iie.utils.hbase.HbaseUtils.change; +import static cn.ac.iie.utils.hbase.HBaseUtils.change; /** * 通联关系日志补全 * * @author qidaijie */ - public class CollectCompletedBolt extends BaseBasicBolt { private final static Logger logger = Logger.getLogger(CollectCompletedBolt.class); private static final long serialVersionUID = 4682827168247333522L; @Override public void prepare(Map stormConf, TopologyContext context) { - } @Override diff --git a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java index 838702f..fc65431 100644 --- a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java @@ -16,7 +16,7 @@ import java.util.HashMap; import java.util.Map; import static cn.ac.iie.utils.general.TransFormUtils.getProxyMessage; -import static cn.ac.iie.utils.hbase.HbaseUtils.change; +import static cn.ac.iie.utils.hbase.HBaseUtils.change; /** * 通联关系日志补全 diff --git a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java index 4947ccd..ca52c03 100644 --- a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java @@ -1,7 +1,7 @@ package cn.ac.iie.bolt.security; import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HbaseUtils; +import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.system.TupleUtils; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; @@ -38,7 +38,7 @@ public class SecurityCompletionBolt extends BaseBasicBolt { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { try { if (TupleUtils.isTick(tuple)) { - HbaseUtils.change(); + HBaseUtils.change(); } else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index 66f5c49..0340776 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -1,8 +1,11 @@ package cn.ac.iie.utils.general; -import cn.ac.iie.bean.*; +import cn.ac.iie.bean.connection.ConnectionRecordLog; +import cn.ac.iie.bean.proxy.ProxySessionRecordLog; +import cn.ac.iie.bean.radius.RadiusSessionRecordLog; +import cn.ac.iie.bean.security.SecurityPolicyLog; import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HbaseUtils; +import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.system.SnowflakeId; import com.alibaba.fastjson.JSONObject; import com.zdjizhi.utils.IpLookup; @@ -77,7 +80,7 @@ public class TransFormUtils { proxySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); proxySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); //TODO 集成AAA数据subscribe_id -数据端补全 - proxySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); + proxySessionRecordLog.setCommon_subscriber_id(HBaseUtils.getAccount(clientIp)); proxySessionRecordLog.setHttp_domain(getTopDomain(null, proxySessionRecordLog.getHttp_host())); return JSONObject.toJSONString(proxySessionRecordLog); } catch (Exception e) { @@ -107,7 +110,7 @@ public class TransFormUtils { securitySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); securitySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); //TODO 集成AAA数据subscribe_id -数据端补全 - securitySessionRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); + securitySessionRecordLog.setCommon_subscriber_id(HBaseUtils.getAccount(clientIp)); securitySessionRecordLog.setHttp_domain(getTopDomain(securitySessionRecordLog.getSsl_sni(), securitySessionRecordLog.getHttp_host())); return JSONObject.toJSONString(securitySessionRecordLog); } catch (Exception e) { @@ -125,7 +128,7 @@ public class TransFormUtils { * @return 补全后的日志 */ public static String getCollectProtocolMessage(String message) { - CollectProtocolRecordLog collectProtocolRecordLog = JSONObject.parseObject(message, CollectProtocolRecordLog.class); + ConnectionRecordLog collectProtocolRecordLog = JSONObject.parseObject(message, ConnectionRecordLog.class); String serverIp = collectProtocolRecordLog.getCommon_server_ip(); String clientIp = collectProtocolRecordLog.getCommon_client_ip(); try { @@ -136,7 +139,7 @@ public class TransFormUtils { collectProtocolRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); collectProtocolRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); //TODO 集成AAA数据subscribe_id -数据端补全 - collectProtocolRecordLog.setCommon_subscriber_id(HbaseUtils.getAccount(clientIp)); + collectProtocolRecordLog.setCommon_subscriber_id(HBaseUtils.getAccount(clientIp)); collectProtocolRecordLog.setHttp_domain(getTopDomain(collectProtocolRecordLog.getSsl_sni(), collectProtocolRecordLog.getHttp_host())); return JSONObject.toJSONString(collectProtocolRecordLog); } catch (Exception e) { @@ -152,7 +155,7 @@ public class TransFormUtils { * @param host host * @return 顶级域名 */ - public static String getTopDomain(String sni, String host) { + private static String getTopDomain(String sni, String host) { if (StringUtil.isNotBlank(sni)) { return getDomain(sni); } else if (StringUtil.isNotBlank(host)) { @@ -181,10 +184,5 @@ public class TransFormUtils { return ""; } - public static void main(String[] args) { - String s = ipLookup.cityLookupDetail("2001:470:19:790::24"); - System.out.println(s); - } - } diff --git a/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java index b8fecda..f1db9aa 100644 --- a/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java +++ b/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java @@ -16,12 +16,14 @@ import java.util.HashMap; import java.util.Map; /** + * HBase 工具类 + * * @author qidaijie */ -@SuppressWarnings("all") -public class HbaseUtils { - private final static Logger logger = Logger.getLogger(HbaseUtils.class); - private static Map subIdMap = new HashMap<>(16); + +public class HBaseUtils { + private final static Logger logger = Logger.getLogger(HBaseUtils.class); + private static Map subIdMap = new HashMap<>(32); private static Connection connection; private static Long time; @@ -55,29 +57,30 @@ public class HbaseUtils { * @param endTime 结束时间 */ private static void timestampsFilter(Long startTime, Long endTime) { + Long begin = System.currentTimeMillis(); Table table = null; - TableName tableName = TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME); - Admin admin = null; + ResultScanner scanner = null; + Scan scan2 = new Scan(); try { table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); - Scan scan2 = new Scan(); scan2.setTimeRange(startTime, endTime); - ResultScanner scanner = table.getScanner(scan2); + scanner = table.getScanner(scan2); + logger.warn("读取HBase变量耗时：" + (System.currentTimeMillis() - begin)); for (Result result : scanner) { Cell[] cells = result.rawCells(); for (Cell cell : cells) { subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); } } - admin = connection.getAdmin(); - admin.flush(tableName); logger.warn("当前集合长度" + subIdMap.keySet().size()); - logger.warn("更新后集合keys：" + subIdMap.keySet()); + logger.warn("更新缓存耗时：" + (System.currentTimeMillis() - begin)); time = endTime; - scanner.close(); } catch (IOException e) { e.printStackTrace(); } finally { + if (scanner != null) { + scanner.close(); + } if (table != null) { try { table.close(); @@ -92,6 +95,7 @@ public class HbaseUtils { * 获取所有的 key value */ private static void getAll() { + Long begin = System.currentTimeMillis(); try { Table table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); Scan scan2 = new Scan(); @@ -102,18 +106,20 @@ public class HbaseUtils { subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); } } - logger.warn("获取全量后集合keys：" + subIdMap.keySet()); + logger.warn("获取全量后集合长度：" + subIdMap.size()); + logger.warn("获取全量耗时：" + (System.currentTimeMillis() - begin)); scanner.close(); } catch (IOException e) { e.printStackTrace(); } } + /** * 获取 account * - * @param ip - * @return + * @param ip client_ip + * @return account */ public static String getAccount(String ip) { if (StringUtil.isNotBlank(ip)) { diff --git a/src/main/java/log4j.properties b/src/main/java/log4j.properties index c355401..17c0e9a 100644 --- a/src/main/java/log4j.properties +++ b/src/main/java/log4j.properties @@ -12,7 +12,7 @@ log4j.appender.file.Threshold=error log4j.appender.file.encoding=UTF-8 log4j.appender.file.Append=true #路径请用相对路径，做好相关测试输出到应用目下 -log4j.appender.file.file=galaxy-name.log +log4j.appender.file.file=storm-topology.log log4j.appender.file.DatePattern='.'yyyy-MM-dd log4j.appender.file.layout=org.apache.log4j.PatternLayout #log4j.appender.file.layout.ConversionPattern=%d{HH:mm:ss} %X{ip} [%t] %5p %c{1} %m%n diff --git a/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java new file mode 100644 index 0000000..5af4124 --- /dev/null +++ b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java @@ -0,0 +1,140 @@ +package cn.ac.iie.test.hbase; + +import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.hbase.HBaseUtils; +import com.zdjizhi.utils.StringUtil; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.log4j.Logger; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class HBaseTest { + private final static Logger logger = Logger.getLogger(HBaseTest.class); + private static Map subIdMap = new HashMap<>(16); + private static Connection connection; + private static Long time; + + + static { + // 管理Hbase的配置信息 + Configuration configuration = HBaseConfiguration.create(); + // 设置zookeeper节点 + configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); +// configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); +// configuration.setInt("zookeeper.session.timeout", 2000); + try { + connection = ConnectionFactory.createConnection(configuration); + time = System.currentTimeMillis(); + } catch (IOException e) { + logger.error("获取HBase连接失败" + e); + e.printStackTrace(); + } + } + +// @Before +// public void connHBase() { +// +// } + + @Test + public void change() { + Long begin = System.currentTimeMillis(); + System.gc(); + Long start = Runtime.getRuntime().freeMemory(); + System.out.println("开始内存"+start); + getAll(); + System.gc(); + Long end = Runtime.getRuntime().freeMemory(); + System.out.println("结束内存"+end); + System.out.println( "一个HashMap对象占内存: " + (end - start)); + System.out.println(System.currentTimeMillis() - begin); + } + + /** + * 获取变更内容 + * + * @param startTime 开始时间 + * @param endTime 结束时间 + */ + private static void timestampsFilter(Long startTime, Long endTime) { + Table table = null; + TableName tableName = TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME); + Admin admin = null; + try { + table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); + Scan scan2 = new Scan(); +// scan2.setCaching(500); + scan2.setTimeRange(startTime, endTime); + ResultScanner scanner = table.getScanner(scan2); + scanner.next().isEmpty(); + for (Result result : scanner) { + Cell[] cells = result.rawCells(); + for (Cell cell : cells) { + subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); + } + } + admin = connection.getAdmin(); + admin.flush(tableName); + logger.warn("当前集合长度" + subIdMap.keySet().size()); + logger.warn("更新后集合keys：" + subIdMap.keySet()); + time = endTime; + scanner.close(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (table != null) { + try { + table.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + /** + * 获取所有的 key value + */ + public static void getAll() { + try { + Table table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); + Scan scan2 = new Scan(); + ResultScanner scanner = table.getScanner(scan2); + for (Result result : scanner) { + Cell[] cells = result.rawCells(); + for (Cell cell : cells) { + subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); + } + } + logger.warn("获取全量后集合size：" + subIdMap.size()); + scanner.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + + /** + * 获取 account + * + * @param ip client_ip + * @return account + */ + public static String getAccount(String ip) { + if (StringUtil.isNotBlank(ip)) { + return subIdMap.get(ip); + } else { + return ""; + } + } +} From 784a426ee1e00bb8a83f7efe99b728ac6ab26dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Fri, 15 Nov 2019 11:43:06 +0800 Subject: [PATCH 10/27] =?UTF-8?q?=E6=B5=81=E7=9A=84=E8=BF=BD=E8=B8=AAID?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E7=B1=BB=E5=9E=8B=E6=94=B9=E5=8F=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java index 6b852cd..2883835 100644 --- a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java +++ b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java @@ -52,7 +52,7 @@ public class PublicSessionRecordLog { private String common_app_label; private String common_address_list; private String common_stream_error; - private String common_stream_trace_id; + private long common_stream_trace_id; public int getCommon_service() { return common_service; @@ -374,11 +374,11 @@ public class PublicSessionRecordLog { this.common_stream_error = common_stream_error; } - public String getCommon_stream_trace_id() { + public long getCommon_stream_trace_id() { return common_stream_trace_id; } - public void setCommon_stream_trace_id(String common_stream_trace_id) { + public void setCommon_stream_trace_id(long common_stream_trace_id) { this.common_stream_trace_id = common_stream_trace_id; } } From 2efd619314754b140d70eba64ddbec8786161858 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Fri, 15 Nov 2019 15:52:06 +0800 Subject: [PATCH 11/27] Deleted src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java, src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java, src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java, src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java files --- .../ac/iie/bean/CollectProtocolRecordLog.java | 682 ------------------ .../cn/ac/iie/bean/ProxySessionRecordLog.java | 202 ------ .../ac/iie/bean/RadiusSessionRecordLog.java | 87 --- .../cn/ac/iie/bean/SecurityPolicyLog.java | 682 ------------------ 4 files changed, 1653 deletions(-) delete mode 100644 src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java delete mode 100644 src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java delete mode 100644 src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java delete mode 100644 src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java diff --git a/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java b/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java deleted file mode 100644 index a4d82e8..0000000 --- a/src/main/java/cn/ac/iie/bean/CollectProtocolRecordLog.java +++ /dev/null @@ -1,682 +0,0 @@ -package cn.ac.iie.bean; - -/** - * 除radius之外，通联日志类 - * - * @author qidaijie - */ -public class CollectProtocolRecordLog extends PublicSessionRecordLog { - - -//TODO HTTP协议属性 21 - - private String http_url; - private String http_host; - private String http_domain; - private String http_request_line; - private String http_response_line; - private String http_request_header; - private String http_response_header; - private String http_request_body; - private String http_response_body; - private String http_request_body_key; - private String http_response_body_key; - private int http_proxy_flag; - private int http_sequence; - private String http_snapshot; - private String http_cookie; - private String http_referer; - private String http_user_agent; - private String http_content_length; - private String http_content_type; - private String http_set_cookie; - private String http_version; - -//TODO MAIL协议属性 9 - - private String mail_protocol_type; - private String mail_sender; - private String mail_receiver; - private String mail_subject; - private String mail_content; - private String mail_attachment_name; - private String mail_attachment_content; - private String mail_eml_file; - private String mail_snapshot; - -//TODO 3DNS协议属性 18 - - private int dns_message_id; - private int dns_qr; - private int dns_opcode; - private int dns_aa; - private int dns_tc; - private int dns_rd; - private int dns_ra; - private int dns_rcode; - private int dns_qdcount; - private int dns_ancount; - private int dns_nscount; - private int dns_arcount; - private String dns_qname; - private int dns_qtype; - private int dns_qclass; - private String dns_cname; - private int dns_sub; - private String dns_rr; - -//TODO SSL协议属性 13 - - - private int ssl_pinningst; - private int ssl_intercept_state; - private int ssl_server_side_latency; - private int ssl_client_side_latency; - private int ssl_cert_verify; - private int ssl_con_latency_ms; - private String ssl_version; - private String ssl_sni; - private String ssl_san; - private String ssl_cn; - private String ssl_server_side_version; - private String ssl_client_side_version; - private String ssl_error; - - //TODO FTP协议属性 2 - private String ftp_url; - private String ftp_content; - - - //TODO BGP协议属性 3 - private int bgp_type; - private String bgp_as_num; - private String bgp_route; - - //TODO VOIP协议属性 4 - private String voip_calling_account; - private String voip_called_account; - private String voip_calling_number; - private String voip_called_number; - - //TODO STREAMING_MEDIA协议属性 2 - private String streaming_media_url; - private String streaming_media_protocol; - - - public String getHttp_url() { - return http_url; - } - - public void setHttp_url(String http_url) { - this.http_url = http_url; - } - - public String getHttp_host() { - return http_host; - } - - public void setHttp_host(String http_host) { - this.http_host = http_host; - } - - public String getHttp_domain() { - return http_domain; - } - - public void setHttp_domain(String http_domain) { - this.http_domain = http_domain; - } - - public String getHttp_request_line() { - return http_request_line; - } - - public void setHttp_request_line(String http_request_line) { - this.http_request_line = http_request_line; - } - - public String getHttp_response_line() { - return http_response_line; - } - - public void setHttp_response_line(String http_response_line) { - this.http_response_line = http_response_line; - } - - public String getHttp_request_header() { - return http_request_header; - } - - public void setHttp_request_header(String http_request_header) { - this.http_request_header = http_request_header; - } - - public String getHttp_response_header() { - return http_response_header; - } - - public void setHttp_response_header(String http_response_header) { - this.http_response_header = http_response_header; - } - - public String getHttp_request_body() { - return http_request_body; - } - - public void setHttp_request_body(String http_request_body) { - this.http_request_body = http_request_body; - } - - public String getHttp_response_body() { - return http_response_body; - } - - public void setHttp_response_body(String http_response_body) { - this.http_response_body = http_response_body; - } - - public String getHttp_request_body_key() { - return http_request_body_key; - } - - public void setHttp_request_body_key(String http_request_body_key) { - this.http_request_body_key = http_request_body_key; - } - - public String getHttp_response_body_key() { - return http_response_body_key; - } - - public void setHttp_response_body_key(String http_response_body_key) { - this.http_response_body_key = http_response_body_key; - } - - public int getHttp_proxy_flag() { - return http_proxy_flag; - } - - public void setHttp_proxy_flag(int http_proxy_flag) { - this.http_proxy_flag = http_proxy_flag; - } - - public int getHttp_sequence() { - return http_sequence; - } - - public void setHttp_sequence(int http_sequence) { - this.http_sequence = http_sequence; - } - - public String getHttp_snapshot() { - return http_snapshot; - } - - public void setHttp_snapshot(String http_snapshot) { - this.http_snapshot = http_snapshot; - } - - public String getHttp_cookie() { - return http_cookie; - } - - public void setHttp_cookie(String http_cookie) { - this.http_cookie = http_cookie; - } - - public String getHttp_referer() { - return http_referer; - } - - public void setHttp_referer(String http_referer) { - this.http_referer = http_referer; - } - - public String getHttp_user_agent() { - return http_user_agent; - } - - public void setHttp_user_agent(String http_user_agent) { - this.http_user_agent = http_user_agent; - } - - public String getHttp_content_length() { - return http_content_length; - } - - public void setHttp_content_length(String http_content_length) { - this.http_content_length = http_content_length; - } - - public String getHttp_content_type() { - return http_content_type; - } - - public void setHttp_content_type(String http_content_type) { - this.http_content_type = http_content_type; - } - - public String getHttp_set_cookie() { - return http_set_cookie; - } - - public void setHttp_set_cookie(String http_set_cookie) { - this.http_set_cookie = http_set_cookie; - } - - public String getHttp_version() { - return http_version; - } - - public void setHttp_version(String http_version) { - this.http_version = http_version; - } - - public String getMail_protocol_type() { - return mail_protocol_type; - } - - public void setMail_protocol_type(String mail_protocol_type) { - this.mail_protocol_type = mail_protocol_type; - } - - public String getMail_sender() { - return mail_sender; - } - - public void setMail_sender(String mail_sender) { - this.mail_sender = mail_sender; - } - - public String getMail_receiver() { - return mail_receiver; - } - - public void setMail_receiver(String mail_receiver) { - this.mail_receiver = mail_receiver; - } - - public String getMail_subject() { - return mail_subject; - } - - public void setMail_subject(String mail_subject) { - this.mail_subject = mail_subject; - } - - public String getMail_content() { - return mail_content; - } - - public void setMail_content(String mail_content) { - this.mail_content = mail_content; - } - - public String getMail_attachment_name() { - return mail_attachment_name; - } - - public void setMail_attachment_name(String mail_attachment_name) { - this.mail_attachment_name = mail_attachment_name; - } - - public String getMail_attachment_content() { - return mail_attachment_content; - } - - public void setMail_attachment_content(String mail_attachment_content) { - this.mail_attachment_content = mail_attachment_content; - } - - public String getMail_eml_file() { - return mail_eml_file; - } - - public void setMail_eml_file(String mail_eml_file) { - this.mail_eml_file = mail_eml_file; - } - - public String getMail_snapshot() { - return mail_snapshot; - } - - public void setMail_snapshot(String mail_snapshot) { - this.mail_snapshot = mail_snapshot; - } - - public int getDns_message_id() { - return dns_message_id; - } - - public void setDns_message_id(int dns_message_id) { - this.dns_message_id = dns_message_id; - } - - public int getDns_qr() { - return dns_qr; - } - - public void setDns_qr(int dns_qr) { - this.dns_qr = dns_qr; - } - - public int getDns_opcode() { - return dns_opcode; - } - - public void setDns_opcode(int dns_opcode) { - this.dns_opcode = dns_opcode; - } - - public int getDns_aa() { - return dns_aa; - } - - public void setDns_aa(int dns_aa) { - this.dns_aa = dns_aa; - } - - public int getDns_tc() { - return dns_tc; - } - - public void setDns_tc(int dns_tc) { - this.dns_tc = dns_tc; - } - - public int getDns_rd() { - return dns_rd; - } - - public void setDns_rd(int dns_rd) { - this.dns_rd = dns_rd; - } - - public int getDns_ra() { - return dns_ra; - } - - public void setDns_ra(int dns_ra) { - this.dns_ra = dns_ra; - } - - public int getDns_rcode() { - return dns_rcode; - } - - public void setDns_rcode(int dns_rcode) { - this.dns_rcode = dns_rcode; - } - - public int getDns_qdcount() { - return dns_qdcount; - } - - public void setDns_qdcount(int dns_qdcount) { - this.dns_qdcount = dns_qdcount; - } - - public int getDns_ancount() { - return dns_ancount; - } - - public void setDns_ancount(int dns_ancount) { - this.dns_ancount = dns_ancount; - } - - public int getDns_nscount() { - return dns_nscount; - } - - public void setDns_nscount(int dns_nscount) { - this.dns_nscount = dns_nscount; - } - - public int getDns_arcount() { - return dns_arcount; - } - - public void setDns_arcount(int dns_arcount) { - this.dns_arcount = dns_arcount; - } - - public String getDns_qname() { - return dns_qname; - } - - public void setDns_qname(String dns_qname) { - this.dns_qname = dns_qname; - } - - public int getDns_qtype() { - return dns_qtype; - } - - public void setDns_qtype(int dns_qtype) { - this.dns_qtype = dns_qtype; - } - - public int getDns_qclass() { - return dns_qclass; - } - - public void setDns_qclass(int dns_qclass) { - this.dns_qclass = dns_qclass; - } - - public String getDns_cname() { - return dns_cname; - } - - public void setDns_cname(String dns_cname) { - this.dns_cname = dns_cname; - } - - public int getDns_sub() { - return dns_sub; - } - - public void setDns_sub(int dns_sub) { - this.dns_sub = dns_sub; - } - - public String getDns_rr() { - return dns_rr; - } - - public void setDns_rr(String dns_rr) { - this.dns_rr = dns_rr; - } - - public int getSsl_pinningst() { - return ssl_pinningst; - } - - public void setSsl_pinningst(int ssl_pinningst) { - this.ssl_pinningst = ssl_pinningst; - } - - public int getSsl_intercept_state() { - return ssl_intercept_state; - } - - public void setSsl_intercept_state(int ssl_intercept_state) { - this.ssl_intercept_state = ssl_intercept_state; - } - - public int getSsl_server_side_latency() { - return ssl_server_side_latency; - } - - public void setSsl_server_side_latency(int ssl_server_side_latency) { - this.ssl_server_side_latency = ssl_server_side_latency; - } - - public int getSsl_client_side_latency() { - return ssl_client_side_latency; - } - - public void setSsl_client_side_latency(int ssl_client_side_latency) { - this.ssl_client_side_latency = ssl_client_side_latency; - } - - public int getSsl_cert_verify() { - return ssl_cert_verify; - } - - public void setSsl_cert_verify(int ssl_cert_verify) { - this.ssl_cert_verify = ssl_cert_verify; - } - - public int getSsl_con_latency_ms() { - return ssl_con_latency_ms; - } - - public void setSsl_con_latency_ms(int ssl_con_latency_ms) { - this.ssl_con_latency_ms = ssl_con_latency_ms; - } - - public String getSsl_version() { - return ssl_version; - } - - public void setSsl_version(String ssl_version) { - this.ssl_version = ssl_version; - } - - public String getSsl_sni() { - return ssl_sni; - } - - public void setSsl_sni(String ssl_sni) { - this.ssl_sni = ssl_sni; - } - - public String getSsl_san() { - return ssl_san; - } - - public void setSsl_san(String ssl_san) { - this.ssl_san = ssl_san; - } - - public String getSsl_cn() { - return ssl_cn; - } - - public void setSsl_cn(String ssl_cn) { - this.ssl_cn = ssl_cn; - } - - public String getSsl_server_side_version() { - return ssl_server_side_version; - } - - public void setSsl_server_side_version(String ssl_server_side_version) { - this.ssl_server_side_version = ssl_server_side_version; - } - - public String getSsl_client_side_version() { - return ssl_client_side_version; - } - - public void setSsl_client_side_version(String ssl_client_side_version) { - this.ssl_client_side_version = ssl_client_side_version; - } - - public String getSsl_error() { - return ssl_error; - } - - public void setSsl_error(String ssl_error) { - this.ssl_error = ssl_error; - } - - public String getFtp_url() { - return ftp_url; - } - - public void setFtp_url(String ftp_url) { - this.ftp_url = ftp_url; - } - - public String getFtp_content() { - return ftp_content; - } - - public void setFtp_content(String ftp_content) { - this.ftp_content = ftp_content; - } - - public int getBgp_type() { - return bgp_type; - } - - public void setBgp_type(int bgp_type) { - this.bgp_type = bgp_type; - } - - public String getBgp_as_num() { - return bgp_as_num; - } - - public void setBgp_as_num(String bgp_as_num) { - this.bgp_as_num = bgp_as_num; - } - - public String getBgp_route() { - return bgp_route; - } - - public void setBgp_route(String bgp_route) { - this.bgp_route = bgp_route; - } - - public String getVoip_calling_account() { - return voip_calling_account; - } - - public void setVoip_calling_account(String voip_calling_account) { - this.voip_calling_account = voip_calling_account; - } - - public String getVoip_called_account() { - return voip_called_account; - } - - public void setVoip_called_account(String voip_called_account) { - this.voip_called_account = voip_called_account; - } - - public String getVoip_calling_number() { - return voip_calling_number; - } - - public void setVoip_calling_number(String voip_calling_number) { - this.voip_calling_number = voip_calling_number; - } - - public String getVoip_called_number() { - return voip_called_number; - } - - public void setVoip_called_number(String voip_called_number) { - this.voip_called_number = voip_called_number; - } - - - public String getStreaming_media_url() { - return streaming_media_url; - } - - public void setStreaming_media_url(String streaming_media_url) { - this.streaming_media_url = streaming_media_url; - } - - public String getStreaming_media_protocol() { - return streaming_media_protocol; - } - - public void setStreaming_media_protocol(String streaming_media_protocol) { - this.streaming_media_protocol = streaming_media_protocol; - } -} diff --git a/src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java b/src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java deleted file mode 100644 index 441205d..0000000 --- a/src/main/java/cn/ac/iie/bean/ProxySessionRecordLog.java +++ /dev/null @@ -1,202 +0,0 @@ -package cn.ac.iie.bean; - - - -/** - * @author qidaijie - */ -public class ProxySessionRecordLog extends PublicSessionRecordLog{ - - -//TODO HTTP协议属性 21 - - private String http_url; - private String http_host; - private String http_domain; - private String http_request_line; - private String http_response_line; - private String http_request_header; - private String http_response_header; - private String http_request_body; - private String http_response_body; - private String http_request_body_key; - private String http_response_body_key; - private int http_proxy_flag; - private int http_sequence; - private String http_snapshot; - private String http_cookie; - private String http_referer; - private String http_user_agent; - private String http_content_length; - private String http_content_type; - private String http_set_cookie; - private String http_version; - - public String getHttp_url() { - return http_url; - } - - public void setHttp_url(String http_url) { - this.http_url = http_url; - } - - public String getHttp_host() { - return http_host; - } - - public void setHttp_host(String http_host) { - this.http_host = http_host; - } - - public String getHttp_domain() { - return http_domain; - } - - public void setHttp_domain(String http_domain) { - this.http_domain = http_domain; - } - - public String getHttp_request_line() { - return http_request_line; - } - - public void setHttp_request_line(String http_request_line) { - this.http_request_line = http_request_line; - } - - public String getHttp_response_line() { - return http_response_line; - } - - public void setHttp_response_line(String http_response_line) { - this.http_response_line = http_response_line; - } - - public String getHttp_request_header() { - return http_request_header; - } - - public void setHttp_request_header(String http_request_header) { - this.http_request_header = http_request_header; - } - - public String getHttp_response_header() { - return http_response_header; - } - - public void setHttp_response_header(String http_response_header) { - this.http_response_header = http_response_header; - } - - public String getHttp_request_body() { - return http_request_body; - } - - public void setHttp_request_body(String http_request_body) { - this.http_request_body = http_request_body; - } - - public String getHttp_response_body() { - return http_response_body; - } - - public void setHttp_response_body(String http_response_body) { - this.http_response_body = http_response_body; - } - - public String getHttp_request_body_key() { - return http_request_body_key; - } - - public void setHttp_request_body_key(String http_request_body_key) { - this.http_request_body_key = http_request_body_key; - } - - public String getHttp_response_body_key() { - return http_response_body_key; - } - - public void setHttp_response_body_key(String http_response_body_key) { - this.http_response_body_key = http_response_body_key; - } - - public int getHttp_proxy_flag() { - return http_proxy_flag; - } - - public void setHttp_proxy_flag(int http_proxy_flag) { - this.http_proxy_flag = http_proxy_flag; - } - - public int getHttp_sequence() { - return http_sequence; - } - - public void setHttp_sequence(int http_sequence) { - this.http_sequence = http_sequence; - } - - public String getHttp_snapshot() { - return http_snapshot; - } - - public void setHttp_snapshot(String http_snapshot) { - this.http_snapshot = http_snapshot; - } - - public String getHttp_cookie() { - return http_cookie; - } - - public void setHttp_cookie(String http_cookie) { - this.http_cookie = http_cookie; - } - - public String getHttp_referer() { - return http_referer; - } - - public void setHttp_referer(String http_referer) { - this.http_referer = http_referer; - } - - public String getHttp_user_agent() { - return http_user_agent; - } - - public void setHttp_user_agent(String http_user_agent) { - this.http_user_agent = http_user_agent; - } - - public String getHttp_content_length() { - return http_content_length; - } - - public void setHttp_content_length(String http_content_length) { - this.http_content_length = http_content_length; - } - - public String getHttp_content_type() { - return http_content_type; - } - - public void setHttp_content_type(String http_content_type) { - this.http_content_type = http_content_type; - } - - public String getHttp_set_cookie() { - return http_set_cookie; - } - - public void setHttp_set_cookie(String http_set_cookie) { - this.http_set_cookie = http_set_cookie; - } - - public String getHttp_version() { - return http_version; - } - - public void setHttp_version(String http_version) { - this.http_version = http_version; - } -} diff --git a/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java deleted file mode 100644 index d4f7679..0000000 --- a/src/main/java/cn/ac/iie/bean/RadiusSessionRecordLog.java +++ /dev/null @@ -1,87 +0,0 @@ -package cn.ac.iie.bean; - - -/** - * Radius 日志 - * - * @author qidaijie - */ -public class RadiusSessionRecordLog extends PublicSessionRecordLog { - - -//TODO RADIUS协议属性 8 - - private int radius_session_timeout; - private int radius_idle_timeout; - private int radius_acct_status_type; - private int radius_acct_terminate_cause; - private int radius_packet_type; - private String radius_nas_ip; - private String radius_framed_ip; - private String radius_account; - - - public int getRadius_session_timeout() { - return radius_session_timeout; - } - - public void setRadius_session_timeout(int radius_session_timeout) { - this.radius_session_timeout = radius_session_timeout; - } - - public int getRadius_idle_timeout() { - return radius_idle_timeout; - } - - public void setRadius_idle_timeout(int radius_idle_timeout) { - this.radius_idle_timeout = radius_idle_timeout; - } - - public int getRadius_acct_status_type() { - return radius_acct_status_type; - } - - public void setRadius_acct_status_type(int radius_acct_status_type) { - this.radius_acct_status_type = radius_acct_status_type; - } - - public int getRadius_acct_terminate_cause() { - return radius_acct_terminate_cause; - } - - public void setRadius_acct_terminate_cause(int radius_acct_terminate_cause) { - this.radius_acct_terminate_cause = radius_acct_terminate_cause; - } - - public int getRadius_packet_type() { - return radius_packet_type; - } - - public void setRadius_packet_type(int radius_packet_type) { - this.radius_packet_type = radius_packet_type; - } - - public String getRadius_nas_ip() { - return radius_nas_ip; - } - - public void setRadius_nas_ip(String radius_nas_ip) { - this.radius_nas_ip = radius_nas_ip; - } - - public String getRadius_framed_ip() { - return radius_framed_ip; - } - - public void setRadius_framed_ip(String radius_framed_ip) { - this.radius_framed_ip = radius_framed_ip; - } - - public String getRadius_account() { - return radius_account; - } - - public void setRadius_account(String radius_account) { - this.radius_account = radius_account; - } -} diff --git a/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java b/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java deleted file mode 100644 index 799a8bd..0000000 --- a/src/main/java/cn/ac/iie/bean/SecurityPolicyLog.java +++ /dev/null @@ -1,682 +0,0 @@ -package cn.ac.iie.bean; - -/** - * 安全策略 - * - * @author qidaijie - */ -public class SecurityPolicyLog extends PublicSessionRecordLog { - - -//TODO HTTP协议属性 21 - - private String http_url; - private String http_host; - private String http_domain; - private String http_request_line; - private String http_response_line; - private String http_request_header; - private String http_response_header; - private String http_request_body; - private String http_response_body; - private String http_request_body_key; - private String http_response_body_key; - private int http_proxy_flag; - private int http_sequence; - private String http_snapshot; - private String http_cookie; - private String http_referer; - private String http_user_agent; - private String http_content_length; - private String http_content_type; - private String http_set_cookie; - private String http_version; - -//TODO MAIL协议属性 9 - - private String mail_protocol_type; - private String mail_sender; - private String mail_receiver; - private String mail_subject; - private String mail_content; - private String mail_attachment_name; - private String mail_attachment_content; - private String mail_eml_file; - private String mail_snapshot; - -//TODO 3DNS协议属性 18 - - private int dns_message_id; - private int dns_qr; - private int dns_opcode; - private int dns_aa; - private int dns_tc; - private int dns_rd; - private int dns_ra; - private int dns_rcode; - private int dns_qdcount; - private int dns_ancount; - private int dns_nscount; - private int dns_arcount; - private String dns_qname; - private int dns_qtype; - private int dns_qclass; - private String dns_cname; - private int dns_sub; - private String dns_rr; - -//TODO SSL协议属性 13 - - - private int ssl_pinningst; - private int ssl_intercept_state; - private int ssl_server_side_latency; - private int ssl_client_side_latency; - private int ssl_cert_verify; - private int ssl_con_latency_ms; - private String ssl_version; - private String ssl_sni; - private String ssl_san; - private String ssl_cn; - private String ssl_server_side_version; - private String ssl_client_side_version; - private String ssl_error; - - //TODO FTP协议属性 2 - private String ftp_url; - private String ftp_content; - - - //TODO BGP协议属性 3 - private int bgp_type; - private String bgp_as_num; - private String bgp_route; - - //TODO VOIP协议属性 4 - private String voip_calling_account; - private String voip_called_account; - private String voip_calling_number; - private String voip_called_number; - - //TODO STREAMING_MEDIA协议属性 2 - private String streaming_media_url; - private String streaming_media_protocol; - - - public String getHttp_url() { - return http_url; - } - - public void setHttp_url(String http_url) { - this.http_url = http_url; - } - - public String getHttp_host() { - return http_host; - } - - public void setHttp_host(String http_host) { - this.http_host = http_host; - } - - public String getHttp_domain() { - return http_domain; - } - - public void setHttp_domain(String http_domain) { - this.http_domain = http_domain; - } - - public String getHttp_request_line() { - return http_request_line; - } - - public void setHttp_request_line(String http_request_line) { - this.http_request_line = http_request_line; - } - - public String getHttp_response_line() { - return http_response_line; - } - - public void setHttp_response_line(String http_response_line) { - this.http_response_line = http_response_line; - } - - public String getHttp_request_header() { - return http_request_header; - } - - public void setHttp_request_header(String http_request_header) { - this.http_request_header = http_request_header; - } - - public String getHttp_response_header() { - return http_response_header; - } - - public void setHttp_response_header(String http_response_header) { - this.http_response_header = http_response_header; - } - - public String getHttp_request_body() { - return http_request_body; - } - - public void setHttp_request_body(String http_request_body) { - this.http_request_body = http_request_body; - } - - public String getHttp_response_body() { - return http_response_body; - } - - public void setHttp_response_body(String http_response_body) { - this.http_response_body = http_response_body; - } - - public String getHttp_request_body_key() { - return http_request_body_key; - } - - public void setHttp_request_body_key(String http_request_body_key) { - this.http_request_body_key = http_request_body_key; - } - - public String getHttp_response_body_key() { - return http_response_body_key; - } - - public void setHttp_response_body_key(String http_response_body_key) { - this.http_response_body_key = http_response_body_key; - } - - public int getHttp_proxy_flag() { - return http_proxy_flag; - } - - public void setHttp_proxy_flag(int http_proxy_flag) { - this.http_proxy_flag = http_proxy_flag; - } - - public int getHttp_sequence() { - return http_sequence; - } - - public void setHttp_sequence(int http_sequence) { - this.http_sequence = http_sequence; - } - - public String getHttp_snapshot() { - return http_snapshot; - } - - public void setHttp_snapshot(String http_snapshot) { - this.http_snapshot = http_snapshot; - } - - public String getHttp_cookie() { - return http_cookie; - } - - public void setHttp_cookie(String http_cookie) { - this.http_cookie = http_cookie; - } - - public String getHttp_referer() { - return http_referer; - } - - public void setHttp_referer(String http_referer) { - this.http_referer = http_referer; - } - - public String getHttp_user_agent() { - return http_user_agent; - } - - public void setHttp_user_agent(String http_user_agent) { - this.http_user_agent = http_user_agent; - } - - public String getHttp_content_length() { - return http_content_length; - } - - public void setHttp_content_length(String http_content_length) { - this.http_content_length = http_content_length; - } - - public String getHttp_content_type() { - return http_content_type; - } - - public void setHttp_content_type(String http_content_type) { - this.http_content_type = http_content_type; - } - - public String getHttp_set_cookie() { - return http_set_cookie; - } - - public void setHttp_set_cookie(String http_set_cookie) { - this.http_set_cookie = http_set_cookie; - } - - public String getHttp_version() { - return http_version; - } - - public void setHttp_version(String http_version) { - this.http_version = http_version; - } - - public String getMail_protocol_type() { - return mail_protocol_type; - } - - public void setMail_protocol_type(String mail_protocol_type) { - this.mail_protocol_type = mail_protocol_type; - } - - public String getMail_sender() { - return mail_sender; - } - - public void setMail_sender(String mail_sender) { - this.mail_sender = mail_sender; - } - - public String getMail_receiver() { - return mail_receiver; - } - - public void setMail_receiver(String mail_receiver) { - this.mail_receiver = mail_receiver; - } - - public String getMail_subject() { - return mail_subject; - } - - public void setMail_subject(String mail_subject) { - this.mail_subject = mail_subject; - } - - public String getMail_content() { - return mail_content; - } - - public void setMail_content(String mail_content) { - this.mail_content = mail_content; - } - - public String getMail_attachment_name() { - return mail_attachment_name; - } - - public void setMail_attachment_name(String mail_attachment_name) { - this.mail_attachment_name = mail_attachment_name; - } - - public String getMail_attachment_content() { - return mail_attachment_content; - } - - public void setMail_attachment_content(String mail_attachment_content) { - this.mail_attachment_content = mail_attachment_content; - } - - public String getMail_eml_file() { - return mail_eml_file; - } - - public void setMail_eml_file(String mail_eml_file) { - this.mail_eml_file = mail_eml_file; - } - - public String getMail_snapshot() { - return mail_snapshot; - } - - public void setMail_snapshot(String mail_snapshot) { - this.mail_snapshot = mail_snapshot; - } - - public int getDns_message_id() { - return dns_message_id; - } - - public void setDns_message_id(int dns_message_id) { - this.dns_message_id = dns_message_id; - } - - public int getDns_qr() { - return dns_qr; - } - - public void setDns_qr(int dns_qr) { - this.dns_qr = dns_qr; - } - - public int getDns_opcode() { - return dns_opcode; - } - - public void setDns_opcode(int dns_opcode) { - this.dns_opcode = dns_opcode; - } - - public int getDns_aa() { - return dns_aa; - } - - public void setDns_aa(int dns_aa) { - this.dns_aa = dns_aa; - } - - public int getDns_tc() { - return dns_tc; - } - - public void setDns_tc(int dns_tc) { - this.dns_tc = dns_tc; - } - - public int getDns_rd() { - return dns_rd; - } - - public void setDns_rd(int dns_rd) { - this.dns_rd = dns_rd; - } - - public int getDns_ra() { - return dns_ra; - } - - public void setDns_ra(int dns_ra) { - this.dns_ra = dns_ra; - } - - public int getDns_rcode() { - return dns_rcode; - } - - public void setDns_rcode(int dns_rcode) { - this.dns_rcode = dns_rcode; - } - - public int getDns_qdcount() { - return dns_qdcount; - } - - public void setDns_qdcount(int dns_qdcount) { - this.dns_qdcount = dns_qdcount; - } - - public int getDns_ancount() { - return dns_ancount; - } - - public void setDns_ancount(int dns_ancount) { - this.dns_ancount = dns_ancount; - } - - public int getDns_nscount() { - return dns_nscount; - } - - public void setDns_nscount(int dns_nscount) { - this.dns_nscount = dns_nscount; - } - - public int getDns_arcount() { - return dns_arcount; - } - - public void setDns_arcount(int dns_arcount) { - this.dns_arcount = dns_arcount; - } - - public String getDns_qname() { - return dns_qname; - } - - public void setDns_qname(String dns_qname) { - this.dns_qname = dns_qname; - } - - public int getDns_qtype() { - return dns_qtype; - } - - public void setDns_qtype(int dns_qtype) { - this.dns_qtype = dns_qtype; - } - - public int getDns_qclass() { - return dns_qclass; - } - - public void setDns_qclass(int dns_qclass) { - this.dns_qclass = dns_qclass; - } - - public String getDns_cname() { - return dns_cname; - } - - public void setDns_cname(String dns_cname) { - this.dns_cname = dns_cname; - } - - public int getDns_sub() { - return dns_sub; - } - - public void setDns_sub(int dns_sub) { - this.dns_sub = dns_sub; - } - - public String getDns_rr() { - return dns_rr; - } - - public void setDns_rr(String dns_rr) { - this.dns_rr = dns_rr; - } - - public int getSsl_pinningst() { - return ssl_pinningst; - } - - public void setSsl_pinningst(int ssl_pinningst) { - this.ssl_pinningst = ssl_pinningst; - } - - public int getSsl_intercept_state() { - return ssl_intercept_state; - } - - public void setSsl_intercept_state(int ssl_intercept_state) { - this.ssl_intercept_state = ssl_intercept_state; - } - - public int getSsl_server_side_latency() { - return ssl_server_side_latency; - } - - public void setSsl_server_side_latency(int ssl_server_side_latency) { - this.ssl_server_side_latency = ssl_server_side_latency; - } - - public int getSsl_client_side_latency() { - return ssl_client_side_latency; - } - - public void setSsl_client_side_latency(int ssl_client_side_latency) { - this.ssl_client_side_latency = ssl_client_side_latency; - } - - public int getSsl_cert_verify() { - return ssl_cert_verify; - } - - public void setSsl_cert_verify(int ssl_cert_verify) { - this.ssl_cert_verify = ssl_cert_verify; - } - - public int getSsl_con_latency_ms() { - return ssl_con_latency_ms; - } - - public void setSsl_con_latency_ms(int ssl_con_latency_ms) { - this.ssl_con_latency_ms = ssl_con_latency_ms; - } - - public String getSsl_version() { - return ssl_version; - } - - public void setSsl_version(String ssl_version) { - this.ssl_version = ssl_version; - } - - public String getSsl_sni() { - return ssl_sni; - } - - public void setSsl_sni(String ssl_sni) { - this.ssl_sni = ssl_sni; - } - - public String getSsl_san() { - return ssl_san; - } - - public void setSsl_san(String ssl_san) { - this.ssl_san = ssl_san; - } - - public String getSsl_cn() { - return ssl_cn; - } - - public void setSsl_cn(String ssl_cn) { - this.ssl_cn = ssl_cn; - } - - public String getSsl_server_side_version() { - return ssl_server_side_version; - } - - public void setSsl_server_side_version(String ssl_server_side_version) { - this.ssl_server_side_version = ssl_server_side_version; - } - - public String getSsl_client_side_version() { - return ssl_client_side_version; - } - - public void setSsl_client_side_version(String ssl_client_side_version) { - this.ssl_client_side_version = ssl_client_side_version; - } - - public String getSsl_error() { - return ssl_error; - } - - public void setSsl_error(String ssl_error) { - this.ssl_error = ssl_error; - } - - public String getFtp_url() { - return ftp_url; - } - - public void setFtp_url(String ftp_url) { - this.ftp_url = ftp_url; - } - - public String getFtp_content() { - return ftp_content; - } - - public void setFtp_content(String ftp_content) { - this.ftp_content = ftp_content; - } - - public int getBgp_type() { - return bgp_type; - } - - public void setBgp_type(int bgp_type) { - this.bgp_type = bgp_type; - } - - public String getBgp_as_num() { - return bgp_as_num; - } - - public void setBgp_as_num(String bgp_as_num) { - this.bgp_as_num = bgp_as_num; - } - - public String getBgp_route() { - return bgp_route; - } - - public void setBgp_route(String bgp_route) { - this.bgp_route = bgp_route; - } - - public String getVoip_calling_account() { - return voip_calling_account; - } - - public void setVoip_calling_account(String voip_calling_account) { - this.voip_calling_account = voip_calling_account; - } - - public String getVoip_called_account() { - return voip_called_account; - } - - public void setVoip_called_account(String voip_called_account) { - this.voip_called_account = voip_called_account; - } - - public String getVoip_calling_number() { - return voip_calling_number; - } - - public void setVoip_calling_number(String voip_calling_number) { - this.voip_calling_number = voip_calling_number; - } - - public String getVoip_called_number() { - return voip_called_number; - } - - public void setVoip_called_number(String voip_called_number) { - this.voip_called_number = voip_called_number; - } - - - public String getStreaming_media_url() { - return streaming_media_url; - } - - public void setStreaming_media_url(String streaming_media_url) { - this.streaming_media_url = streaming_media_url; - } - - public String getStreaming_media_protocol() { - return streaming_media_protocol; - } - - public void setStreaming_media_protocol(String streaming_media_protocol) { - this.streaming_media_protocol = streaming_media_protocol; - } -} From 57fbb78d11cb648d4578ad3a2d0eb3197d16c54d Mon Sep 17 00:00:00 2001 From: qidaijie Date: Thu, 28 Nov 2019 11:53:35 +0800 Subject: [PATCH 12/27] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20domain=20=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E6=AD=A3=E5=88=99=E5=8C=B9=E9=85=8D=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- properties/core-site.xml | 71 ----------- properties/hbase-site.xml | 77 ------------ properties/hdfs-site.xml | 116 ------------------ properties/redis_config.properties | 19 --- .../cn/ac/iie/common/FlowWriteConfig.java | 2 +- .../ac/iie/utils/general/TransFormUtils.java | 44 ++++--- .../{HbaseUtils.java => HBaseUtils.java} | 0 .../java/cn/ac/iie/test/hbase/HBaseTest.java | 20 +-- 8 files changed, 40 insertions(+), 309 deletions(-) delete mode 100644 properties/core-site.xml delete mode 100644 properties/hbase-site.xml delete mode 100644 properties/hdfs-site.xml delete mode 100644 properties/redis_config.properties rename src/main/java/cn/ac/iie/utils/hbase/{HbaseUtils.java => HBaseUtils.java} (100%) diff --git a/properties/core-site.xml b/properties/core-site.xml deleted file mode 100644 index 93dfb1d..0000000 --- a/properties/core-site.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - - - - - - fs.defaultFS - hdfs://ns1 - - - hadoop.tmp.dir - file:/opt/hadoop/tmp - - - io.file.buffer.size - 131702 - - - hadoop.proxyuser.root.hosts - * - - - hadoop.proxyuser.root.groups - * - - - hadoop.logfile.size - 10000000 - The max size of each log file - - - - hadoop.logfile.count - 1 - The max number of log files - - - ha.zookeeper.quorum - master:2181,slave1:2181,slave2:2181 - - -    -     fs.hdfs.impl   -     org.apache.hadoop.hdfs.DistributedFileSystem   -     The FileSystem for hdfs: uris.   - - - -io.compression.codecs -com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec - - -io.compression.codec.lzo.class -com.hadoop.compression.lzo.LzoCodec - - - diff --git a/properties/hbase-site.xml b/properties/hbase-site.xml deleted file mode 100644 index 54554e4..0000000 --- a/properties/hbase-site.xml +++ /dev/null @@ -1,77 +0,0 @@ - - - - - - hbase.rootdir - hdfs://ns1/hbase-1.4.9 - - - hbase.cluster.distributed - true - - - hbase.zookeeper.quorum - 192.168.40.119,192.168.40.122,192.168.40.123 - - -hbase.master.info.port -60010 - - - - phoenix.schema.isNamespaceMappingEnabled - true - - - phoenix.schema.mapSystemTablesToNamespace - true - - - hbase.client.keyvalue.maxsize - 99428800 - - - hbase.server.keyvalue.maxsize - 99428800 - - - hbase.regionserver.wal.codec - org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec - - - phoenix.query.timeoutMs - 1800000 - - - hbase.rpc.timeout - 1200000 - - - hbase.client.scanner.caching - 1000 - - - hbase.client.scanner.timeout.period - 1200000 - - diff --git a/properties/hdfs-site.xml b/properties/hdfs-site.xml deleted file mode 100644 index 1e148b7..0000000 --- a/properties/hdfs-site.xml +++ /dev/null @@ -1,116 +0,0 @@ - - - - - - - - - dfs.namenode.name.dir - file:/home/ceiec/hadoop/dfs/name - - - dfs.datanode.data.dir - file:/home/ceiec/hadoop/dfs/data - - - dfs.replication - 2 - - - dfs.namenode.secondary.http-address - 192.168.40.119:9001 - - - dfs.webhdfs.enabled - true - - - dfs.permissions - false - - - dfs.permissions.enabled - false - - - dfs.nameservices - ns1 - - - dfs.blocksize - 134217728 - - - dfs.ha.namenodes.ns1 - nn1,nn2 - - - - dfs.namenode.rpc-address.ns1.nn1 - 192.168.40.119:8020 - - - - dfs.namenode.http-address.ns1.nn1 - 192.168.40.119:50070 - - - - dfs.namenode.rpc-address.ns1.nn2 - 192.168.40.122:8020 - - - - dfs.namenode.http-address.ns1.nn2 - 192.168.40.122:50070 - - - - dfs.namenode.shared.edits.dir - qjournal://192.168.40.119:8485;192.168.40.122:8485;192.168.40.123:8485/ns1 - - - - dfs.journalnode.edits.dir - /home/ceiec/hadoop/journal - - - - dfs.client.failover.proxy.provider.ns1 - org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider - - - - dfs.ha.fencing.methods - sshfence - - - - dfs.ha.fencing.ssh.private-key-files - /root/.ssh/id_rsa - - - - dfs.ha.fencing.ssh.connect-timeout - 30000 - - - - dfs.ha.automatic-failover.enabled - true - - - diff --git a/properties/redis_config.properties b/properties/redis_config.properties deleted file mode 100644 index 3c529de..0000000 --- a/properties/redis_config.properties +++ /dev/null @@ -1,19 +0,0 @@ -#*****************jedis连接参数设置********************* -#redis服务器ip -redis.ip=192.168.40.153 -#redis服务器端口号 -redis.port=19000 -#与服务器建立连接的超时时间 -redis.timeout=3000 -#************************jedis池参数设置******************* -#jedis的最大活跃连接数 -redis.pool.maxActive=200 -#jedis最大空闲连接数 -redis.pool.maxIdle=5 -#jedis池没有连接对象返回时，等待可用连接的最大时间，单位毫秒，默认值为-1，表示永不超时。 -#如果超过等待时间，则直接抛出JedisConnectionException -redis.pool.maxWait=-1 -#从池中获取连接的时候，是否进行有效检查 -redis.pool.testOnBorrow=true -#归还连接的时候，是否进行有效检查 -redis.pool.testOnReturn=true diff --git a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java index cb5fc4c..f91a57a 100644 --- a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java +++ b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java @@ -10,7 +10,7 @@ public class FlowWriteConfig { public static final String LOG_STRING_SPLITTER = "\t"; public static final String SQL_STRING_SPLITTER = "#"; - public static final String SEGMENTATION = ","; + public static final String DOMAIN_SPLITTER = "."; /** * System diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index 0340776..79e067e 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -12,6 +12,9 @@ import com.zdjizhi.utils.IpLookup; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -25,7 +28,10 @@ import java.util.regex.Pattern; public class TransFormUtils { private static Logger logger = Logger.getLogger(TransFormUtils.class); - private static Pattern WEB_PATTERN = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.cc|\\.me|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)"); + private final static Set PUBLIC_SUFFIX_SET = new HashSet( + Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se" + .split("\\|"))); + private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})"); private static IpLookup ipLookup = new IpLookup.Builder(false) .loadDataFileV4(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") .loadDataFileV6(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") @@ -156,10 +162,10 @@ public class TransFormUtils { * @return 顶级域名 */ private static String getTopDomain(String sni, String host) { - if (StringUtil.isNotBlank(sni)) { - return getDomain(sni); - } else if (StringUtil.isNotBlank(host)) { - return getDomain(host); + if (StringUtil.isNotBlank(host)) { + return getDomainName(host); + } else if (StringUtil.isNotBlank(sni)) { + return getDomainName(sni); } else { return ""; } @@ -169,19 +175,27 @@ public class TransFormUtils { /** * 根据url截取顶级域名 * - * @param url 网站url + * @param host 网站url * @return 顶级域名 */ - private static String getDomain(String url) { - try { - Matcher matcher = WEB_PATTERN.matcher(url); - if (matcher.find()) { - return matcher.group(); - } - } catch (Exception e) { - e.printStackTrace(); + private static String getDomainName(String host) { + if (host.endsWith(FlowWriteConfig.DOMAIN_SPLITTER)) { + host = host.substring(0, host.length() - 1); } - return ""; + if (IP_PATTERN.matcher(host).matches()) { + return host; + } + int index = 0; + String candidate = host; + for (; index >= 0; ) { + index = candidate.indexOf(FlowWriteConfig.DOMAIN_SPLITTER); + String subCandidate = candidate.substring(index + 1); + if (PUBLIC_SUFFIX_SET.contains(subCandidate)) { + return candidate; + } + candidate = subCandidate; + } + return candidate; } diff --git a/src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java similarity index 100% rename from src/main/java/cn/ac/iie/utils/hbase/HbaseUtils.java rename to src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java diff --git a/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java index 5af4124..9e94387 100644 --- a/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java +++ b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java @@ -48,16 +48,16 @@ public class HBaseTest { @Test public void change() { - Long begin = System.currentTimeMillis(); - System.gc(); - Long start = Runtime.getRuntime().freeMemory(); - System.out.println("开始内存"+start); - getAll(); - System.gc(); - Long end = Runtime.getRuntime().freeMemory(); - System.out.println("结束内存"+end); - System.out.println( "一个HashMap对象占内存: " + (end - start)); - System.out.println(System.currentTimeMillis() - begin); +// Long begin = System.currentTimeMillis(); +// System.gc(); +// Long start = Runtime.getRuntime().freeMemory(); +// System.out.println("开始内存"+start); +// getAll(); +// System.gc(); +// Long end = Runtime.getRuntime().freeMemory(); +// System.out.println("结束内存"+end); +// System.out.println( "一个HashMap对象占内存: " + (end - start)); +// System.out.println(System.currentTimeMillis() - begin); } /** From 7aac9e03d58446273996a64df6e4cb25adedfe32 Mon Sep 17 00:00:00 2001 From: qidaijie Date: Tue, 3 Dec 2019 14:00:44 +0800 Subject: [PATCH 13/27] =?UTF-8?q?=E5=85=AC=E5=85=B1=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=20common=5Fsub=5Faction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- properties/core-site.xml | 71 +++++++++++ properties/hbase-site.xml | 77 ++++++++++++ properties/hdfs-site.xml | 116 ++++++++++++++++++ properties/service_flow_config.properties | 13 +- .../ac/iie/bean/PublicSessionRecordLog.java | 9 ++ .../cn/ac/iie/common/FlowWriteConfig.java | 5 +- .../ac/iie/utils/general/TransFormUtils.java | 2 +- .../cn/ac/iie/utils/hbase/HBaseUtils.java | 36 +++--- .../ac/iie/utils/influxdb/InfluxDbUtils.java | 16 +++ .../java/cn/ac/iie/utils/system/IpUtils.java | 56 +++++++++ src/test/java/cn/ac/iie/test/DomainUtils.java | 61 +++++++-- src/test/java/cn/ac/iie/test/a.json | 85 +++++++++++++ .../java/cn/ac/iie/test/hbase/HBaseTest.java | 78 ++++++------ .../java/cn/ac/iie/test/hbase/IpUtils.java | 63 ++++++++++ src/test/java/cn/ac/iie/test/test.java | 16 ++- 16 files changed, 629 insertions(+), 77 deletions(-) create mode 100644 properties/core-site.xml create mode 100644 properties/hbase-site.xml create mode 100644 properties/hdfs-site.xml create mode 100644 src/main/java/cn/ac/iie/utils/system/IpUtils.java create mode 100644 src/test/java/cn/ac/iie/test/a.json create mode 100644 src/test/java/cn/ac/iie/test/hbase/IpUtils.java diff --git a/pom.xml b/pom.xml index def6d01..44528db 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ properties **/*.properties - **/*.xml + false diff --git a/properties/core-site.xml b/properties/core-site.xml new file mode 100644 index 0000000..93dfb1d --- /dev/null +++ b/properties/core-site.xml @@ -0,0 +1,71 @@ + + + + + + + + + fs.defaultFS + hdfs://ns1 + + + hadoop.tmp.dir + file:/opt/hadoop/tmp + + + io.file.buffer.size + 131702 + + + hadoop.proxyuser.root.hosts + * + + + hadoop.proxyuser.root.groups + * + + + hadoop.logfile.size + 10000000 + The max size of each log file + + + + hadoop.logfile.count + 1 + The max number of log files + + + ha.zookeeper.quorum + master:2181,slave1:2181,slave2:2181 + + +    +     fs.hdfs.impl   +     org.apache.hadoop.hdfs.DistributedFileSystem   +     The FileSystem for hdfs: uris.   + + + +io.compression.codecs +com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec + + +io.compression.codec.lzo.class +com.hadoop.compression.lzo.LzoCodec + + + diff --git a/properties/hbase-site.xml b/properties/hbase-site.xml new file mode 100644 index 0000000..54554e4 --- /dev/null +++ b/properties/hbase-site.xml @@ -0,0 +1,77 @@ + + + + + + hbase.rootdir + hdfs://ns1/hbase-1.4.9 + + + hbase.cluster.distributed + true + + + hbase.zookeeper.quorum + 192.168.40.119,192.168.40.122,192.168.40.123 + + +hbase.master.info.port +60010 + + + + phoenix.schema.isNamespaceMappingEnabled + true + + + phoenix.schema.mapSystemTablesToNamespace + true + + + hbase.client.keyvalue.maxsize + 99428800 + + + hbase.server.keyvalue.maxsize + 99428800 + + + hbase.regionserver.wal.codec + org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec + + + phoenix.query.timeoutMs + 1800000 + + + hbase.rpc.timeout + 1200000 + + + hbase.client.scanner.caching + 1000 + + + hbase.client.scanner.timeout.period + 1200000 + + diff --git a/properties/hdfs-site.xml b/properties/hdfs-site.xml new file mode 100644 index 0000000..1e148b7 --- /dev/null +++ b/properties/hdfs-site.xml @@ -0,0 +1,116 @@ + + + + + + + + + dfs.namenode.name.dir + file:/home/ceiec/hadoop/dfs/name + + + dfs.datanode.data.dir + file:/home/ceiec/hadoop/dfs/data + + + dfs.replication + 2 + + + dfs.namenode.secondary.http-address + 192.168.40.119:9001 + + + dfs.webhdfs.enabled + true + + + dfs.permissions + false + + + dfs.permissions.enabled + false + + + dfs.nameservices + ns1 + + + dfs.blocksize + 134217728 + + + dfs.ha.namenodes.ns1 + nn1,nn2 + + + + dfs.namenode.rpc-address.ns1.nn1 + 192.168.40.119:8020 + + + + dfs.namenode.http-address.ns1.nn1 + 192.168.40.119:50070 + + + + dfs.namenode.rpc-address.ns1.nn2 + 192.168.40.122:8020 + + + + dfs.namenode.http-address.ns1.nn2 + 192.168.40.122:50070 + + + + dfs.namenode.shared.edits.dir + qjournal://192.168.40.119:8485;192.168.40.122:8485;192.168.40.123:8485/ns1 + + + + dfs.journalnode.edits.dir + /home/ceiec/hadoop/journal + + + + dfs.client.failover.proxy.provider.ns1 + org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider + + + + dfs.ha.fencing.methods + sshfence + + + + dfs.ha.fencing.ssh.private-key-files + /root/.ssh/id_rsa + + + + dfs.ha.fencing.ssh.connect-timeout + 30000 + + + + dfs.ha.automatic-failover.enabled + true + + + diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index 8646d78..24a5401 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -6,9 +6,10 @@ zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 #hbase zookeeper地址 hbase.zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 +#hbase.zookeeper.servers=192.168.40.224:2182 #hbase tablename -hbase.table.name=subcriber_info +hbase.table.name=subscriber_info #latest/earliest auto.offset.reset=latest @@ -17,22 +18,22 @@ auto.offset.reset=latest kafka.topic=SECURITY-EVENT-LOG #读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； -group.id=security-policy-191112 +group.id=security-policy-191114 #输出topic results.output.topic=SECURITY-EVENT-COMPLETED-LOG #storm topology workers -topology.workers=2 +topology.workers=3 #spout并行度 建议与kafka分区数相同 -spout.parallelism=3 +spout.parallelism=6 #处理补全操作的bolt并行度-worker的倍数 -datacenter.bolt.parallelism=10 +datacenter.bolt.parallelism=12 #写入kafka的并行度10 -kafka.bolt.parallelism=10 +kafka.bolt.parallelism=12 #定位库地址 ip.library=/home/ceiec/topology/dat/ diff --git a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java index 2883835..541e7b1 100644 --- a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java +++ b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java @@ -41,6 +41,7 @@ public class PublicSessionRecordLog { private String common_user_region; private String common_client_ip; private String common_device_id; + private String common_sub_action; private String common_isp; private String common_sled_ip; private String common_client_location; @@ -381,4 +382,12 @@ public class PublicSessionRecordLog { public void setCommon_stream_trace_id(long common_stream_trace_id) { this.common_stream_trace_id = common_stream_trace_id; } + + public String getCommon_sub_action() { + return common_sub_action; + } + + public void setCommon_sub_action(String common_sub_action) { + this.common_sub_action = common_sub_action; + } } diff --git a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java index f91a57a..f16e0da 100644 --- a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java +++ b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java @@ -8,10 +8,9 @@ import cn.ac.iie.utils.system.FlowWriteConfigurations; */ public class FlowWriteConfig { - public static final String LOG_STRING_SPLITTER = "\t"; - public static final String SQL_STRING_SPLITTER = "#"; + public static final int IPV4_TYPE = 1; + public static final int IPV6_TYPE = 2; public static final String DOMAIN_SPLITTER = "."; - /** * System */ diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index 79e067e..72003c0 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -60,7 +60,7 @@ public class TransFormUtils { radiusSessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); return JSONObject.toJSONString(radiusSessionRecordLog); } catch (Exception e) { - logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常"); + logger.error("{} 日志解析过程出现异常" + FlowWriteConfig.KAFKA_TOPIC); e.printStackTrace(); return ""; } diff --git a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java index f1db9aa..068f619 100644 --- a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java +++ b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java @@ -1,7 +1,8 @@ package cn.ac.iie.utils.hbase; import cn.ac.iie.common.FlowWriteConfig; -import com.zdjizhi.utils.StringUtil; +import cn.ac.iie.utils.system.IpUtils; +import io.netty.util.collection.IntObjectHashMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; @@ -14,6 +15,7 @@ import org.apache.log4j.Logger; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * HBase 工具类 @@ -23,7 +25,7 @@ import java.util.Map; public class HBaseUtils { private final static Logger logger = Logger.getLogger(HBaseUtils.class); - private static Map subIdMap = new HashMap<>(32); + private static Map subIdMap = new ConcurrentHashMap<>(333334); private static Connection connection; private static Long time; @@ -32,6 +34,9 @@ public class HBaseUtils { Configuration configuration = HBaseConfiguration.create(); // 设置zookeeper节点 configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); + configuration.set("hbase.client.retries.number", "3"); + configuration.set("hbase.bulkload.retries.number", "3"); + configuration.set("zookeeper.recovery.retry", "3"); try { connection = ConnectionFactory.createConnection(configuration); time = System.currentTimeMillis(); @@ -47,7 +52,7 @@ public class HBaseUtils { */ public static void change() { Long nowTime = System.currentTimeMillis(); - timestampsFilter(time - 1500, nowTime + 500); + timestampsFilter(time - 1000, nowTime + 500); } /** @@ -65,15 +70,23 @@ public class HBaseUtils { table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); scan2.setTimeRange(startTime, endTime); scanner = table.getScanner(scan2); - logger.warn("读取HBase变量耗时：" + (System.currentTimeMillis() - begin)); for (Result result : scanner) { Cell[] cells = result.rawCells(); for (Cell cell : cells) { - subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); + String key = Bytes.toString(CellUtil.cloneRow(cell)); + String value = Bytes.toString(CellUtil.cloneValue(cell)); + if (subIdMap.containsKey(key)) { + if (!value.equals(subIdMap.get(key))) { + subIdMap.put(key, value); + } + } else { + subIdMap.put(key, value); + } } } + Long end = System.currentTimeMillis(); logger.warn("当前集合长度" + subIdMap.keySet().size()); - logger.warn("更新缓存耗时：" + (System.currentTimeMillis() - begin)); + logger.warn("更新缓存耗时：" + (end - begin) + "开始时间：" + begin + "结束时间：" + end); time = endTime; } catch (IOException e) { e.printStackTrace(); @@ -114,18 +127,13 @@ public class HBaseUtils { } } - /** * 获取 account * - * @param ip client_ip + * @param clientIp client_ip * @return account */ - public static String getAccount(String ip) { - if (StringUtil.isNotBlank(ip)) { - return subIdMap.get(ip); - } else { - return ""; - } + public static String getAccount(String clientIp) { + return subIdMap.get(clientIp); } } diff --git a/src/main/java/cn/ac/iie/utils/influxdb/InfluxDbUtils.java b/src/main/java/cn/ac/iie/utils/influxdb/InfluxDbUtils.java index 124344f..c51589b 100644 --- a/src/main/java/cn/ac/iie/utils/influxdb/InfluxDbUtils.java +++ b/src/main/java/cn/ac/iie/utils/influxdb/InfluxDbUtils.java @@ -45,6 +45,22 @@ public class InfluxDbUtils { } } + /** + * 记录对准失败次数-即内存中没有对应的key + * + * @param failure 对准失败量 + */ + public static void sendHBaseFailure(int failure) { + if (failure != 0) { + InfluxDB client = InfluxDBFactory.connect(FlowWriteConfig.INFLUX_IP, FlowWriteConfig.INFLUX_USERNAME, FlowWriteConfig.INFLUX_PASSWORD); + Point point1 = Point.measurement("sendHBaseFailure") + .tag("topic", FlowWriteConfig.KAFKA_TOPIC) + .field("failure", failure) + .build(); + client.write("BusinessMonitor", "", point1); + } + } + /** * 获取本机IP * diff --git a/src/main/java/cn/ac/iie/utils/system/IpUtils.java b/src/main/java/cn/ac/iie/utils/system/IpUtils.java new file mode 100644 index 0000000..94fefa6 --- /dev/null +++ b/src/main/java/cn/ac/iie/utils/system/IpUtils.java @@ -0,0 +1,56 @@ +package cn.ac.iie.utils.system; + +/** + * IP工具类 + * + * @author qidaijie + */ +public class IpUtils { + /** + * IPV4 正则 + */ + private static final String IPV4 = "^((\\d|[1-9]\\d|1\\d\\d|2([0-4]\\d|5[0-5]))\\.){4}$"; + /** + * IPV6正则 + */ + private static final String IPV6 = "^(([\\da-fA-F]{1,4}):){8}$"; + + + /** + * 判断IP类型 v4 or v6 + * + * @param ip IP + * @return 1:v4 2:v6 3:abnormal + */ + public static int validIPAddress(String ip) { + return String.format("%s.", ip).matches(IPV4) ? 1 : String.format("%s:", ip).matches(IPV6) ? 2 : 3; + } + + /** + * ip字符串转整数 + * ip是.分割的整数字符串,按照r进制转十进制的规律,按权相加求和,这里的权是256. + * + * @param ip IP + * @return ip(int) + */ + public static int ipChangeInt(String ip) { + //分割ip + String[] ipSplit = ip.split("\\."); + int result = 0; + for (int i = 0; i < 4; i++) { + Integer ipSubInteger = Integer.parseInt(ipSplit[i]); + //正则验证不能为负数 + if (ipSubInteger > 255) { + result = 0; + break; + } + result += (ipSubInteger << (24 - i * 8)); + } + return result; + } + + public static void main(String[] args) { + System.out.println(validIPAddress("192.254.254.254")); + System.out.println(ipChangeInt("254.254.254.254")); + } +} diff --git a/src/test/java/cn/ac/iie/test/DomainUtils.java b/src/test/java/cn/ac/iie/test/DomainUtils.java index e7bdf78..ec2cce4 100644 --- a/src/test/java/cn/ac/iie/test/DomainUtils.java +++ b/src/test/java/cn/ac/iie/test/DomainUtils.java @@ -5,33 +5,72 @@ import com.zdjizhi.utils.StringUtil; import javax.xml.bind.SchemaOutputResolver; import java.net.MalformedURLException; import java.net.URL; -import java.util.LinkedList; -import java.util.List; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; public class DomainUtils { - private static Pattern pattern = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.cc|\\.me|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)"); + private static Pattern pattern = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.me|\\.cc|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)"); + + private final static Set PublicSuffixSet = new HashSet( + Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se" + .split("\\|"))); + private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})"); public static void main(String[] args) { - System.out.println(getTopDomain("agoo-report.m.taobao.com")); + System.out.println(getTopDomain("http://www.ccb.com")); + System.out.println(getDomainName("www.comaa.com")); + } private static String getTopDomain(String url) { -// try { - //获取值转换为小写 -// String host = new URL(url).getHost().toLowerCase();//news.hexun.com + try { +// 获取值转换为小写 + String host = new URL(url).getHost().toLowerCase();//news.hexun.com // Pattern pattern = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.cc|\\.me|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)"); - Matcher matcher = pattern.matcher(url); + Matcher matcher = IP_PATTERN.matcher(host); if (matcher.find()){ return matcher.group(); } -// } catch (MalformedURLException e) { -// e.printStackTrace(); -// } + } catch (MalformedURLException e) { + e.printStackTrace(); + } return null; } + +// private final static Set PublicSuffixSet = new HashSet( +// Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se" +// .split("\\|"))); +// private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})"); + + /** + * 获取url的顶级域名 +// * @param url + * @return + */ + public static String getDomainName(String host) { +// String host = url.getHost(); + if (host.endsWith(".")){ + host = host.substring(0, host.length() - 1); + } + if (IP_PATTERN.matcher(host).matches()){ + return host; + } + int index = 0; + String candidate = host; + for (; index >= 0;) { + index = candidate.indexOf('.'); + String subCandidate = candidate.substring(index + 1); + if (PublicSuffixSet.contains(subCandidate)) { + return candidate; + } + candidate = subCandidate; + } + return candidate; + } + + } diff --git a/src/test/java/cn/ac/iie/test/a.json b/src/test/java/cn/ac/iie/test/a.json new file mode 100644 index 0000000..e70c1b8 --- /dev/null +++ b/src/test/java/cn/ac/iie/test/a.json @@ -0,0 +1,85 @@ +{ + "bgp_type": 0, + "common_action": 16, + "common_address_list": "", + "common_address_type": 4, + "common_app_id": 0, + "common_app_label": "", + "common_c2s_byte_num": 639, + "common_c2s_pkt_num": 1, + "common_client_asn": "36351", + "common_client_ip": "75.126.99.155", + "common_client_location": "Dallas\tTexas\tUnited States", + "common_client_port": 40846, + "common_con_duration_ms": 113814, + "common_device_id": "4586496", + "common_direction": 1, + "common_encapsulation": 8, + "common_end_time": 1574842412, + "common_entrance_id": 8, + "common_has_dup_traffic": 1, + "common_isp": "China Telecom", + "common_l4_protocol": "MPLS", + "common_link_id": 1, + "common_log_id": 172027081238036520, + "common_policy_id": 902, + "common_protocol_id": 0, + "common_recv_time": 1574842413, + "common_s2c_byte_num": 1360, + "common_s2c_pkt_num": 26, + "common_schema_type": "BGP", + "common_server_asn": "9050", + "common_server_ip": "92.85.69.150", + "common_server_location": "Romania", + "common_server_port": 53, + "common_service": 8, + "common_sled_ip": "192.168.10.58", + "common_start_time": 1574842361, + "common_stream_dir": 1, + "common_stream_error": "", + "common_stream_trace_id": 0, + "common_subscriber_id": "zareP", + "common_user_region": "973ebGTTwdBhecbqI9U724LJdyHWV3BOUIcy4jgtpd221GV2QSOLMZc2awba3GfqKCiQxfirv5NjptRbawXDIpw4pJ0Xg4WZJSKW", + "common_user_tags": "qeT9tif1iRp1qCq6pauMO0RqsV13ktQm4Jlp4ZBOFeaQufoJMbC5tQ70ebDI1F9Ffw8c580e9yd27v96M6i4CPN8mEDw1mIkMexT", + "dns_aa": 0, + "dns_ancount": 0, + "dns_arcount": 0, + "dns_message_id": 0, + "dns_nscount": 0, + "dns_opcode": 0, + "dns_qclass": 0, + "dns_qdcount": 0, + "dns_qr": 0, + "dns_qtype": 0, + "dns_ra": 0, + "dns_rcode": 0, + "dns_rd": 0, + "dns_sub": 0, + "dns_tc": 0, + "http_content_length": "48895", + "http_content_type": "application/x-jpg", + "http_domain": "zhiyin.cn", + "http_host": "v.zhiyin.cn97991", + "http_proxy_flag": 1, + "http_referer": "", + "http_request_body": "", + "http_request_body_key": "", + "http_request_header": "", + "http_request_line": "", + "http_response_body": "", + "http_response_body_key": "", + "http_response_header": "", + "http_response_line": "", + "http_sequence": 6, + "http_set_cookie": "", + "http_snapshot": "", + "http_url": "http://v.zhiyin.cn/watch/295.html1661741", + "http_user_agent": "", + "http_version": "http1", + "ssl_cert_verify": 0, + "ssl_client_side_latency": 0, + "ssl_con_latency_ms": 0, + "ssl_intercept_state": 0, + "ssl_pinningst": 0, + "ssl_server_side_latency": 0 +} diff --git a/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java index 9e94387..44f176a 100644 --- a/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java +++ b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java @@ -2,7 +2,9 @@ package cn.ac.iie.test.hbase; import cn.ac.iie.common.FlowWriteConfig; import cn.ac.iie.utils.hbase.HBaseUtils; +import cn.ac.iie.utils.system.IpUtils; import com.zdjizhi.utils.StringUtil; +import io.netty.util.collection.IntObjectHashMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; @@ -17,10 +19,13 @@ import org.junit.Test; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; public class HBaseTest { private final static Logger logger = Logger.getLogger(HBaseTest.class); - private static Map subIdMap = new HashMap<>(16); +// private static Map subIdMap = new ConcurrentHashMap(13333334); + private static Map subIdMap = new HashMap<>(13333334); private static Connection connection; private static Long time; @@ -29,9 +34,11 @@ public class HBaseTest { // 管理Hbase的配置信息 Configuration configuration = HBaseConfiguration.create(); // 设置zookeeper节点 - configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); // configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); -// configuration.setInt("zookeeper.session.timeout", 2000); + configuration.set("hbase.zookeeper.quorum", "192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181"); + configuration.set("hbase.client.retries.number", "3"); + configuration.set("hbase.bulkload.retries.number", "3"); + configuration.set("zookeeper.recovery.retry", "3"); try { connection = ConnectionFactory.createConnection(configuration); time = System.currentTimeMillis(); @@ -41,22 +48,10 @@ public class HBaseTest { } } -// @Before -// public void connHBase() { -// -// } - @Test public void change() { // Long begin = System.currentTimeMillis(); -// System.gc(); -// Long start = Runtime.getRuntime().freeMemory(); -// System.out.println("开始内存"+start); // getAll(); -// System.gc(); -// Long end = Runtime.getRuntime().freeMemory(); -// System.out.println("结束内存"+end); -// System.out.println( "一个HashMap对象占内存: " + (end - start)); // System.out.println(System.currentTimeMillis() - begin); } @@ -67,31 +62,39 @@ public class HBaseTest { * @param endTime 结束时间 */ private static void timestampsFilter(Long startTime, Long endTime) { + Long begin = System.currentTimeMillis(); Table table = null; - TableName tableName = TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME); - Admin admin = null; + ResultScanner scanner = null; + Scan scan2 = new Scan(); try { table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); - Scan scan2 = new Scan(); -// scan2.setCaching(500); scan2.setTimeRange(startTime, endTime); - ResultScanner scanner = table.getScanner(scan2); - scanner.next().isEmpty(); + scanner = table.getScanner(scan2); for (Result result : scanner) { Cell[] cells = result.rawCells(); for (Cell cell : cells) { - subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); +// int key = Integer.parseInt(Bytes.toString(CellUtil.cloneRow(cell))); + String key = Bytes.toString(CellUtil.cloneRow(cell)); + String value = Bytes.toString(CellUtil.cloneValue(cell)); + if (subIdMap.containsKey(key)) { + if (!value.equals(subIdMap.get(key))) { + subIdMap.put(key, value); + } + } else { + subIdMap.put(key, value); + } } } - admin = connection.getAdmin(); - admin.flush(tableName); + Long end = System.currentTimeMillis(); logger.warn("当前集合长度" + subIdMap.keySet().size()); - logger.warn("更新后集合keys：" + subIdMap.keySet()); + logger.warn("更新缓存耗时：" + (end - begin) + "开始时间：" + begin + "结束时间：" + end); time = endTime; - scanner.close(); } catch (IOException e) { e.printStackTrace(); } finally { + if (scanner != null) { + scanner.close(); + } if (table != null) { try { table.close(); @@ -105,7 +108,8 @@ public class HBaseTest { /** * 获取所有的 key value */ - public static void getAll() { + private static void getAll() { + Long begin = System.currentTimeMillis(); try { Table table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); Scan scan2 = new Scan(); @@ -113,28 +117,32 @@ public class HBaseTest { for (Result result : scanner) { Cell[] cells = result.rawCells(); for (Cell cell : cells) { +// subIdMap.put(Integer.valueOf(Bytes.toString(CellUtil.cloneRow(cell))), Bytes.toString(CellUtil.cloneValue(cell))); subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); } } - logger.warn("获取全量后集合size：" + subIdMap.size()); + logger.warn("获取全量后集合长度：" + subIdMap.size()); + logger.warn("获取全量耗时：" + (System.currentTimeMillis() - begin)); scanner.close(); } catch (IOException e) { e.printStackTrace(); } } - /** * 获取 account * - * @param ip client_ip + * @param clientIp client_ip * @return account */ - public static String getAccount(String ip) { - if (StringUtil.isNotBlank(ip)) { - return subIdMap.get(ip); - } else { - return ""; + public static String getAccount(String clientIp) { + int ipType = cn.ac.iie.utils.system.IpUtils.validIPAddress(clientIp); + String account = ""; + if (ipType == FlowWriteConfig.IPV4_TYPE) { + account = subIdMap.get(IpUtils.ipChangeInt(clientIp)); + } else if (ipType == FlowWriteConfig.IPV6_TYPE) { + account = subIdMap.get(clientIp); } + return account; } } diff --git a/src/test/java/cn/ac/iie/test/hbase/IpUtils.java b/src/test/java/cn/ac/iie/test/hbase/IpUtils.java new file mode 100644 index 0000000..1cfebe0 --- /dev/null +++ b/src/test/java/cn/ac/iie/test/hbase/IpUtils.java @@ -0,0 +1,63 @@ +package cn.ac.iie.test.hbase; + +import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.general.EncryptionUtils; +import org.apache.log4j.Logger; +import org.slf4j.LoggerFactory; + + +public class IpUtils { + private static Logger logger = Logger.getLogger(IpUtils.class); + + public static void main(String[] args) { + System.out.println(System.currentTimeMillis()); + System.out.println(System.currentTimeMillis() - 60000); + } + + /** + * ip字符串转整数 + * ip是.分割的整数字符串,按照r进制转十进制的规律,按权相加求和,这里的权是256. + * + * @param ip + * @return + */ + public static int ip2Int(String ip) { + String[] ipStrs = ip.split("\\.");//分割ip + int result = 0; + for (int i = 0; i < 4; i++) { + Integer ipSubInteger = Integer.parseInt(ipStrs[i]); + if (ipSubInteger > 255) {//正则验证不能为负数 + result = 0; + break; + } + result += (ipSubInteger << (24 - i * 8)); + } + return result; + } + + + /** + * 整数转ip + * + * @param ip + * @return + */ + public static String int2Ip(int ip) { + StringBuilder builder = new StringBuilder(String.valueOf(ip >>> 24)); + builder.append("."); + builder.append(String.valueOf((ip & 0X00FFFFFF) >>> 16)); + builder.append("."); + builder.append(String.valueOf((ip & 0X0000FFFF) >>> 8)); + builder.append("."); + builder.append(String.valueOf(ip & 0X000000FF)); + return builder.toString(); + } + + + public static int validIPAddress(String ip) { + String ipv4 = "^((\\d|[1-9]\\d|1\\d\\d|2([0-4]\\d|5[0-5]))\\.){4}$"; + //8个1-4位+: + String ipv6 = "^(([\\da-fA-F]{1,4}):){8}$"; + return String.format("%s.", ip).matches(ipv4) ? 1 : String.format("%s:", ip).matches(ipv6) ? 2 : 3; + } +} diff --git a/src/test/java/cn/ac/iie/test/test.java b/src/test/java/cn/ac/iie/test/test.java index f65cd5f..7876fa2 100644 --- a/src/test/java/cn/ac/iie/test/test.java +++ b/src/test/java/cn/ac/iie/test/test.java @@ -1,25 +1,29 @@ package cn.ac.iie.test; +import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.general.TransFormUtils; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; +import org.apache.log4j.Logger; import org.junit.Test; import java.util.HashMap; import java.util.Map; public class test { + private static Logger logger = Logger.getLogger(test.class); public static void main(String[] args) { String message = "{\"str_ea_m-t-r-a-ceid\":\"JSON\",\"uid\":\"0\"}"; // SessionRecordLog sessionRecordLog = JSONObject.parseObject(message, SessionRecordLog.class); // System.out.println(JSONObject.toJSONString(sessionRecordLog)); + JSONObject obj = JSONObject.parseObject(message); + obj.put("abc","bca"); + System.out.println(obj.toString()); + } @Test public void test2() { - Map map = new HashMap<>(); - map.put("a","a"); - map.put("b","a"); - map.put("c","a"); - - System.out.println(map.keySet()); + logger.error("{} 日志解析过程出现异常" + FlowWriteConfig.KAFKA_TOPIC); } } From fb22d60bfe5b974b402e94b10dfe38930bc50003 Mon Sep 17 00:00:00 2001 From: qidaijie Date: Fri, 6 Dec 2019 18:50:31 +0800 Subject: [PATCH 14/27] =?UTF-8?q?=E6=9B=BF=E6=8D=A2=E6=9F=90=E4=BA=9B?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E7=B1=BB=E5=9E=8B=E4=B8=BAInteger=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=EF=BC=8C=E9=98=B2=E6=AD=A2int=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E4=B8=BA0=E5=BD=B1=E5=93=8D=E6=9C=AC=E8=BA=AB=E8=AF=AD?= =?UTF-8?q?=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- properties/service_flow_config.properties | 13 +-- .../ac/iie/bean/PublicSessionRecordLog.java | 30 ++++--- .../bean/connection/ConnectionRecordLog.java | 66 +++++++------- .../iie/bean/security/SecurityPolicyLog.java | 32 ++++--- .../ac/iie/topology/LogFlowWriteTopology.java | 3 - .../cn/ac/iie/utils/hbase/HBaseUtils.java | 5 +- src/test/java/cn/ac/iie/test/a.json | 86 +------------------ .../java/cn/ac/iie/test/bean/Student.java | 22 +++++ .../java/cn/ac/iie/test/hbase/HBaseTest.java | 23 +++-- src/test/java/cn/ac/iie/test/test.java | 23 ++--- 10 files changed, 114 insertions(+), 189 deletions(-) create mode 100644 src/test/java/cn/ac/iie/test/bean/Student.java diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index 24a5401..66f5690 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -1,12 +1,13 @@ #管理kafka地址 -bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092 +#bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092 +bootstrap.servers=192.168.40.186:9092 #zookeeper 地址 zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 #hbase zookeeper地址 -hbase.zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 -#hbase.zookeeper.servers=192.168.40.224:2182 +#hbase.zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 +hbase.zookeeper.servers=192.168.40.224:2182 #hbase tablename hbase.table.name=subscriber_info @@ -15,13 +16,13 @@ hbase.table.name=subscriber_info auto.offset.reset=latest #kafka broker下的topic名称 -kafka.topic=SECURITY-EVENT-LOG +kafka.topic=CONNECTION-RECORD-LOG #读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； group.id=security-policy-191114 #输出topic -results.output.topic=SECURITY-EVENT-COMPLETED-LOG +results.output.topic=CONNECTION-RECORD-COMPLETED-LOG #storm topology workers topology.workers=3 @@ -37,7 +38,7 @@ kafka.bolt.parallelism=12 #定位库地址 ip.library=/home/ceiec/topology/dat/ -#ip.library=D:/dat/ +#ip.library=D:\\workerSpace\\K18-Phase2\\3.0.2019115\\log-stream-completion\\ #kafka批量条数 batch.insert.num=2000 diff --git a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java index 541e7b1..123ccff 100644 --- a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java +++ b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java @@ -63,13 +63,6 @@ public class PublicSessionRecordLog { this.common_service = common_service; } - public int getCommon_direction() { - return common_direction; - } - - public void setCommon_direction(int common_direction) { - this.common_direction = common_direction; - } public long getCommon_recv_time() { return common_recv_time; @@ -127,13 +120,6 @@ public class PublicSessionRecordLog { this.common_link_id = common_link_id; } - public int getCommon_encapsulation() { - return common_encapsulation; - } - - public void setCommon_encapsulation(int common_encapsulation) { - this.common_encapsulation = common_encapsulation; - } public int getCommon_server_port() { return common_server_port; @@ -223,6 +209,22 @@ public class PublicSessionRecordLog { this.common_stream_dir = common_stream_dir; } + public int getCommon_direction() { + return common_direction; + } + + public void setCommon_direction(int common_direction) { + this.common_direction = common_direction; + } + + public int getCommon_encapsulation() { + return common_encapsulation; + } + + public void setCommon_encapsulation(int common_encapsulation) { + this.common_encapsulation = common_encapsulation; + } + public int getCommon_has_dup_traffic() { return common_has_dup_traffic; } diff --git a/src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java b/src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java index e73c2d2..01d9ee4 100644 --- a/src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java +++ b/src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java @@ -49,8 +49,8 @@ public class ConnectionRecordLog extends PublicSessionRecordLog { //TODO 3DNS协议属性 18 private int dns_message_id; - private int dns_qr; - private int dns_opcode; + private Integer dns_qr; + private Integer dns_opcode; private int dns_aa; private int dns_tc; private int dns_rd; @@ -70,11 +70,11 @@ public class ConnectionRecordLog extends PublicSessionRecordLog { //TODO SSL协议属性 13 - private int ssl_pinningst; - private int ssl_intercept_state; + private Integer ssl_pinningst; + private Integer ssl_intercept_state; private int ssl_server_side_latency; private int ssl_client_side_latency; - private int ssl_cert_verify; + private Integer ssl_cert_verify; private int ssl_con_latency_ms; private String ssl_version; private String ssl_sni; @@ -161,6 +161,23 @@ public class ConnectionRecordLog extends PublicSessionRecordLog { this.http_response_header = http_response_header; } + + public Integer getSsl_pinningst() { + return ssl_pinningst; + } + + public void setSsl_pinningst(Integer ssl_pinningst) { + this.ssl_pinningst = ssl_pinningst; + } + + public Integer getSsl_intercept_state() { + return ssl_intercept_state; + } + + public void setSsl_intercept_state(Integer ssl_intercept_state) { + this.ssl_intercept_state = ssl_intercept_state; + } + public String getHttp_request_body() { return http_request_body; } @@ -185,6 +202,14 @@ public class ConnectionRecordLog extends PublicSessionRecordLog { this.http_request_body_key = http_request_body_key; } + public Integer getSsl_cert_verify() { + return ssl_cert_verify; + } + + public void setSsl_cert_verify(Integer ssl_cert_verify) { + this.ssl_cert_verify = ssl_cert_verify; + } + public String getHttp_response_body_key() { return http_response_body_key; } @@ -353,19 +378,19 @@ public class ConnectionRecordLog extends PublicSessionRecordLog { this.dns_message_id = dns_message_id; } - public int getDns_qr() { + public Integer getDns_qr() { return dns_qr; } - public void setDns_qr(int dns_qr) { + public void setDns_qr(Integer dns_qr) { this.dns_qr = dns_qr; } - public int getDns_opcode() { + public Integer getDns_opcode() { return dns_opcode; } - public void setDns_opcode(int dns_opcode) { + public void setDns_opcode(Integer dns_opcode) { this.dns_opcode = dns_opcode; } @@ -489,21 +514,6 @@ public class ConnectionRecordLog extends PublicSessionRecordLog { this.dns_rr = dns_rr; } - public int getSsl_pinningst() { - return ssl_pinningst; - } - - public void setSsl_pinningst(int ssl_pinningst) { - this.ssl_pinningst = ssl_pinningst; - } - - public int getSsl_intercept_state() { - return ssl_intercept_state; - } - - public void setSsl_intercept_state(int ssl_intercept_state) { - this.ssl_intercept_state = ssl_intercept_state; - } public int getSsl_server_side_latency() { return ssl_server_side_latency; @@ -521,14 +531,6 @@ public class ConnectionRecordLog extends PublicSessionRecordLog { this.ssl_client_side_latency = ssl_client_side_latency; } - public int getSsl_cert_verify() { - return ssl_cert_verify; - } - - public void setSsl_cert_verify(int ssl_cert_verify) { - this.ssl_cert_verify = ssl_cert_verify; - } - public int getSsl_con_latency_ms() { return ssl_con_latency_ms; } diff --git a/src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java b/src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java index e266b5e..d6a4e54 100644 --- a/src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java +++ b/src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java @@ -49,8 +49,8 @@ public class SecurityPolicyLog extends PublicSessionRecordLog { //TODO 3DNS协议属性 18 private int dns_message_id; - private int dns_qr; - private int dns_opcode; + private Integer dns_qr; + private Integer dns_opcode; private int dns_aa; private int dns_tc; private int dns_rd; @@ -70,11 +70,11 @@ public class SecurityPolicyLog extends PublicSessionRecordLog { //TODO SSL协议属性 13 - private int ssl_pinningst; - private int ssl_intercept_state; + private Integer ssl_pinningst; + private Integer ssl_intercept_state; private int ssl_server_side_latency; private int ssl_client_side_latency; - private int ssl_cert_verify; + private Integer ssl_cert_verify; private int ssl_con_latency_ms; private String ssl_version; private String ssl_sni; @@ -104,7 +104,6 @@ public class SecurityPolicyLog extends PublicSessionRecordLog { private String streaming_media_url; private String streaming_media_protocol; - public String getHttp_url() { return http_url; } @@ -353,19 +352,19 @@ public class SecurityPolicyLog extends PublicSessionRecordLog { this.dns_message_id = dns_message_id; } - public int getDns_qr() { + public Integer getDns_qr() { return dns_qr; } - public void setDns_qr(int dns_qr) { + public void setDns_qr(Integer dns_qr) { this.dns_qr = dns_qr; } - public int getDns_opcode() { + public Integer getDns_opcode() { return dns_opcode; } - public void setDns_opcode(int dns_opcode) { + public void setDns_opcode(Integer dns_opcode) { this.dns_opcode = dns_opcode; } @@ -489,19 +488,19 @@ public class SecurityPolicyLog extends PublicSessionRecordLog { this.dns_rr = dns_rr; } - public int getSsl_pinningst() { + public Integer getSsl_pinningst() { return ssl_pinningst; } - public void setSsl_pinningst(int ssl_pinningst) { + public void setSsl_pinningst(Integer ssl_pinningst) { this.ssl_pinningst = ssl_pinningst; } - public int getSsl_intercept_state() { + public Integer getSsl_intercept_state() { return ssl_intercept_state; } - public void setSsl_intercept_state(int ssl_intercept_state) { + public void setSsl_intercept_state(Integer ssl_intercept_state) { this.ssl_intercept_state = ssl_intercept_state; } @@ -521,11 +520,11 @@ public class SecurityPolicyLog extends PublicSessionRecordLog { this.ssl_client_side_latency = ssl_client_side_latency; } - public int getSsl_cert_verify() { + public Integer getSsl_cert_verify() { return ssl_cert_verify; } - public void setSsl_cert_verify(int ssl_cert_verify) { + public void setSsl_cert_verify(Integer ssl_cert_verify) { this.ssl_cert_verify = ssl_cert_verify; } @@ -665,7 +664,6 @@ public class SecurityPolicyLog extends PublicSessionRecordLog { this.voip_called_number = voip_called_number; } - public String getStreaming_media_url() { return streaming_media_url; } diff --git a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java index c5c5090..d4eda58 100644 --- a/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java +++ b/src/main/java/cn/ac/iie/topology/LogFlowWriteTopology.java @@ -68,17 +68,14 @@ public class LogFlowWriteTopology { builder.setBolt("ProxyCompletionBolt", new ProxyCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("ProxyCompletionBolt"); break; - case "RADIUS-RECORD-LOG": builder.setBolt("RadiusCompletionBolt", new RadiusCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("RadiusCompletionBolt"); break; - case "CONNECTION-RECORD-LOG": builder.setBolt("CollectCompletedBolt", new CollectCompletedBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("CollectCompletedBolt"); break; - case "SECURITY-EVENT-LOG": builder.setBolt("SecurityCompletionBolt", new SecurityCompletionBolt(), FlowWriteConfig.DATACENTER_BOLT_PARALLELISM).localOrShuffleGrouping("LogFlowWriteSpout"); builder.setBolt("NtcLogSendBolt", new NtcLogSendBolt(), FlowWriteConfig.KAFKA_BOLT_PARALLELISM).localOrShuffleGrouping("SecurityCompletionBolt"); diff --git a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java index 068f619..2b60e43 100644 --- a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java +++ b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java @@ -15,7 +15,7 @@ import org.apache.log4j.Logger; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListMap; /** * HBase 工具类 @@ -25,7 +25,8 @@ import java.util.concurrent.ConcurrentHashMap; public class HBaseUtils { private final static Logger logger = Logger.getLogger(HBaseUtils.class); - private static Map subIdMap = new ConcurrentHashMap<>(333334); + private static Map subIdMap = new HashMap<>(333334); +// private static Map subIdMap = new ConcurrentSkipListMap<>(); private static Connection connection; private static Long time; diff --git a/src/test/java/cn/ac/iie/test/a.json b/src/test/java/cn/ac/iie/test/a.json index e70c1b8..e80ed52 100644 --- a/src/test/java/cn/ac/iie/test/a.json +++ b/src/test/java/cn/ac/iie/test/a.json @@ -1,85 +1 @@ -{ - "bgp_type": 0, - "common_action": 16, - "common_address_list": "", - "common_address_type": 4, - "common_app_id": 0, - "common_app_label": "", - "common_c2s_byte_num": 639, - "common_c2s_pkt_num": 1, - "common_client_asn": "36351", - "common_client_ip": "75.126.99.155", - "common_client_location": "Dallas\tTexas\tUnited States", - "common_client_port": 40846, - "common_con_duration_ms": 113814, - "common_device_id": "4586496", - "common_direction": 1, - "common_encapsulation": 8, - "common_end_time": 1574842412, - "common_entrance_id": 8, - "common_has_dup_traffic": 1, - "common_isp": "China Telecom", - "common_l4_protocol": "MPLS", - "common_link_id": 1, - "common_log_id": 172027081238036520, - "common_policy_id": 902, - "common_protocol_id": 0, - "common_recv_time": 1574842413, - "common_s2c_byte_num": 1360, - "common_s2c_pkt_num": 26, - "common_schema_type": "BGP", - "common_server_asn": "9050", - "common_server_ip": "92.85.69.150", - "common_server_location": "Romania", - "common_server_port": 53, - "common_service": 8, - "common_sled_ip": "192.168.10.58", - "common_start_time": 1574842361, - "common_stream_dir": 1, - "common_stream_error": "", - "common_stream_trace_id": 0, - "common_subscriber_id": "zareP", - "common_user_region": "973ebGTTwdBhecbqI9U724LJdyHWV3BOUIcy4jgtpd221GV2QSOLMZc2awba3GfqKCiQxfirv5NjptRbawXDIpw4pJ0Xg4WZJSKW", - "common_user_tags": "qeT9tif1iRp1qCq6pauMO0RqsV13ktQm4Jlp4ZBOFeaQufoJMbC5tQ70ebDI1F9Ffw8c580e9yd27v96M6i4CPN8mEDw1mIkMexT", - "dns_aa": 0, - "dns_ancount": 0, - "dns_arcount": 0, - "dns_message_id": 0, - "dns_nscount": 0, - "dns_opcode": 0, - "dns_qclass": 0, - "dns_qdcount": 0, - "dns_qr": 0, - "dns_qtype": 0, - "dns_ra": 0, - "dns_rcode": 0, - "dns_rd": 0, - "dns_sub": 0, - "dns_tc": 0, - "http_content_length": "48895", - "http_content_type": "application/x-jpg", - "http_domain": "zhiyin.cn", - "http_host": "v.zhiyin.cn97991", - "http_proxy_flag": 1, - "http_referer": "", - "http_request_body": "", - "http_request_body_key": "", - "http_request_header": "", - "http_request_line": "", - "http_response_body": "", - "http_response_body_key": "", - "http_response_header": "", - "http_response_line": "", - "http_sequence": 6, - "http_set_cookie": "", - "http_snapshot": "", - "http_url": "http://v.zhiyin.cn/watch/295.html1661741", - "http_user_agent": "", - "http_version": "http1", - "ssl_cert_verify": 0, - "ssl_client_side_latency": 0, - "ssl_con_latency_ms": 0, - "ssl_intercept_state": 0, - "ssl_pinningst": 0, - "ssl_server_side_latency": 0 -} +{"common_stream_dir":3,"common_address_type":4,"common_client_ip":"82.200.242.225","common_server_ip":"82.200.242.69","common_client_port":59387,"common_server_port":1812,"common_c2s_pkt_num":2,"common_s2c_pkt_num":1,"common_c2s_byte_num":507,"common_s2c_byte_num":151,"common_start_time":1575534194,"common_end_time":1575534195,"common_con_duration_ms":1000,"common_stream_trace_id":0,"common_l4_protocol":"IPv4_UDP","common_address_list":"59387-1812-82.200.242.225-82.200.242.69","radius_packet_type":1,"radius_account":"Kuanysh79143","radius_service_type":2,"radius_acct_session_id":"473332153","radius_framed_ip":"82.200.242.225","common_policy_id":0,"common_service":162,"common_entrance_id":0,"common_direction":0,"common_device_id":0,"common_encapsulation":14,"common_link_id":0,"common_sled_ip":"192.168.40.119","common_schema_type":"RADIUS"} \ No newline at end of file diff --git a/src/test/java/cn/ac/iie/test/bean/Student.java b/src/test/java/cn/ac/iie/test/bean/Student.java new file mode 100644 index 0000000..3383bc0 --- /dev/null +++ b/src/test/java/cn/ac/iie/test/bean/Student.java @@ -0,0 +1,22 @@ +package cn.ac.iie.test.bean; + +public class Student { + private String name; + private Integer age; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Integer getAge() { + return age; + } + + public void setAge(Integer age) { + this.age = age; + } +} diff --git a/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java index 44f176a..a6fb619 100644 --- a/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java +++ b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java @@ -1,10 +1,7 @@ package cn.ac.iie.test.hbase; import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.system.IpUtils; -import com.zdjizhi.utils.StringUtil; -import io.netty.util.collection.IntObjectHashMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; @@ -13,19 +10,18 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.apache.log4j.Logger; -import org.junit.Before; import org.junit.Test; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListMap; public class HBaseTest { private final static Logger logger = Logger.getLogger(HBaseTest.class); // private static Map subIdMap = new ConcurrentHashMap(13333334); private static Map subIdMap = new HashMap<>(13333334); + private static Map testMap = new ConcurrentSkipListMap<>(); private static Connection connection; private static Long time; @@ -35,7 +31,8 @@ public class HBaseTest { Configuration configuration = HBaseConfiguration.create(); // 设置zookeeper节点 // configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); - configuration.set("hbase.zookeeper.quorum", "192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181"); +// configuration.set("hbase.zookeeper.quorum", "192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181"); + configuration.set("hbase.zookeeper.quorum", "192.168.40.224:2182"); configuration.set("hbase.client.retries.number", "3"); configuration.set("hbase.bulkload.retries.number", "3"); configuration.set("zookeeper.recovery.retry", "3"); @@ -50,9 +47,9 @@ public class HBaseTest { @Test public void change() { -// Long begin = System.currentTimeMillis(); -// getAll(); -// System.out.println(System.currentTimeMillis() - begin); + Long begin = System.currentTimeMillis(); + getAll(); + System.out.println(System.currentTimeMillis() - begin); } /** @@ -118,10 +115,12 @@ public class HBaseTest { Cell[] cells = result.rawCells(); for (Cell cell : cells) { // subIdMap.put(Integer.valueOf(Bytes.toString(CellUtil.cloneRow(cell))), Bytes.toString(CellUtil.cloneValue(cell))); - subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); +// subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); + testMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); } } - logger.warn("获取全量后集合长度：" + subIdMap.size()); +// logger.warn("获取全量后集合长度：" + subIdMap.size()); + logger.warn("获取全量后集合长度：" + testMap.size()); logger.warn("获取全量耗时：" + (System.currentTimeMillis() - begin)); scanner.close(); } catch (IOException e) { diff --git a/src/test/java/cn/ac/iie/test/test.java b/src/test/java/cn/ac/iie/test/test.java index 7876fa2..aa5b7db 100644 --- a/src/test/java/cn/ac/iie/test/test.java +++ b/src/test/java/cn/ac/iie/test/test.java @@ -1,29 +1,16 @@ package cn.ac.iie.test; -import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.general.TransFormUtils; -import com.alibaba.fastjson.JSON; +import cn.ac.iie.test.bean.Student; import com.alibaba.fastjson.JSONObject; import org.apache.log4j.Logger; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; public class test { private static Logger logger = Logger.getLogger(test.class); + public static void main(String[] args) { - String message = "{\"str_ea_m-t-r-a-ceid\":\"JSON\",\"uid\":\"0\"}"; -// SessionRecordLog sessionRecordLog = JSONObject.parseObject(message, SessionRecordLog.class); -// System.out.println(JSONObject.toJSONString(sessionRecordLog)); - JSONObject obj = JSONObject.parseObject(message); - obj.put("abc","bca"); - System.out.println(obj.toString()); - + String message = "{\"name\":\"aaa\"}"; + Student student = JSONObject.parseObject(message, Student.class); + System.out.println(JSONObject.toJSONString(student)); } - @Test - public void test2() { - logger.error("{} 日志解析过程出现异常" + FlowWriteConfig.KAFKA_TOPIC); - } } From a7e25af8756157372bac3e4ab5fe21fa3580fc61 Mon Sep 17 00:00:00 2001 From: qidaijie Date: Tue, 17 Dec 2019 10:42:06 +0800 Subject: [PATCH 15/27] =?UTF-8?q?=E4=BF=AE=E6=94=B9=20domain=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E6=96=B9=E5=BC=8F=E4=B8=BA=20=E4=BD=BF=E7=94=A8guava?= =?UTF-8?q?=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 7 ++ properties/service_flow_config.properties | 2 +- .../ac/iie/utils/general/TransFormUtils.java | 37 ++++---- .../cn/ac/iie/utils/hbase/HBaseUtils.java | 3 +- .../cn/ac/iie/utils/kafka/KafkaLogNtc.java | 2 +- src/test/java/cn/ac/iie/test/DomainUtils.java | 87 +++++++++++-------- .../java/cn/ac/iie/test/hbase/HBaseTest.java | 2 +- src/test/java/cn/ac/iie/test/test.java | 16 ++-- 8 files changed, 90 insertions(+), 66 deletions(-) diff --git a/pom.xml b/pom.xml index 44528db..df7d7ab 100644 --- a/pom.xml +++ b/pom.xml @@ -263,5 +263,12 @@ + + + com.google.guava + guava + 25.1-jre + + diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index 66f5690..c307cc7 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -74,4 +74,4 @@ influx.ip=http://192.168.40.151:8086 influx.username=admin #influx密码 -influx.password=admin +influx.password=admin \ No newline at end of file diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index 72003c0..ffed96b 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -7,7 +7,9 @@ import cn.ac.iie.bean.security.SecurityPolicyLog; import cn.ac.iie.common.FlowWriteConfig; import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.system.SnowflakeId; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; +import com.google.common.net.InternetDomainName; import com.zdjizhi.utils.IpLookup; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; @@ -28,10 +30,9 @@ import java.util.regex.Pattern; public class TransFormUtils { private static Logger logger = Logger.getLogger(TransFormUtils.class); - private final static Set PUBLIC_SUFFIX_SET = new HashSet( - Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se" - .split("\\|"))); - private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})"); + // private final static Set PUBLIC_SUFFIX_SET = new HashSet( +// Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se" +// .split("\\|"))); private static IpLookup ipLookup = new IpLookup.Builder(false) .loadDataFileV4(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") .loadDataFileV6(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") @@ -58,9 +59,10 @@ public class TransFormUtils { radiusSessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); radiusSessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); radiusSessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); + radiusSessionRecordLog.setCommon_subscriber_id(radiusSessionRecordLog.getRadius_account()); return JSONObject.toJSONString(radiusSessionRecordLog); } catch (Exception e) { - logger.error("{} 日志解析过程出现异常" + FlowWriteConfig.KAFKA_TOPIC); + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常"); e.printStackTrace(); return ""; } @@ -149,7 +151,8 @@ public class TransFormUtils { collectProtocolRecordLog.setHttp_domain(getTopDomain(collectProtocolRecordLog.getSsl_sni(), collectProtocolRecordLog.getHttp_host())); return JSONObject.toJSONString(collectProtocolRecordLog); } catch (Exception e) { - logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e); + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常"); + e.printStackTrace(); return ""; } } @@ -179,23 +182,13 @@ public class TransFormUtils { * @return 顶级域名 */ private static String getDomainName(String host) { - if (host.endsWith(FlowWriteConfig.DOMAIN_SPLITTER)) { - host = host.substring(0, host.length() - 1); + String domain = ""; + try { + domain = InternetDomainName.from(host).topPrivateDomain().toString(); + } catch (Exception e) { + logger.error("host解析顶级域名异常: " + e.getMessage()); } - if (IP_PATTERN.matcher(host).matches()) { - return host; - } - int index = 0; - String candidate = host; - for (; index >= 0; ) { - index = candidate.indexOf(FlowWriteConfig.DOMAIN_SPLITTER); - String subCandidate = candidate.substring(index + 1); - if (PUBLIC_SUFFIX_SET.contains(subCandidate)) { - return candidate; - } - candidate = subCandidate; - } - return candidate; + return domain; } diff --git a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java index 2b60e43..dfcf3b8 100644 --- a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java +++ b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java @@ -26,7 +26,7 @@ import java.util.concurrent.ConcurrentSkipListMap; public class HBaseUtils { private final static Logger logger = Logger.getLogger(HBaseUtils.class); private static Map subIdMap = new HashMap<>(333334); -// private static Map subIdMap = new ConcurrentSkipListMap<>(); + // private static Map subIdMap = new ConcurrentSkipListMap<>(); private static Connection connection; private static Long time; @@ -137,4 +137,5 @@ public class HBaseUtils { public static String getAccount(String clientIp) { return subIdMap.get(clientIp); } + } diff --git a/src/main/java/cn/ac/iie/utils/kafka/KafkaLogNtc.java b/src/main/java/cn/ac/iie/utils/kafka/KafkaLogNtc.java index 1bf76eb..78deae0 100644 --- a/src/main/java/cn/ac/iie/utils/kafka/KafkaLogNtc.java +++ b/src/main/java/cn/ac/iie/utils/kafka/KafkaLogNtc.java @@ -72,7 +72,7 @@ public class KafkaLogNtc { properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); properties.put("acks", "1"); properties.put("linger.ms", "2"); - properties.put("request.timeout.ms", 60000); + properties.put("request.timeout.ms", 30000); properties.put("batch.size", 262144); properties.put("buffer.memory", 33554432); // properties.put("compression.type", "snappy"); diff --git a/src/test/java/cn/ac/iie/test/DomainUtils.java b/src/test/java/cn/ac/iie/test/DomainUtils.java index ec2cce4..3a0a427 100644 --- a/src/test/java/cn/ac/iie/test/DomainUtils.java +++ b/src/test/java/cn/ac/iie/test/DomainUtils.java @@ -1,67 +1,50 @@ package cn.ac.iie.test; +import com.google.common.net.InternetDomainName; import com.zdjizhi.utils.StringUtil; import javax.xml.bind.SchemaOutputResolver; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; public class DomainUtils { - - private static Pattern pattern = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.me|\\.cc|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)"); - private final static Set PublicSuffixSet = new HashSet( - Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se" + Arrays.asList("com|edu|gov|int|mil|net|org|biz|info|pro|name|museum|coop|aero|xxx|idv|top|xyz|xin|vip|win|red|wang|co|mobi|travel|club|post|rec|asia" .split("\\|"))); - private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})"); public static void main(String[] args) { - System.out.println(getTopDomain("http://www.ccb.com")); - System.out.println(getDomainName("www.comaa.com")); +// InternetDomainName.from("foo.co.uk").topPrivateDomain().toString(); + String host = "www.aaa.co.uk"; +// if (host.contains(":")){ +// String s = host.split(":")[0]; +// System.out.println(InternetDomainName.from(s)); +// System.out.println(InternetDomainName.from(s).topPrivateDomain()); +// }else { + System.out.println(InternetDomainName.from(host).topDomainUnderRegistrySuffix()); + System.out.println(InternetDomainName.from(host).topPrivateDomain()); +// } +// System.out.println(InternetDomainName.from("shence.hupu.com").topPrivateDomain()); } - private static String getTopDomain(String url) { - try { -// 获取值转换为小写 - String host = new URL(url).getHost().toLowerCase();//news.hexun.com -// Pattern pattern = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.cc|\\.me|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)"); - Matcher matcher = IP_PATTERN.matcher(host); - if (matcher.find()){ - return matcher.group(); - } - } catch (MalformedURLException e) { - e.printStackTrace(); - } - return null; - } - - -// private final static Set PublicSuffixSet = new HashSet( -// Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se" -// .split("\\|"))); -// private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})"); - /** * 获取url的顶级域名 -// * @param url + * // * @param url + * * @return */ public static String getDomainName(String host) { -// String host = url.getHost(); - if (host.endsWith(".")){ + if (host.endsWith(".") || host.contains("/")) { host = host.substring(0, host.length() - 1); } - if (IP_PATTERN.matcher(host).matches()){ - return host; - } int index = 0; String candidate = host; - for (; index >= 0;) { + for (; index >= 0; ) { index = candidate.indexOf('.'); String subCandidate = candidate.substring(index + 1); if (PublicSuffixSet.contains(subCandidate)) { @@ -73,4 +56,38 @@ public class DomainUtils { } + public static String getSourceDomain(String host) { + if (host.endsWith(".")) { + host = host.substring(0, host.length() - 1); + } + String[] hostStr = host.split("\\."); + int length = hostStr.length; + if (hostStr.length >= 2) { + if (PublicSuffixSet.contains(hostStr[length - 2])) { + return hostStr[length - 3] + "." + hostStr[length - 2] + "." + hostStr[length - 1]; + } else { + return hostStr[length - 2] + "." + hostStr[length - 1]; + } + } else { + return host; + } + } + + + // 定义正则表达式，域名的根需要自定义，这里不全 + private static final String RE_TOP = "[\\w-]+\\.(com.cn|net.cn|gov.cn|org\\.nz|org.cn|com|net|org|gov|cc|biz|info|cn|co)\\b()*"; + private static Pattern pattern = Pattern.compile(RE_TOP, Pattern.CASE_INSENSITIVE); + + public static String getTopDomain(String url) { + String result = url; + try { + Matcher matcher = pattern.matcher(url); + matcher.find(); + result = matcher.group(); + } catch (Exception e) { + System.out.println("[getTopDomain ERROR]====>"); + e.printStackTrace(); + } + return result; + } } diff --git a/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java index a6fb619..26d9d09 100644 --- a/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java +++ b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java @@ -32,7 +32,7 @@ public class HBaseTest { // 设置zookeeper节点 // configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); // configuration.set("hbase.zookeeper.quorum", "192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181"); - configuration.set("hbase.zookeeper.quorum", "192.168.40.224:2182"); + configuration.set("hbase.zookeeper.quorum", "192.168.40.186:2182"); configuration.set("hbase.client.retries.number", "3"); configuration.set("hbase.bulkload.retries.number", "3"); configuration.set("zookeeper.recovery.retry", "3"); diff --git a/src/test/java/cn/ac/iie/test/test.java b/src/test/java/cn/ac/iie/test/test.java index aa5b7db..e361e31 100644 --- a/src/test/java/cn/ac/iie/test/test.java +++ b/src/test/java/cn/ac/iie/test/test.java @@ -1,16 +1,22 @@ package cn.ac.iie.test; -import cn.ac.iie.test.bean.Student; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; +import com.google.common.net.InternetDomainName; import org.apache.log4j.Logger; public class test { private static Logger logger = Logger.getLogger(test.class); public static void main(String[] args) { - String message = "{\"name\":\"aaa\"}"; - Student student = JSONObject.parseObject(message, Student.class); - System.out.println(JSONObject.toJSONString(student)); - } + String message = "{\"common_schema_type\":\"SSL\",\"ssl_sni\":\"www.myssl.cn\",\"common_server_ip\":\"101.37.81.250\",\"common_client_ip\":\"192.168.50.12\",\"common_server_port\":443,\"common_client_port\":51398,\"common_stream_dir\":3,\"common_address_type\":4,\"common_s2c_pkt_num\":1,\"common_s2c_byte_num\":0,\"common_c2s_pkt_num\":3,\"common_c2s_byte_num\":517,\"common_stream_trace_id\":7629423224647,\"common_l4_protocol\":\"IPv4_TCP\",\"common_address_list\":\"51398-443-192.168.50.12-101.37.81.250\",\"common_sled_ip\":\"192.168.40.161\",\"common_start_time\":1575008799,\"common_end_time\":1575008799,\"common_con_duration_ms\":0,\"common_policy_id\":282,\"common_service\":0,\"common_action\":128,\"common_user_region\":\"{\\\"protocol\\\":\\\"SSL\\\"}\"}"; + JSONObject jsonObject = JSON.parseObject(message); + String common_server_ip = jsonObject.getString("common_server_ip"); + String common_client_ip = jsonObject.getString("common_client_ip"); + jsonObject.put("common__recv_time",(System.currentTimeMillis() / 1000)); + + System.out.println(jsonObject.toString()); + + } } From 98a4a66a5f9e62cfb662b86d06ef724f1ea03b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Tue, 17 Dec 2019 14:31:30 +0800 Subject: [PATCH 16/27] =?UTF-8?q?=E5=8A=A8=E6=80=81=E8=8E=B7=E5=8F=96schem?= =?UTF-8?q?a=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 14 +- properties/service_flow_config.properties | 25 +- .../bolt/security/SecurityCompletionBolt.java | 3 +- .../cn/ac/iie/common/FlowWriteConfig.java | 5 +- .../utils/general/schema/TransFormUtils.java | 227 ++++++++++++++++++ .../cn/ac/iie/utils/hbase/HBaseUtils.java | 1 + .../cn/ac/iie/utils/http/HttpClientUtil.java | 51 ++++ .../cn/ac/iie/utils/json/JsonParseUtil.java | 180 ++++++++++++++ src/test/java/cn/ac/iie/test/DomainUtils.java | 2 +- src/test/java/cn/ac/iie/test/UtilTest.java | 27 +++ 10 files changed, 515 insertions(+), 20 deletions(-) create mode 100644 src/main/java/cn/ac/iie/utils/general/schema/TransFormUtils.java create mode 100644 src/main/java/cn/ac/iie/utils/http/HttpClientUtil.java create mode 100644 src/main/java/cn/ac/iie/utils/json/JsonParseUtil.java create mode 100644 src/test/java/cn/ac/iie/test/UtilTest.java diff --git a/pom.xml b/pom.xml index df7d7ab..83ac80e 100644 --- a/pom.xml +++ b/pom.xml @@ -144,6 +144,11 @@ fastjson 1.2.59 + + cglib + cglib-nodep + 3.2.4 + com.zdjizhi @@ -262,12 +267,11 @@ - - - com.google.guava - guava - 25.1-jre + org.junit.jupiter + junit-jupiter-api + 5.3.2 + compile diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index c307cc7..1e1bf00 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -1,6 +1,6 @@ #管理kafka地址 -#bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092 -bootstrap.servers=192.168.40.186:9092 +bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092 +#bootstrap.servers=192.168.40.186:9092 #zookeeper 地址 zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 @@ -16,33 +16,34 @@ hbase.table.name=subscriber_info auto.offset.reset=latest #kafka broker下的topic名称 -kafka.topic=CONNECTION-RECORD-LOG +kafka.topic=SECURITY-EVENT-LOG #读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； -group.id=security-policy-191114 +group.id=security-policy-191216 #输出topic -results.output.topic=CONNECTION-RECORD-COMPLETED-LOG +results.output.topic=SECURITY-EVENT-COMPLETED-LOG #storm topology workers -topology.workers=3 +topology.workers=1 #spout并行度 建议与kafka分区数相同 -spout.parallelism=6 +spout.parallelism=3 #处理补全操作的bolt并行度-worker的倍数 -datacenter.bolt.parallelism=12 +datacenter.bolt.parallelism=1 #写入kafka的并行度10 -kafka.bolt.parallelism=12 +kafka.bolt.parallelism=3 #定位库地址 -ip.library=/home/ceiec/topology/dat/ +#ip.library=/home/ceiec/topology/dat/ #ip.library=D:\\workerSpace\\K18-Phase2\\3.0.2019115\\log-stream-completion\\ - +ip.library=D:\\dat\\ #kafka批量条数 batch.insert.num=2000 - +#网关的schema位置 +schema.http=http://192.168.40.224:9999/metadata/schema/v1/fields/security_event_log #数据中心（UID） data.center.id.num=15 diff --git a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java index ca52c03..76bc8af 100644 --- a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.Map; import static cn.ac.iie.utils.general.TransFormUtils.getSecurityMessage; +import static cn.ac.iie.utils.general.schema.TransFormUtils.dealCommonMessage; /** * 通联关系日志补全 @@ -42,7 +43,7 @@ public class SecurityCompletionBolt extends BaseBasicBolt { } else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { - basicOutputCollector.emit(new Values(getSecurityMessage(message))); + basicOutputCollector.emit(new Values(dealCommonMessage(message))); } } } catch (Exception e) { diff --git a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java index f16e0da..17f03da 100644 --- a/src/main/java/cn/ac/iie/common/FlowWriteConfig.java +++ b/src/main/java/cn/ac/iie/common/FlowWriteConfig.java @@ -49,6 +49,9 @@ public class FlowWriteConfig { public static final String IP_LIBRARY = FlowWriteConfigurations.getStringProperty(0, "ip.library"); - + /** + * http + */ + public static final String SCHEMA_HTTP = FlowWriteConfigurations.getStringProperty(0, "schema.http"); } \ No newline at end of file diff --git a/src/main/java/cn/ac/iie/utils/general/schema/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/schema/TransFormUtils.java new file mode 100644 index 0000000..07f8f01 --- /dev/null +++ b/src/main/java/cn/ac/iie/utils/general/schema/TransFormUtils.java @@ -0,0 +1,227 @@ +package cn.ac.iie.utils.general.schema; + + +import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.hbase.HBaseUtils; +import cn.ac.iie.utils.json.JsonParseUtil; +import cn.ac.iie.utils.system.SnowflakeId; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.google.common.net.InternetDomainName; +import com.zdjizhi.utils.IpLookup; +import com.zdjizhi.utils.StringUtil; +import org.apache.log4j.Logger; + +import java.util.*; +import java.util.regex.Pattern; + + +/** + * 描述:转换或补全工具类 + * + * @author qidaijie + * @create 2018-08-13 15:11 + */ + +public class TransFormUtils { + private static Logger logger = Logger.getLogger(TransFormUtils.class); + private final static Set PUBLIC_SUFFIX_SET = new HashSet( + Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se" + .split("\\|"))); + private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})"); + private static IpLookup ipLookup = new IpLookup.Builder(false) + .loadDataFileV4(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") + .loadDataFileV6(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") + .loadAsnDataFileV4(FlowWriteConfig.IP_LIBRARY + "asn_v4.mmdb") + .loadAsnDataFileV6(FlowWriteConfig.IP_LIBRARY + "asn_v6.mmdb") + .build(); + + //在内存中加载反射类用的map + private static HashMap map = JsonParseUtil.getMapFromhttp(FlowWriteConfig.SCHEMA_HTTP); + //反射成一个类 + private static Object mapObject = JsonParseUtil.generateObject(map); + //获取任务列表 + private static ArrayList jobList = JsonParseUtil.getJobListFromHttp(FlowWriteConfig.SCHEMA_HTTP); + + + /** + * 解析日志，并补全 + * 补domain,补subscriber_id + * + * @param message Security原始日志 + * @return 补全后的日志 + *

+ * current_timestamp + * snowflake_id + * geo_ip_detail + * geo_asn + * radius_match + * geo_ip_country + * geo_asn + * sub_domain + * sub_domain + */ + public static String dealCommonMessage(String message) { + + Object object = JSONObject.parseObject(message, mapObject.getClass()); +// System.out.println("补全之前 ===》 "+JSON.toJSONString(object)); + try { + for (String[] strings : jobList) { + + if (strings[2].equals("current_timestamp")) { + JsonParseUtil.setValue(object, strings[1], getCurrentTime()); + } else if (strings[2].equals("snowflake_id")) { + JsonParseUtil.setValue(object, strings[1], getSnowflakeId()); + } else if (strings[2].equals("geo_ip_detail")) { + JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(JsonParseUtil.getValue(object, strings[0]).toString())); + } else if (strings[2].equals("geo_asn")) { + JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(JsonParseUtil.getValue(object, strings[0]).toString())); + } else if (strings[2].equals("radius_match")) { + JsonParseUtil.setValue(object,strings[1],HBaseUtils.getAccount(JsonParseUtil.getValue(object,strings[0]).toString())); +// JsonParseUtil.setValue(object, strings[1], "aaaaaaaaa"); + } else if (strings[2].equals("geo_ip_country")) { + JsonParseUtil.setValue(object, strings[1], getGeoIpCountry(JsonParseUtil.getValue(object, strings[0]).toString())); + } else if (strings[0].equals("http_host") && strings[2].equals("sub_domain")) { + JsonParseUtil.setValue(object,strings[1],getTopDomain(null,JsonParseUtil.getValue(object,strings[0]).toString())); + } else if (strings[0].equals("ssl_sni") && strings[2].equals("sub_domain")) { + if (StringUtil.isBlank(JsonParseUtil.getValue(object, strings[1]).toString())) { + JsonParseUtil.setValue(object,strings[1],getTopDomain(JsonParseUtil.getValue(object,strings[0]).toString(),null)); + } + + } + } + + + return JSONObject.toJSONString(object); +// System.out.println("补全之后 ===》 "+JSON.toJSONString(object)); + + } catch (Exception e) { + logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常"); + e.printStackTrace(); + return ""; + } + } + + + /** + * 有sni通过sni获取域名，有host根据host获取域名 + * + * @param sni sni + * @param host host + * @return 顶级域名 + */ + private static String getTopDomain(String sni, String host) { + if (StringUtil.isNotBlank(host)) { + return getDomainName(host); + } else if (StringUtil.isNotBlank(sni)) { + return getDomainName(sni); + } else { + return ""; + } + } + + + /** + * 根据url截取顶级域名 + * + * @param host 网站url + * @return 顶级域名 + */ + private static String getDomainName(String host) { + String domain = ""; + try { + domain = InternetDomainName.from(host).topPrivateDomain().toString(); + } catch (Exception e) { + logger.error("host解析顶级域名异常: " + e.getMessage()); + } + return domain; + } + + /** + * 生成当前时间戳的操作 + */ + private static long getCurrentTime() { + return (System.currentTimeMillis() / 1000); + } + + /** + * 雪花模型生成id + * + * @return + */ + private static long getSnowflakeId() { + + return SnowflakeId.generateId(); + } + + /** + * 根据clientIp获取location信息 + * + * @param ip + * @return + */ + private static String getGeoIpDetail(String ip) { + + return ipLookup.cityLookupDetail(ip); + } + + /** + * 根据ip获取asn信息 + * + * @param ip + * @return + */ + private static String getGeoAsn(String ip) { + + return ipLookup.asnLookup(ip, true); + } + + /** + * 根据ip获取country信息 + * + * @param ip + * @return + */ + private static String getGeoIpCountry(String ip) { + + return ipLookup.countryLookup(ip); + } + + /** + * radius借助hbase补齐 + * + * @param ip + * @return + */ + private static String radiusMatch(String ip) { + return HBaseUtils.getAccount(ip); + } + + /** + * switch 匹配合适的方法 + * current_timestamp + * snowflake_id + * geo_ip_detail + * geo_asn + * radius_match + * geo_ip_country + * geo_asn + * sub_domain + * sub_domain + * @param func + */ +//TODO 行不通的原因是无法统一一个确定的返回值类型 + /* private static String switchFunc(String func){ + switch (func){ + case "current_timestamp": + return String.valueOf(getCurrentTime()); + case "snowflake_id": + case "geo_ip_detail": + case "geo_asn": + case "radius_match": + case "geo_ip_country": + case "sub_domain": + } + return func; + }*/ +} diff --git a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java index dfcf3b8..f386003 100644 --- a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java +++ b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentSkipListMap; + /** * HBase 工具类 * diff --git a/src/main/java/cn/ac/iie/utils/http/HttpClientUtil.java b/src/main/java/cn/ac/iie/utils/http/HttpClientUtil.java new file mode 100644 index 0000000..2aa8885 --- /dev/null +++ b/src/main/java/cn/ac/iie/utils/http/HttpClientUtil.java @@ -0,0 +1,51 @@ +package cn.ac.iie.utils.http; + +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; + +/** + * 获取网关schema的工具类 + */ +public class HttpClientUtil { + public static String requestByGetMethod(String s) { + CloseableHttpClient httpClient = HttpClients.createDefault(); + StringBuilder entityStringBuilder = null; + try { + HttpGet get = new HttpGet(s); + CloseableHttpResponse httpResponse = null; + httpResponse = httpClient.execute(get); + try { + HttpEntity entity = httpResponse.getEntity(); + entityStringBuilder = new StringBuilder(); + if (null != entity) { + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(httpResponse.getEntity().getContent(), "UTF-8"), 8 * 1024); + String line = null; + while ((line = bufferedReader.readLine()) != null) { + entityStringBuilder.append(line); + } + } + } finally { + httpResponse.close(); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + try { + if (httpClient != null) { + httpClient.close(); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + return entityStringBuilder.toString(); + } + +} diff --git a/src/main/java/cn/ac/iie/utils/json/JsonParseUtil.java b/src/main/java/cn/ac/iie/utils/json/JsonParseUtil.java new file mode 100644 index 0000000..ec8b7f5 --- /dev/null +++ b/src/main/java/cn/ac/iie/utils/json/JsonParseUtil.java @@ -0,0 +1,180 @@ +package cn.ac.iie.utils.json; + +import cn.ac.iie.utils.http.HttpClientUtil; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import net.sf.cglib.beans.BeanGenerator; +import net.sf.cglib.beans.BeanMap; + +import java.util.*; + +/** + * 使用fastjson解析json的工具类 + */ +public class JsonParseUtil { + + /** + * 模式匹配，给定一个类型字符串返回一个类类型 + * + * @param type + * @return + */ + + public static Class getClassName(String type) { + Class clazz = int.class; + + switch (type) { + case "int": + clazz = int.class; + break; + case "double": + clazz = double.class; + break; + case "float": + clazz = float.class; + break; + case "long": + clazz = long.class; + break; + case "char": + clazz = char.class; + break; + case "byte": + clazz = byte.class; + break; + case "boolean": + clazz = boolean.class; + break; + case "short": + clazz = short.class; + break; + default: + clazz = String.class; + } + return clazz; + } + + /** + * 根据反射生成对象的方法 + * + * @param properties + * @return 生成的Object类型的对象 + */ + public static Object generateObject(Map properties) { + BeanGenerator generator = new BeanGenerator(); + Set keySet = properties.keySet(); + for (Iterator i = keySet.iterator(); i.hasNext(); ) { + String key = (String) i.next(); + generator.addProperty(key, (Class) properties.get(key)); + } + return generator.create(); + } + + /** + * 获取属性值的方法 + * + * @param obj + * @param property + * @return 属性的值 + */ + public static Object getValue(Object obj, String property) { + BeanMap beanMap = BeanMap.create(obj); + return beanMap.get(property); + } + + /** + * 更新属性值的方法 + * + * @param obj + * @param property + * @param value + */ + public static void setValue(Object obj, String property, Object value) { + BeanMap beanMap = BeanMap.create(obj); + beanMap.put(property, value); + } + + /** + * 通过获取String类型的网关schema链接来获取map，用于生成一个Object类型的对象 + * + * @param http + * @return 用于反射生成schema类型的对象的一个map集合 + */ + public static HashMap getMapFromhttp(String http) { + HashMap map = new HashMap<>(); + + String schema = HttpClientUtil.requestByGetMethod(http); + Object data = JSON.parseObject(schema).get("data"); + + //获取fields，并转化为数组，数组的每个元素都是一个name doc type + JSONObject schemaJson = JSON.parseObject(data.toString()); + JSONArray fields = (JSONArray) schemaJson.get("fields"); + + for (Object field : fields) { + String name = JSON.parseObject(field.toString()).get("name").toString(); + String type = JSON.parseObject(field.toString()).get("type").toString(); + + //组合用来生成实体类的map + + map.put(name, getClassName(type)); + + } + return map; + } + + + /** + * 根据http链接获取schema，解析之后返回一个任务列表 (useList toList funcList) + * @param http + * @return + */ + public static ArrayList getJobListFromHttp(String http) { + ArrayList list = new ArrayList<>(); + + String schema = HttpClientUtil.requestByGetMethod(http); + //解析data + Object data = JSON.parseObject(schema).get("data"); + + //获取fields，并转化为数组，数组的每个元素都是一个name doc type + JSONObject schemaJson = JSON.parseObject(data.toString()); + JSONArray fields = (JSONArray) schemaJson.get("fields"); + + for (Object field : fields) { + Object doc = JSON.parseObject(field.toString()).get("doc"); + String name = JSON.parseObject(field.toString()).get("name").toString(); + + if (doc != null) { + Object format = JSON.parseObject(doc.toString()).get("format"); + + if (format != null) { + Object functions = JSON.parseObject(format.toString()).get("functions"); + Object appendTo = JSON.parseObject(format.toString()).get("appendTo"); + + if (appendTo != null) { + String[] functionArray = functions.toString().split(","); + String[] appendToArray = appendTo.toString().split(","); + + for (int i = 0; i < functionArray.length; i++) { +// useList.add(name); +// toList.add(appendToArray[i]); +// funcList.add(functionArray[i]); + list.add(new String[]{name, appendToArray[i], functionArray[i]}); + + } + } else { +// useList.add(name); +// funcList.add(functions.toString()); +// toList.add(name); + list.add(new String[]{name, name, functions.toString()}); + } + + } + } + + } + return list; + } + + +} \ No newline at end of file diff --git a/src/test/java/cn/ac/iie/test/DomainUtils.java b/src/test/java/cn/ac/iie/test/DomainUtils.java index 3a0a427..a693047 100644 --- a/src/test/java/cn/ac/iie/test/DomainUtils.java +++ b/src/test/java/cn/ac/iie/test/DomainUtils.java @@ -25,7 +25,7 @@ public class DomainUtils { // System.out.println(InternetDomainName.from(s)); // System.out.println(InternetDomainName.from(s).topPrivateDomain()); // }else { - System.out.println(InternetDomainName.from(host).topDomainUnderRegistrySuffix()); +// System.out.println(InternetDomainName.from(host).topDomainUnderRegistrySuffix()); System.out.println(InternetDomainName.from(host).topPrivateDomain()); // } diff --git a/src/test/java/cn/ac/iie/test/UtilTest.java b/src/test/java/cn/ac/iie/test/UtilTest.java new file mode 100644 index 0000000..c03b75d --- /dev/null +++ b/src/test/java/cn/ac/iie/test/UtilTest.java @@ -0,0 +1,27 @@ +package cn.ac.iie.test; + +import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.json.JsonParseUtil; + +import java.util.ArrayList; +import java.util.HashMap; + +public class UtilTest { + + + private static HashMap map = JsonParseUtil.getMapFromhttp(FlowWriteConfig.SCHEMA_HTTP); + //反射成一个类 + private static Object mapObject = JsonParseUtil.generateObject(map); + //获取任务列表 + private static ArrayList jobList = JsonParseUtil.getJobListFromHttp(FlowWriteConfig.SCHEMA_HTTP); + + public static void main(String[] args) { + + + for (String[] strings : jobList) { + System.out.println(strings[0]); + System.out.println(strings[1]); + System.out.println(strings[2]); + } + } +} From 00b8f6d2d2073e27f7272aad69916bd0eba4320c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Tue, 17 Dec 2019 14:42:30 +0800 Subject: [PATCH 17/27] =?UTF-8?q?=E5=8A=A8=E6=80=81=E8=8E=B7=E5=8F=96schem?= =?UTF-8?q?a=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ac/iie/bean/PublicSessionRecordLog.java | 395 ---------- .../bean/connection/ConnectionRecordLog.java | 686 ------------------ .../iie/bean/proxy/ProxySessionRecordLog.java | 203 ------ .../bean/radius/RadiusSessionRecordLog.java | 89 --- .../iie/bean/security/SecurityPolicyLog.java | 682 ----------------- .../bolt/collect/CollectCompletedBolt.java | 10 +- .../iie/bolt/proxy/ProxyCompletionBolt.java | 4 +- .../iie/bolt/radius/RadiusCompletionBolt.java | 4 +- .../bolt/security/SecurityCompletionBolt.java | 4 +- .../ac/iie/utils/general/TransFormUtils.java | 255 ++++--- .../utils/general/schema/TransFormUtils.java | 227 ------ 11 files changed, 155 insertions(+), 2404 deletions(-) delete mode 100644 src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java delete mode 100644 src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java delete mode 100644 src/main/java/cn/ac/iie/bean/proxy/ProxySessionRecordLog.java delete mode 100644 src/main/java/cn/ac/iie/bean/radius/RadiusSessionRecordLog.java delete mode 100644 src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java delete mode 100644 src/main/java/cn/ac/iie/utils/general/schema/TransFormUtils.java diff --git a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java deleted file mode 100644 index 123ccff..0000000 --- a/src/main/java/cn/ac/iie/bean/PublicSessionRecordLog.java +++ /dev/null @@ -1,395 +0,0 @@ -package cn.ac.iie.bean; - - -/** - * 公共类 - * - * @author qidaijie - */ -public class PublicSessionRecordLog { - -//TODO 基础属性 41 int22 - - private int common_service; - private int common_direction; - private long common_recv_time; - private int common_address_type; - private int common_policy_id; - private int common_action; - private int common_client_port; - private int common_entrance_id; - private int common_link_id; - private int common_encapsulation; - private int common_server_port; - private int common_app_id; - private int common_protocol_id; - private int common_c2s_pkt_num; - private int common_s2c_pkt_num; - private long common_c2s_byte_num; - private long common_s2c_byte_num; - private int common_start_time; - private int common_end_time; - private int common_con_duration_ms; - private int common_stream_dir; - private int common_has_dup_traffic; - private long common_log_id; - - private String common_schema_type; - - private String common_l4_protocol; - private String common_user_tags; - private String common_user_region; - private String common_client_ip; - private String common_device_id; - private String common_sub_action; - private String common_isp; - private String common_sled_ip; - private String common_client_location; - private String common_client_asn; - private String common_subscriber_id; - private String common_server_ip; - private String common_server_location; - private String common_server_asn; - private String common_app_label; - private String common_address_list; - private String common_stream_error; - private long common_stream_trace_id; - - public int getCommon_service() { - return common_service; - } - - public void setCommon_service(int common_service) { - this.common_service = common_service; - } - - - public long getCommon_recv_time() { - return common_recv_time; - } - - public void setCommon_recv_time(long common_recv_time) { - this.common_recv_time = common_recv_time; - } - - public int getCommon_address_type() { - return common_address_type; - } - - public void setCommon_address_type(int common_address_type) { - this.common_address_type = common_address_type; - } - - public int getCommon_policy_id() { - return common_policy_id; - } - - public void setCommon_policy_id(int common_policy_id) { - this.common_policy_id = common_policy_id; - } - - public int getCommon_action() { - return common_action; - } - - public void setCommon_action(int common_action) { - this.common_action = common_action; - } - - public int getCommon_client_port() { - return common_client_port; - } - - public void setCommon_client_port(int common_client_port) { - this.common_client_port = common_client_port; - } - - public int getCommon_entrance_id() { - return common_entrance_id; - } - - public void setCommon_entrance_id(int common_entrance_id) { - this.common_entrance_id = common_entrance_id; - } - - public int getCommon_link_id() { - return common_link_id; - } - - public void setCommon_link_id(int common_link_id) { - this.common_link_id = common_link_id; - } - - - public int getCommon_server_port() { - return common_server_port; - } - - public void setCommon_server_port(int common_server_port) { - this.common_server_port = common_server_port; - } - - public int getCommon_app_id() { - return common_app_id; - } - - public void setCommon_app_id(int common_app_id) { - this.common_app_id = common_app_id; - } - - public int getCommon_protocol_id() { - return common_protocol_id; - } - - public void setCommon_protocol_id(int common_protocol_id) { - this.common_protocol_id = common_protocol_id; - } - - public int getCommon_c2s_pkt_num() { - return common_c2s_pkt_num; - } - - public void setCommon_c2s_pkt_num(int common_c2s_pkt_num) { - this.common_c2s_pkt_num = common_c2s_pkt_num; - } - - public int getCommon_s2c_pkt_num() { - return common_s2c_pkt_num; - } - - public void setCommon_s2c_pkt_num(int common_s2c_pkt_num) { - this.common_s2c_pkt_num = common_s2c_pkt_num; - } - - public long getCommon_c2s_byte_num() { - return common_c2s_byte_num; - } - - public void setCommon_c2s_byte_num(long common_c2s_byte_num) { - this.common_c2s_byte_num = common_c2s_byte_num; - } - - public long getCommon_s2c_byte_num() { - return common_s2c_byte_num; - } - - public void setCommon_s2c_byte_num(long common_s2c_byte_num) { - this.common_s2c_byte_num = common_s2c_byte_num; - } - - public int getCommon_start_time() { - return common_start_time; - } - - public void setCommon_start_time(int common_start_time) { - this.common_start_time = common_start_time; - } - - public int getCommon_end_time() { - return common_end_time; - } - - public void setCommon_end_time(int common_end_time) { - this.common_end_time = common_end_time; - } - - public int getCommon_con_duration_ms() { - return common_con_duration_ms; - } - - public void setCommon_con_duration_ms(int common_con_duration_ms) { - this.common_con_duration_ms = common_con_duration_ms; - } - - public int getCommon_stream_dir() { - return common_stream_dir; - } - - public void setCommon_stream_dir(int common_stream_dir) { - this.common_stream_dir = common_stream_dir; - } - - public int getCommon_direction() { - return common_direction; - } - - public void setCommon_direction(int common_direction) { - this.common_direction = common_direction; - } - - public int getCommon_encapsulation() { - return common_encapsulation; - } - - public void setCommon_encapsulation(int common_encapsulation) { - this.common_encapsulation = common_encapsulation; - } - - public int getCommon_has_dup_traffic() { - return common_has_dup_traffic; - } - - public void setCommon_has_dup_traffic(int common_has_dup_traffic) { - this.common_has_dup_traffic = common_has_dup_traffic; - } - - public String getCommon_schema_type() { - return common_schema_type; - } - - public void setCommon_schema_type(String common_schema_type) { - this.common_schema_type = common_schema_type; - } - - public long getCommon_log_id() { - return common_log_id; - } - - public void setCommon_log_id(long common_log_id) { - this.common_log_id = common_log_id; - } - - public String getCommon_l4_protocol() { - return common_l4_protocol; - } - - public void setCommon_l4_protocol(String common_l4_protocol) { - this.common_l4_protocol = common_l4_protocol; - } - - public String getCommon_user_tags() { - return common_user_tags; - } - - public void setCommon_user_tags(String common_user_tags) { - this.common_user_tags = common_user_tags; - } - - public String getCommon_user_region() { - return common_user_region; - } - - public void setCommon_user_region(String common_user_region) { - this.common_user_region = common_user_region; - } - - public String getCommon_client_ip() { - return common_client_ip; - } - - public void setCommon_client_ip(String common_client_ip) { - this.common_client_ip = common_client_ip; - } - - public String getCommon_device_id() { - return common_device_id; - } - - public void setCommon_device_id(String common_device_id) { - this.common_device_id = common_device_id; - } - - public String getCommon_isp() { - return common_isp; - } - - public void setCommon_isp(String common_isp) { - this.common_isp = common_isp; - } - - public String getCommon_sled_ip() { - return common_sled_ip; - } - - public void setCommon_sled_ip(String common_sled_ip) { - this.common_sled_ip = common_sled_ip; - } - - public String getCommon_client_location() { - return common_client_location; - } - - public void setCommon_client_location(String common_client_location) { - this.common_client_location = common_client_location; - } - - public String getCommon_client_asn() { - return common_client_asn; - } - - public void setCommon_client_asn(String common_client_asn) { - this.common_client_asn = common_client_asn; - } - - public String getCommon_subscriber_id() { - return common_subscriber_id; - } - - public void setCommon_subscriber_id(String common_subscriber_id) { - this.common_subscriber_id = common_subscriber_id; - } - - public String getCommon_server_ip() { - return common_server_ip; - } - - public void setCommon_server_ip(String common_server_ip) { - this.common_server_ip = common_server_ip; - } - - public String getCommon_server_location() { - return common_server_location; - } - - public void setCommon_server_location(String common_server_location) { - this.common_server_location = common_server_location; - } - - public String getCommon_server_asn() { - return common_server_asn; - } - - public void setCommon_server_asn(String common_server_asn) { - this.common_server_asn = common_server_asn; - } - - public String getCommon_app_label() { - return common_app_label; - } - - public void setCommon_app_label(String common_app_label) { - this.common_app_label = common_app_label; - } - - public String getCommon_address_list() { - return common_address_list; - } - - public void setCommon_address_list(String common_address_list) { - this.common_address_list = common_address_list; - } - - public String getCommon_stream_error() { - return common_stream_error; - } - - public void setCommon_stream_error(String common_stream_error) { - this.common_stream_error = common_stream_error; - } - - public long getCommon_stream_trace_id() { - return common_stream_trace_id; - } - - public void setCommon_stream_trace_id(long common_stream_trace_id) { - this.common_stream_trace_id = common_stream_trace_id; - } - - public String getCommon_sub_action() { - return common_sub_action; - } - - public void setCommon_sub_action(String common_sub_action) { - this.common_sub_action = common_sub_action; - } -} diff --git a/src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java b/src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java deleted file mode 100644 index 01d9ee4..0000000 --- a/src/main/java/cn/ac/iie/bean/connection/ConnectionRecordLog.java +++ /dev/null @@ -1,686 +0,0 @@ -package cn.ac.iie.bean.connection; - -import cn.ac.iie.bean.PublicSessionRecordLog; - -/** - * 除radius之外 - * - * @author qidaijie - */ -public class ConnectionRecordLog extends PublicSessionRecordLog { - - -//TODO HTTP协议属性 21 - - private String http_url; - private String http_host; - private String http_domain; - private String http_request_line; - private String http_response_line; - private String http_request_header; - private String http_response_header; - private String http_request_body; - private String http_response_body; - private String http_request_body_key; - private String http_response_body_key; - private int http_proxy_flag; - private int http_sequence; - private String http_snapshot; - private String http_cookie; - private String http_referer; - private String http_user_agent; - private String http_content_length; - private String http_content_type; - private String http_set_cookie; - private String http_version; - -//TODO MAIL协议属性 9 - - private String mail_protocol_type; - private String mail_sender; - private String mail_receiver; - private String mail_subject; - private String mail_content; - private String mail_attachment_name; - private String mail_attachment_content; - private String mail_eml_file; - private String mail_snapshot; - -//TODO 3DNS协议属性 18 - - private int dns_message_id; - private Integer dns_qr; - private Integer dns_opcode; - private int dns_aa; - private int dns_tc; - private int dns_rd; - private int dns_ra; - private int dns_rcode; - private int dns_qdcount; - private int dns_ancount; - private int dns_nscount; - private int dns_arcount; - private String dns_qname; - private int dns_qtype; - private int dns_qclass; - private String dns_cname; - private int dns_sub; - private String dns_rr; - -//TODO SSL协议属性 13 - - - private Integer ssl_pinningst; - private Integer ssl_intercept_state; - private int ssl_server_side_latency; - private int ssl_client_side_latency; - private Integer ssl_cert_verify; - private int ssl_con_latency_ms; - private String ssl_version; - private String ssl_sni; - private String ssl_san; - private String ssl_cn; - private String ssl_server_side_version; - private String ssl_client_side_version; - private String ssl_error; - - //TODO FTP协议属性 2 - private String ftp_url; - private String ftp_content; - - - //TODO BGP协议属性 3 - private int bgp_type; - private String bgp_as_num; - private String bgp_route; - - //TODO VOIP协议属性 4 - private String voip_calling_account; - private String voip_called_account; - private String voip_calling_number; - private String voip_called_number; - - //TODO STREAMING_MEDIA协议属性 2 - private String streaming_media_url; - private String streaming_media_protocol; - - - public String getHttp_url() { - return http_url; - } - - public void setHttp_url(String http_url) { - this.http_url = http_url; - } - - public String getHttp_host() { - return http_host; - } - - public void setHttp_host(String http_host) { - this.http_host = http_host; - } - - public String getHttp_domain() { - return http_domain; - } - - public void setHttp_domain(String http_domain) { - this.http_domain = http_domain; - } - - public String getHttp_request_line() { - return http_request_line; - } - - public void setHttp_request_line(String http_request_line) { - this.http_request_line = http_request_line; - } - - public String getHttp_response_line() { - return http_response_line; - } - - public void setHttp_response_line(String http_response_line) { - this.http_response_line = http_response_line; - } - - public String getHttp_request_header() { - return http_request_header; - } - - public void setHttp_request_header(String http_request_header) { - this.http_request_header = http_request_header; - } - - public String getHttp_response_header() { - return http_response_header; - } - - public void setHttp_response_header(String http_response_header) { - this.http_response_header = http_response_header; - } - - - public Integer getSsl_pinningst() { - return ssl_pinningst; - } - - public void setSsl_pinningst(Integer ssl_pinningst) { - this.ssl_pinningst = ssl_pinningst; - } - - public Integer getSsl_intercept_state() { - return ssl_intercept_state; - } - - public void setSsl_intercept_state(Integer ssl_intercept_state) { - this.ssl_intercept_state = ssl_intercept_state; - } - - public String getHttp_request_body() { - return http_request_body; - } - - public void setHttp_request_body(String http_request_body) { - this.http_request_body = http_request_body; - } - - public String getHttp_response_body() { - return http_response_body; - } - - public void setHttp_response_body(String http_response_body) { - this.http_response_body = http_response_body; - } - - public String getHttp_request_body_key() { - return http_request_body_key; - } - - public void setHttp_request_body_key(String http_request_body_key) { - this.http_request_body_key = http_request_body_key; - } - - public Integer getSsl_cert_verify() { - return ssl_cert_verify; - } - - public void setSsl_cert_verify(Integer ssl_cert_verify) { - this.ssl_cert_verify = ssl_cert_verify; - } - - public String getHttp_response_body_key() { - return http_response_body_key; - } - - public void setHttp_response_body_key(String http_response_body_key) { - this.http_response_body_key = http_response_body_key; - } - - public int getHttp_proxy_flag() { - return http_proxy_flag; - } - - public void setHttp_proxy_flag(int http_proxy_flag) { - this.http_proxy_flag = http_proxy_flag; - } - - public int getHttp_sequence() { - return http_sequence; - } - - public void setHttp_sequence(int http_sequence) { - this.http_sequence = http_sequence; - } - - public String getHttp_snapshot() { - return http_snapshot; - } - - public void setHttp_snapshot(String http_snapshot) { - this.http_snapshot = http_snapshot; - } - - public String getHttp_cookie() { - return http_cookie; - } - - public void setHttp_cookie(String http_cookie) { - this.http_cookie = http_cookie; - } - - public String getHttp_referer() { - return http_referer; - } - - public void setHttp_referer(String http_referer) { - this.http_referer = http_referer; - } - - public String getHttp_user_agent() { - return http_user_agent; - } - - public void setHttp_user_agent(String http_user_agent) { - this.http_user_agent = http_user_agent; - } - - public String getHttp_content_length() { - return http_content_length; - } - - public void setHttp_content_length(String http_content_length) { - this.http_content_length = http_content_length; - } - - public String getHttp_content_type() { - return http_content_type; - } - - public void setHttp_content_type(String http_content_type) { - this.http_content_type = http_content_type; - } - - public String getHttp_set_cookie() { - return http_set_cookie; - } - - public void setHttp_set_cookie(String http_set_cookie) { - this.http_set_cookie = http_set_cookie; - } - - public String getHttp_version() { - return http_version; - } - - public void setHttp_version(String http_version) { - this.http_version = http_version; - } - - public String getMail_protocol_type() { - return mail_protocol_type; - } - - public void setMail_protocol_type(String mail_protocol_type) { - this.mail_protocol_type = mail_protocol_type; - } - - public String getMail_sender() { - return mail_sender; - } - - public void setMail_sender(String mail_sender) { - this.mail_sender = mail_sender; - } - - public String getMail_receiver() { - return mail_receiver; - } - - public void setMail_receiver(String mail_receiver) { - this.mail_receiver = mail_receiver; - } - - public String getMail_subject() { - return mail_subject; - } - - public void setMail_subject(String mail_subject) { - this.mail_subject = mail_subject; - } - - public String getMail_content() { - return mail_content; - } - - public void setMail_content(String mail_content) { - this.mail_content = mail_content; - } - - public String getMail_attachment_name() { - return mail_attachment_name; - } - - public void setMail_attachment_name(String mail_attachment_name) { - this.mail_attachment_name = mail_attachment_name; - } - - public String getMail_attachment_content() { - return mail_attachment_content; - } - - public void setMail_attachment_content(String mail_attachment_content) { - this.mail_attachment_content = mail_attachment_content; - } - - public String getMail_eml_file() { - return mail_eml_file; - } - - public void setMail_eml_file(String mail_eml_file) { - this.mail_eml_file = mail_eml_file; - } - - public String getMail_snapshot() { - return mail_snapshot; - } - - public void setMail_snapshot(String mail_snapshot) { - this.mail_snapshot = mail_snapshot; - } - - public int getDns_message_id() { - return dns_message_id; - } - - public void setDns_message_id(int dns_message_id) { - this.dns_message_id = dns_message_id; - } - - public Integer getDns_qr() { - return dns_qr; - } - - public void setDns_qr(Integer dns_qr) { - this.dns_qr = dns_qr; - } - - public Integer getDns_opcode() { - return dns_opcode; - } - - public void setDns_opcode(Integer dns_opcode) { - this.dns_opcode = dns_opcode; - } - - public int getDns_aa() { - return dns_aa; - } - - public void setDns_aa(int dns_aa) { - this.dns_aa = dns_aa; - } - - public int getDns_tc() { - return dns_tc; - } - - public void setDns_tc(int dns_tc) { - this.dns_tc = dns_tc; - } - - public int getDns_rd() { - return dns_rd; - } - - public void setDns_rd(int dns_rd) { - this.dns_rd = dns_rd; - } - - public int getDns_ra() { - return dns_ra; - } - - public void setDns_ra(int dns_ra) { - this.dns_ra = dns_ra; - } - - public int getDns_rcode() { - return dns_rcode; - } - - public void setDns_rcode(int dns_rcode) { - this.dns_rcode = dns_rcode; - } - - public int getDns_qdcount() { - return dns_qdcount; - } - - public void setDns_qdcount(int dns_qdcount) { - this.dns_qdcount = dns_qdcount; - } - - public int getDns_ancount() { - return dns_ancount; - } - - public void setDns_ancount(int dns_ancount) { - this.dns_ancount = dns_ancount; - } - - public int getDns_nscount() { - return dns_nscount; - } - - public void setDns_nscount(int dns_nscount) { - this.dns_nscount = dns_nscount; - } - - public int getDns_arcount() { - return dns_arcount; - } - - public void setDns_arcount(int dns_arcount) { - this.dns_arcount = dns_arcount; - } - - public String getDns_qname() { - return dns_qname; - } - - public void setDns_qname(String dns_qname) { - this.dns_qname = dns_qname; - } - - public int getDns_qtype() { - return dns_qtype; - } - - public void setDns_qtype(int dns_qtype) { - this.dns_qtype = dns_qtype; - } - - public int getDns_qclass() { - return dns_qclass; - } - - public void setDns_qclass(int dns_qclass) { - this.dns_qclass = dns_qclass; - } - - public String getDns_cname() { - return dns_cname; - } - - public void setDns_cname(String dns_cname) { - this.dns_cname = dns_cname; - } - - public int getDns_sub() { - return dns_sub; - } - - public void setDns_sub(int dns_sub) { - this.dns_sub = dns_sub; - } - - public String getDns_rr() { - return dns_rr; - } - - public void setDns_rr(String dns_rr) { - this.dns_rr = dns_rr; - } - - - public int getSsl_server_side_latency() { - return ssl_server_side_latency; - } - - public void setSsl_server_side_latency(int ssl_server_side_latency) { - this.ssl_server_side_latency = ssl_server_side_latency; - } - - public int getSsl_client_side_latency() { - return ssl_client_side_latency; - } - - public void setSsl_client_side_latency(int ssl_client_side_latency) { - this.ssl_client_side_latency = ssl_client_side_latency; - } - - public int getSsl_con_latency_ms() { - return ssl_con_latency_ms; - } - - public void setSsl_con_latency_ms(int ssl_con_latency_ms) { - this.ssl_con_latency_ms = ssl_con_latency_ms; - } - - public String getSsl_version() { - return ssl_version; - } - - public void setSsl_version(String ssl_version) { - this.ssl_version = ssl_version; - } - - public String getSsl_sni() { - return ssl_sni; - } - - public void setSsl_sni(String ssl_sni) { - this.ssl_sni = ssl_sni; - } - - public String getSsl_san() { - return ssl_san; - } - - public void setSsl_san(String ssl_san) { - this.ssl_san = ssl_san; - } - - public String getSsl_cn() { - return ssl_cn; - } - - public void setSsl_cn(String ssl_cn) { - this.ssl_cn = ssl_cn; - } - - public String getSsl_server_side_version() { - return ssl_server_side_version; - } - - public void setSsl_server_side_version(String ssl_server_side_version) { - this.ssl_server_side_version = ssl_server_side_version; - } - - public String getSsl_client_side_version() { - return ssl_client_side_version; - } - - public void setSsl_client_side_version(String ssl_client_side_version) { - this.ssl_client_side_version = ssl_client_side_version; - } - - public String getSsl_error() { - return ssl_error; - } - - public void setSsl_error(String ssl_error) { - this.ssl_error = ssl_error; - } - - public String getFtp_url() { - return ftp_url; - } - - public void setFtp_url(String ftp_url) { - this.ftp_url = ftp_url; - } - - public String getFtp_content() { - return ftp_content; - } - - public void setFtp_content(String ftp_content) { - this.ftp_content = ftp_content; - } - - public int getBgp_type() { - return bgp_type; - } - - public void setBgp_type(int bgp_type) { - this.bgp_type = bgp_type; - } - - public String getBgp_as_num() { - return bgp_as_num; - } - - public void setBgp_as_num(String bgp_as_num) { - this.bgp_as_num = bgp_as_num; - } - - public String getBgp_route() { - return bgp_route; - } - - public void setBgp_route(String bgp_route) { - this.bgp_route = bgp_route; - } - - public String getVoip_calling_account() { - return voip_calling_account; - } - - public void setVoip_calling_account(String voip_calling_account) { - this.voip_calling_account = voip_calling_account; - } - - public String getVoip_called_account() { - return voip_called_account; - } - - public void setVoip_called_account(String voip_called_account) { - this.voip_called_account = voip_called_account; - } - - public String getVoip_calling_number() { - return voip_calling_number; - } - - public void setVoip_calling_number(String voip_calling_number) { - this.voip_calling_number = voip_calling_number; - } - - public String getVoip_called_number() { - return voip_called_number; - } - - public void setVoip_called_number(String voip_called_number) { - this.voip_called_number = voip_called_number; - } - - - public String getStreaming_media_url() { - return streaming_media_url; - } - - public void setStreaming_media_url(String streaming_media_url) { - this.streaming_media_url = streaming_media_url; - } - - public String getStreaming_media_protocol() { - return streaming_media_protocol; - } - - public void setStreaming_media_protocol(String streaming_media_protocol) { - this.streaming_media_protocol = streaming_media_protocol; - } -} diff --git a/src/main/java/cn/ac/iie/bean/proxy/ProxySessionRecordLog.java b/src/main/java/cn/ac/iie/bean/proxy/ProxySessionRecordLog.java deleted file mode 100644 index ba5dc85..0000000 --- a/src/main/java/cn/ac/iie/bean/proxy/ProxySessionRecordLog.java +++ /dev/null @@ -1,203 +0,0 @@ -package cn.ac.iie.bean.proxy; - - -import cn.ac.iie.bean.PublicSessionRecordLog; - -/** - * @author qidaijie - */ -public class ProxySessionRecordLog extends PublicSessionRecordLog { - - -//TODO HTTP协议属性 21 - - private String http_url; - private String http_host; - private String http_domain; - private String http_request_line; - private String http_response_line; - private String http_request_header; - private String http_response_header; - private String http_request_body; - private String http_response_body; - private String http_request_body_key; - private String http_response_body_key; - private int http_proxy_flag; - private int http_sequence; - private String http_snapshot; - private String http_cookie; - private String http_referer; - private String http_user_agent; - private String http_content_length; - private String http_content_type; - private String http_set_cookie; - private String http_version; - - public String getHttp_url() { - return http_url; - } - - public void setHttp_url(String http_url) { - this.http_url = http_url; - } - - public String getHttp_host() { - return http_host; - } - - public void setHttp_host(String http_host) { - this.http_host = http_host; - } - - public String getHttp_domain() { - return http_domain; - } - - public void setHttp_domain(String http_domain) { - this.http_domain = http_domain; - } - - public String getHttp_request_line() { - return http_request_line; - } - - public void setHttp_request_line(String http_request_line) { - this.http_request_line = http_request_line; - } - - public String getHttp_response_line() { - return http_response_line; - } - - public void setHttp_response_line(String http_response_line) { - this.http_response_line = http_response_line; - } - - public String getHttp_request_header() { - return http_request_header; - } - - public void setHttp_request_header(String http_request_header) { - this.http_request_header = http_request_header; - } - - public String getHttp_response_header() { - return http_response_header; - } - - public void setHttp_response_header(String http_response_header) { - this.http_response_header = http_response_header; - } - - public String getHttp_request_body() { - return http_request_body; - } - - public void setHttp_request_body(String http_request_body) { - this.http_request_body = http_request_body; - } - - public String getHttp_response_body() { - return http_response_body; - } - - public void setHttp_response_body(String http_response_body) { - this.http_response_body = http_response_body; - } - - public String getHttp_request_body_key() { - return http_request_body_key; - } - - public void setHttp_request_body_key(String http_request_body_key) { - this.http_request_body_key = http_request_body_key; - } - - public String getHttp_response_body_key() { - return http_response_body_key; - } - - public void setHttp_response_body_key(String http_response_body_key) { - this.http_response_body_key = http_response_body_key; - } - - public int getHttp_proxy_flag() { - return http_proxy_flag; - } - - public void setHttp_proxy_flag(int http_proxy_flag) { - this.http_proxy_flag = http_proxy_flag; - } - - public int getHttp_sequence() { - return http_sequence; - } - - public void setHttp_sequence(int http_sequence) { - this.http_sequence = http_sequence; - } - - public String getHttp_snapshot() { - return http_snapshot; - } - - public void setHttp_snapshot(String http_snapshot) { - this.http_snapshot = http_snapshot; - } - - public String getHttp_cookie() { - return http_cookie; - } - - public void setHttp_cookie(String http_cookie) { - this.http_cookie = http_cookie; - } - - public String getHttp_referer() { - return http_referer; - } - - public void setHttp_referer(String http_referer) { - this.http_referer = http_referer; - } - - public String getHttp_user_agent() { - return http_user_agent; - } - - public void setHttp_user_agent(String http_user_agent) { - this.http_user_agent = http_user_agent; - } - - public String getHttp_content_length() { - return http_content_length; - } - - public void setHttp_content_length(String http_content_length) { - this.http_content_length = http_content_length; - } - - public String getHttp_content_type() { - return http_content_type; - } - - public void setHttp_content_type(String http_content_type) { - this.http_content_type = http_content_type; - } - - public String getHttp_set_cookie() { - return http_set_cookie; - } - - public void setHttp_set_cookie(String http_set_cookie) { - this.http_set_cookie = http_set_cookie; - } - - public String getHttp_version() { - return http_version; - } - - public void setHttp_version(String http_version) { - this.http_version = http_version; - } -} diff --git a/src/main/java/cn/ac/iie/bean/radius/RadiusSessionRecordLog.java b/src/main/java/cn/ac/iie/bean/radius/RadiusSessionRecordLog.java deleted file mode 100644 index 4dd11a9..0000000 --- a/src/main/java/cn/ac/iie/bean/radius/RadiusSessionRecordLog.java +++ /dev/null @@ -1,89 +0,0 @@ -package cn.ac.iie.bean.radius; - - -import cn.ac.iie.bean.PublicSessionRecordLog; - -/** - * Radius 日志 - * - * @author qidaijie - */ -public class RadiusSessionRecordLog extends PublicSessionRecordLog { - - -//TODO RADIUS协议属性 8 - - private int radius_session_timeout; - private int radius_idle_timeout; - private int radius_acct_status_type; - private int radius_acct_terminate_cause; - private int radius_packet_type; - private String radius_nas_ip; - private String radius_framed_ip; - private String radius_account; - - - public int getRadius_session_timeout() { - return radius_session_timeout; - } - - public void setRadius_session_timeout(int radius_session_timeout) { - this.radius_session_timeout = radius_session_timeout; - } - - public int getRadius_idle_timeout() { - return radius_idle_timeout; - } - - public void setRadius_idle_timeout(int radius_idle_timeout) { - this.radius_idle_timeout = radius_idle_timeout; - } - - public int getRadius_acct_status_type() { - return radius_acct_status_type; - } - - public void setRadius_acct_status_type(int radius_acct_status_type) { - this.radius_acct_status_type = radius_acct_status_type; - } - - public int getRadius_acct_terminate_cause() { - return radius_acct_terminate_cause; - } - - public void setRadius_acct_terminate_cause(int radius_acct_terminate_cause) { - this.radius_acct_terminate_cause = radius_acct_terminate_cause; - } - - public int getRadius_packet_type() { - return radius_packet_type; - } - - public void setRadius_packet_type(int radius_packet_type) { - this.radius_packet_type = radius_packet_type; - } - - public String getRadius_nas_ip() { - return radius_nas_ip; - } - - public void setRadius_nas_ip(String radius_nas_ip) { - this.radius_nas_ip = radius_nas_ip; - } - - public String getRadius_framed_ip() { - return radius_framed_ip; - } - - public void setRadius_framed_ip(String radius_framed_ip) { - this.radius_framed_ip = radius_framed_ip; - } - - public String getRadius_account() { - return radius_account; - } - - public void setRadius_account(String radius_account) { - this.radius_account = radius_account; - } -} diff --git a/src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java b/src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java deleted file mode 100644 index d6a4e54..0000000 --- a/src/main/java/cn/ac/iie/bean/security/SecurityPolicyLog.java +++ /dev/null @@ -1,682 +0,0 @@ -package cn.ac.iie.bean.security; - -import cn.ac.iie.bean.PublicSessionRecordLog; - -/** - * 策略 - * - * @author qidaijie - */ -public class SecurityPolicyLog extends PublicSessionRecordLog { - - -//TODO HTTP协议属性 21 - - private String http_url; - private String http_host; - private String http_domain; - private String http_request_line; - private String http_response_line; - private String http_request_header; - private String http_response_header; - private String http_request_body; - private String http_response_body; - private String http_request_body_key; - private String http_response_body_key; - private int http_proxy_flag; - private int http_sequence; - private String http_snapshot; - private String http_cookie; - private String http_referer; - private String http_user_agent; - private String http_content_length; - private String http_content_type; - private String http_set_cookie; - private String http_version; - -//TODO MAIL协议属性 9 - - private String mail_protocol_type; - private String mail_sender; - private String mail_receiver; - private String mail_subject; - private String mail_content; - private String mail_attachment_name; - private String mail_attachment_content; - private String mail_eml_file; - private String mail_snapshot; - -//TODO 3DNS协议属性 18 - - private int dns_message_id; - private Integer dns_qr; - private Integer dns_opcode; - private int dns_aa; - private int dns_tc; - private int dns_rd; - private int dns_ra; - private int dns_rcode; - private int dns_qdcount; - private int dns_ancount; - private int dns_nscount; - private int dns_arcount; - private String dns_qname; - private int dns_qtype; - private int dns_qclass; - private String dns_cname; - private int dns_sub; - private String dns_rr; - -//TODO SSL协议属性 13 - - - private Integer ssl_pinningst; - private Integer ssl_intercept_state; - private int ssl_server_side_latency; - private int ssl_client_side_latency; - private Integer ssl_cert_verify; - private int ssl_con_latency_ms; - private String ssl_version; - private String ssl_sni; - private String ssl_san; - private String ssl_cn; - private String ssl_server_side_version; - private String ssl_client_side_version; - private String ssl_error; - - //TODO FTP协议属性 2 - private String ftp_url; - private String ftp_content; - - - //TODO BGP协议属性 3 - private int bgp_type; - private String bgp_as_num; - private String bgp_route; - - //TODO VOIP协议属性 4 - private String voip_calling_account; - private String voip_called_account; - private String voip_calling_number; - private String voip_called_number; - - //TODO STREAMING_MEDIA协议属性 2 - private String streaming_media_url; - private String streaming_media_protocol; - - public String getHttp_url() { - return http_url; - } - - public void setHttp_url(String http_url) { - this.http_url = http_url; - } - - public String getHttp_host() { - return http_host; - } - - public void setHttp_host(String http_host) { - this.http_host = http_host; - } - - public String getHttp_domain() { - return http_domain; - } - - public void setHttp_domain(String http_domain) { - this.http_domain = http_domain; - } - - public String getHttp_request_line() { - return http_request_line; - } - - public void setHttp_request_line(String http_request_line) { - this.http_request_line = http_request_line; - } - - public String getHttp_response_line() { - return http_response_line; - } - - public void setHttp_response_line(String http_response_line) { - this.http_response_line = http_response_line; - } - - public String getHttp_request_header() { - return http_request_header; - } - - public void setHttp_request_header(String http_request_header) { - this.http_request_header = http_request_header; - } - - public String getHttp_response_header() { - return http_response_header; - } - - public void setHttp_response_header(String http_response_header) { - this.http_response_header = http_response_header; - } - - public String getHttp_request_body() { - return http_request_body; - } - - public void setHttp_request_body(String http_request_body) { - this.http_request_body = http_request_body; - } - - public String getHttp_response_body() { - return http_response_body; - } - - public void setHttp_response_body(String http_response_body) { - this.http_response_body = http_response_body; - } - - public String getHttp_request_body_key() { - return http_request_body_key; - } - - public void setHttp_request_body_key(String http_request_body_key) { - this.http_request_body_key = http_request_body_key; - } - - public String getHttp_response_body_key() { - return http_response_body_key; - } - - public void setHttp_response_body_key(String http_response_body_key) { - this.http_response_body_key = http_response_body_key; - } - - public int getHttp_proxy_flag() { - return http_proxy_flag; - } - - public void setHttp_proxy_flag(int http_proxy_flag) { - this.http_proxy_flag = http_proxy_flag; - } - - public int getHttp_sequence() { - return http_sequence; - } - - public void setHttp_sequence(int http_sequence) { - this.http_sequence = http_sequence; - } - - public String getHttp_snapshot() { - return http_snapshot; - } - - public void setHttp_snapshot(String http_snapshot) { - this.http_snapshot = http_snapshot; - } - - public String getHttp_cookie() { - return http_cookie; - } - - public void setHttp_cookie(String http_cookie) { - this.http_cookie = http_cookie; - } - - public String getHttp_referer() { - return http_referer; - } - - public void setHttp_referer(String http_referer) { - this.http_referer = http_referer; - } - - public String getHttp_user_agent() { - return http_user_agent; - } - - public void setHttp_user_agent(String http_user_agent) { - this.http_user_agent = http_user_agent; - } - - public String getHttp_content_length() { - return http_content_length; - } - - public void setHttp_content_length(String http_content_length) { - this.http_content_length = http_content_length; - } - - public String getHttp_content_type() { - return http_content_type; - } - - public void setHttp_content_type(String http_content_type) { - this.http_content_type = http_content_type; - } - - public String getHttp_set_cookie() { - return http_set_cookie; - } - - public void setHttp_set_cookie(String http_set_cookie) { - this.http_set_cookie = http_set_cookie; - } - - public String getHttp_version() { - return http_version; - } - - public void setHttp_version(String http_version) { - this.http_version = http_version; - } - - public String getMail_protocol_type() { - return mail_protocol_type; - } - - public void setMail_protocol_type(String mail_protocol_type) { - this.mail_protocol_type = mail_protocol_type; - } - - public String getMail_sender() { - return mail_sender; - } - - public void setMail_sender(String mail_sender) { - this.mail_sender = mail_sender; - } - - public String getMail_receiver() { - return mail_receiver; - } - - public void setMail_receiver(String mail_receiver) { - this.mail_receiver = mail_receiver; - } - - public String getMail_subject() { - return mail_subject; - } - - public void setMail_subject(String mail_subject) { - this.mail_subject = mail_subject; - } - - public String getMail_content() { - return mail_content; - } - - public void setMail_content(String mail_content) { - this.mail_content = mail_content; - } - - public String getMail_attachment_name() { - return mail_attachment_name; - } - - public void setMail_attachment_name(String mail_attachment_name) { - this.mail_attachment_name = mail_attachment_name; - } - - public String getMail_attachment_content() { - return mail_attachment_content; - } - - public void setMail_attachment_content(String mail_attachment_content) { - this.mail_attachment_content = mail_attachment_content; - } - - public String getMail_eml_file() { - return mail_eml_file; - } - - public void setMail_eml_file(String mail_eml_file) { - this.mail_eml_file = mail_eml_file; - } - - public String getMail_snapshot() { - return mail_snapshot; - } - - public void setMail_snapshot(String mail_snapshot) { - this.mail_snapshot = mail_snapshot; - } - - public int getDns_message_id() { - return dns_message_id; - } - - public void setDns_message_id(int dns_message_id) { - this.dns_message_id = dns_message_id; - } - - public Integer getDns_qr() { - return dns_qr; - } - - public void setDns_qr(Integer dns_qr) { - this.dns_qr = dns_qr; - } - - public Integer getDns_opcode() { - return dns_opcode; - } - - public void setDns_opcode(Integer dns_opcode) { - this.dns_opcode = dns_opcode; - } - - public int getDns_aa() { - return dns_aa; - } - - public void setDns_aa(int dns_aa) { - this.dns_aa = dns_aa; - } - - public int getDns_tc() { - return dns_tc; - } - - public void setDns_tc(int dns_tc) { - this.dns_tc = dns_tc; - } - - public int getDns_rd() { - return dns_rd; - } - - public void setDns_rd(int dns_rd) { - this.dns_rd = dns_rd; - } - - public int getDns_ra() { - return dns_ra; - } - - public void setDns_ra(int dns_ra) { - this.dns_ra = dns_ra; - } - - public int getDns_rcode() { - return dns_rcode; - } - - public void setDns_rcode(int dns_rcode) { - this.dns_rcode = dns_rcode; - } - - public int getDns_qdcount() { - return dns_qdcount; - } - - public void setDns_qdcount(int dns_qdcount) { - this.dns_qdcount = dns_qdcount; - } - - public int getDns_ancount() { - return dns_ancount; - } - - public void setDns_ancount(int dns_ancount) { - this.dns_ancount = dns_ancount; - } - - public int getDns_nscount() { - return dns_nscount; - } - - public void setDns_nscount(int dns_nscount) { - this.dns_nscount = dns_nscount; - } - - public int getDns_arcount() { - return dns_arcount; - } - - public void setDns_arcount(int dns_arcount) { - this.dns_arcount = dns_arcount; - } - - public String getDns_qname() { - return dns_qname; - } - - public void setDns_qname(String dns_qname) { - this.dns_qname = dns_qname; - } - - public int getDns_qtype() { - return dns_qtype; - } - - public void setDns_qtype(int dns_qtype) { - this.dns_qtype = dns_qtype; - } - - public int getDns_qclass() { - return dns_qclass; - } - - public void setDns_qclass(int dns_qclass) { - this.dns_qclass = dns_qclass; - } - - public String getDns_cname() { - return dns_cname; - } - - public void setDns_cname(String dns_cname) { - this.dns_cname = dns_cname; - } - - public int getDns_sub() { - return dns_sub; - } - - public void setDns_sub(int dns_sub) { - this.dns_sub = dns_sub; - } - - public String getDns_rr() { - return dns_rr; - } - - public void setDns_rr(String dns_rr) { - this.dns_rr = dns_rr; - } - - public Integer getSsl_pinningst() { - return ssl_pinningst; - } - - public void setSsl_pinningst(Integer ssl_pinningst) { - this.ssl_pinningst = ssl_pinningst; - } - - public Integer getSsl_intercept_state() { - return ssl_intercept_state; - } - - public void setSsl_intercept_state(Integer ssl_intercept_state) { - this.ssl_intercept_state = ssl_intercept_state; - } - - public int getSsl_server_side_latency() { - return ssl_server_side_latency; - } - - public void setSsl_server_side_latency(int ssl_server_side_latency) { - this.ssl_server_side_latency = ssl_server_side_latency; - } - - public int getSsl_client_side_latency() { - return ssl_client_side_latency; - } - - public void setSsl_client_side_latency(int ssl_client_side_latency) { - this.ssl_client_side_latency = ssl_client_side_latency; - } - - public Integer getSsl_cert_verify() { - return ssl_cert_verify; - } - - public void setSsl_cert_verify(Integer ssl_cert_verify) { - this.ssl_cert_verify = ssl_cert_verify; - } - - public int getSsl_con_latency_ms() { - return ssl_con_latency_ms; - } - - public void setSsl_con_latency_ms(int ssl_con_latency_ms) { - this.ssl_con_latency_ms = ssl_con_latency_ms; - } - - public String getSsl_version() { - return ssl_version; - } - - public void setSsl_version(String ssl_version) { - this.ssl_version = ssl_version; - } - - public String getSsl_sni() { - return ssl_sni; - } - - public void setSsl_sni(String ssl_sni) { - this.ssl_sni = ssl_sni; - } - - public String getSsl_san() { - return ssl_san; - } - - public void setSsl_san(String ssl_san) { - this.ssl_san = ssl_san; - } - - public String getSsl_cn() { - return ssl_cn; - } - - public void setSsl_cn(String ssl_cn) { - this.ssl_cn = ssl_cn; - } - - public String getSsl_server_side_version() { - return ssl_server_side_version; - } - - public void setSsl_server_side_version(String ssl_server_side_version) { - this.ssl_server_side_version = ssl_server_side_version; - } - - public String getSsl_client_side_version() { - return ssl_client_side_version; - } - - public void setSsl_client_side_version(String ssl_client_side_version) { - this.ssl_client_side_version = ssl_client_side_version; - } - - public String getSsl_error() { - return ssl_error; - } - - public void setSsl_error(String ssl_error) { - this.ssl_error = ssl_error; - } - - public String getFtp_url() { - return ftp_url; - } - - public void setFtp_url(String ftp_url) { - this.ftp_url = ftp_url; - } - - public String getFtp_content() { - return ftp_content; - } - - public void setFtp_content(String ftp_content) { - this.ftp_content = ftp_content; - } - - public int getBgp_type() { - return bgp_type; - } - - public void setBgp_type(int bgp_type) { - this.bgp_type = bgp_type; - } - - public String getBgp_as_num() { - return bgp_as_num; - } - - public void setBgp_as_num(String bgp_as_num) { - this.bgp_as_num = bgp_as_num; - } - - public String getBgp_route() { - return bgp_route; - } - - public void setBgp_route(String bgp_route) { - this.bgp_route = bgp_route; - } - - public String getVoip_calling_account() { - return voip_calling_account; - } - - public void setVoip_calling_account(String voip_calling_account) { - this.voip_calling_account = voip_calling_account; - } - - public String getVoip_called_account() { - return voip_called_account; - } - - public void setVoip_called_account(String voip_called_account) { - this.voip_called_account = voip_called_account; - } - - public String getVoip_calling_number() { - return voip_calling_number; - } - - public void setVoip_calling_number(String voip_calling_number) { - this.voip_calling_number = voip_calling_number; - } - - public String getVoip_called_number() { - return voip_called_number; - } - - public void setVoip_called_number(String voip_called_number) { - this.voip_called_number = voip_called_number; - } - - public String getStreaming_media_url() { - return streaming_media_url; - } - - public void setStreaming_media_url(String streaming_media_url) { - this.streaming_media_url = streaming_media_url; - } - - public String getStreaming_media_protocol() { - return streaming_media_protocol; - } - - public void setStreaming_media_protocol(String streaming_media_protocol) { - this.streaming_media_protocol = streaming_media_protocol; - } -} diff --git a/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java b/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java index 6f923d5..5f6ab55 100644 --- a/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java +++ b/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java @@ -16,8 +16,10 @@ import org.apache.storm.tuple.Values; import java.util.HashMap; import java.util.Map; -import static cn.ac.iie.utils.general.TransFormUtils.getCollectProtocolMessage; -import static cn.ac.iie.utils.hbase.HBaseUtils.change; + +import static cn.ac.iie.utils.general.TransFormUtils.dealCommonMessage; + + /** * 通联关系日志补全 @@ -36,11 +38,11 @@ public class CollectCompletedBolt extends BaseBasicBolt { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { try { if (TupleUtils.isTick(tuple)) { - change(); + HBaseUtils.change(); } else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { - basicOutputCollector.emit(new Values(getCollectProtocolMessage(message))); + basicOutputCollector.emit(new Values(dealCommonMessage(message))); } } } catch (Exception e) { diff --git a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java index fc65431..f89801a 100644 --- a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java @@ -15,7 +15,7 @@ import org.apache.storm.tuple.Values; import java.util.HashMap; import java.util.Map; -import static cn.ac.iie.utils.general.TransFormUtils.getProxyMessage; +import static cn.ac.iie.utils.general.TransFormUtils.dealCommonMessage; import static cn.ac.iie.utils.hbase.HBaseUtils.change; /** @@ -41,7 +41,7 @@ public class ProxyCompletionBolt extends BaseBasicBolt { } else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { - basicOutputCollector.emit(new Values(getProxyMessage(message))); + basicOutputCollector.emit(new Values(dealCommonMessage(message))); } } } catch (Exception e) { diff --git a/src/main/java/cn/ac/iie/bolt/radius/RadiusCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/radius/RadiusCompletionBolt.java index 7771e74..5293059 100644 --- a/src/main/java/cn/ac/iie/bolt/radius/RadiusCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/radius/RadiusCompletionBolt.java @@ -13,7 +13,7 @@ import org.apache.storm.tuple.Values; import java.util.Map; -import static cn.ac.iie.utils.general.TransFormUtils.getRadiusMessage; +import static cn.ac.iie.utils.general.TransFormUtils.dealCommonMessage; /** * 通联关系日志补全 @@ -35,7 +35,7 @@ public class RadiusCompletionBolt extends BaseBasicBolt { try { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { - basicOutputCollector.emit(new Values(getRadiusMessage(message))); + basicOutputCollector.emit(new Values(dealCommonMessage(message))); } } catch (Exception e) { logger.error(FlowWriteConfig.KAFKA_TOPIC + "接收/解析过程出现异常"); diff --git a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java index 76bc8af..9a6b303 100644 --- a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java @@ -16,8 +16,8 @@ import org.apache.storm.tuple.Values; import java.util.HashMap; import java.util.Map; -import static cn.ac.iie.utils.general.TransFormUtils.getSecurityMessage; -import static cn.ac.iie.utils.general.schema.TransFormUtils.dealCommonMessage; +import static cn.ac.iie.utils.general.TransFormUtils.dealCommonMessage; + /** * 通联关系日志补全 diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index ffed96b..5cd1ab5 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -1,23 +1,17 @@ package cn.ac.iie.utils.general; -import cn.ac.iie.bean.connection.ConnectionRecordLog; -import cn.ac.iie.bean.proxy.ProxySessionRecordLog; -import cn.ac.iie.bean.radius.RadiusSessionRecordLog; -import cn.ac.iie.bean.security.SecurityPolicyLog; + import cn.ac.iie.common.FlowWriteConfig; import cn.ac.iie.utils.hbase.HBaseUtils; +import cn.ac.iie.utils.json.JsonParseUtil; import cn.ac.iie.utils.system.SnowflakeId; -import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.google.common.net.InternetDomainName; import com.zdjizhi.utils.IpLookup; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; -import java.util.regex.Matcher; +import java.util.*; import java.util.regex.Pattern; @@ -30,9 +24,10 @@ import java.util.regex.Pattern; public class TransFormUtils { private static Logger logger = Logger.getLogger(TransFormUtils.class); - // private final static Set PUBLIC_SUFFIX_SET = new HashSet( -// Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se" -// .split("\\|"))); + private final static Set PUBLIC_SUFFIX_SET = new HashSet( + Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se" + .split("\\|"))); + private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})"); private static IpLookup ipLookup = new IpLookup.Builder(false) .loadDataFileV4(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") .loadDataFileV6(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") @@ -40,63 +35,12 @@ public class TransFormUtils { .loadAsnDataFileV6(FlowWriteConfig.IP_LIBRARY + "asn_v6.mmdb") .build(); - - /** - * 解析日志，并补全 - * 补subscriber_id，不补domain - * - * @param message radius原始日志 - * @return 补全后的日志 - */ - public static String getRadiusMessage(String message) { - RadiusSessionRecordLog radiusSessionRecordLog = JSONObject.parseObject(message, RadiusSessionRecordLog.class); - String serverIp = radiusSessionRecordLog.getCommon_server_ip(); - String clientIp = radiusSessionRecordLog.getCommon_client_ip(); - try { - radiusSessionRecordLog.setCommon_log_id(SnowflakeId.generateId()); - radiusSessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); - radiusSessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); - radiusSessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); - radiusSessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); - radiusSessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); - radiusSessionRecordLog.setCommon_subscriber_id(radiusSessionRecordLog.getRadius_account()); - return JSONObject.toJSONString(radiusSessionRecordLog); - } catch (Exception e) { - logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常"); - e.printStackTrace(); - return ""; - } - } - - - /** - * 解析日志，并补全 - * 补domain,补subscriber_id - * - * @param message Proxy原始日志 - * @return 补全后的日志 - */ - public static String getProxyMessage(String message) { - ProxySessionRecordLog proxySessionRecordLog = JSONObject.parseObject(message, ProxySessionRecordLog.class); - String serverIp = proxySessionRecordLog.getCommon_server_ip(); - String clientIp = proxySessionRecordLog.getCommon_client_ip(); - try { - proxySessionRecordLog.setCommon_log_id(SnowflakeId.generateId()); - proxySessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); - proxySessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); - proxySessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); - proxySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); - proxySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); - //TODO 集成AAA数据subscribe_id -数据端补全 - proxySessionRecordLog.setCommon_subscriber_id(HBaseUtils.getAccount(clientIp)); - proxySessionRecordLog.setHttp_domain(getTopDomain(null, proxySessionRecordLog.getHttp_host())); - return JSONObject.toJSONString(proxySessionRecordLog); - } catch (Exception e) { - logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常"); - e.printStackTrace(); - return ""; - } - } + //在内存中加载反射类用的map + private static HashMap map = JsonParseUtil.getMapFromhttp(FlowWriteConfig.SCHEMA_HTTP); + //反射成一个类 + private static Object mapObject = JsonParseUtil.generateObject(map); + //获取任务列表 + private static ArrayList jobList = JsonParseUtil.getJobListFromHttp(FlowWriteConfig.SCHEMA_HTTP); /** @@ -105,22 +49,51 @@ public class TransFormUtils { * * @param message Security原始日志 * @return 补全后的日志 + *

+ * current_timestamp + * snowflake_id + * geo_ip_detail + * geo_asn + * radius_match + * geo_ip_country + * geo_asn + * sub_domain + * sub_domain */ - public static String getSecurityMessage(String message) { - SecurityPolicyLog securitySessionRecordLog = JSONObject.parseObject(message, SecurityPolicyLog.class); - String serverIp = securitySessionRecordLog.getCommon_server_ip(); - String clientIp = securitySessionRecordLog.getCommon_client_ip(); + public static String dealCommonMessage(String message) { + + Object object = JSONObject.parseObject(message, mapObject.getClass()); +// System.out.println("补全之前 ===》 "+JSON.toJSONString(object)); try { - securitySessionRecordLog.setCommon_log_id(SnowflakeId.generateId()); - securitySessionRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); - securitySessionRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); - securitySessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); - securitySessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); - securitySessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); - //TODO 集成AAA数据subscribe_id -数据端补全 - securitySessionRecordLog.setCommon_subscriber_id(HBaseUtils.getAccount(clientIp)); - securitySessionRecordLog.setHttp_domain(getTopDomain(securitySessionRecordLog.getSsl_sni(), securitySessionRecordLog.getHttp_host())); - return JSONObject.toJSONString(securitySessionRecordLog); + for (String[] strings : jobList) { + + if (strings[2].equals("current_timestamp")) { + JsonParseUtil.setValue(object, strings[1], getCurrentTime()); + } else if (strings[2].equals("snowflake_id")) { + JsonParseUtil.setValue(object, strings[1], getSnowflakeId()); + } else if (strings[2].equals("geo_ip_detail")) { + JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(JsonParseUtil.getValue(object, strings[0]).toString())); + } else if (strings[2].equals("geo_asn")) { + JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(JsonParseUtil.getValue(object, strings[0]).toString())); + } else if (strings[2].equals("radius_match")) { + JsonParseUtil.setValue(object,strings[1],HBaseUtils.getAccount(JsonParseUtil.getValue(object,strings[0]).toString())); +// JsonParseUtil.setValue(object, strings[1], "aaaaaaaaa"); + } else if (strings[2].equals("geo_ip_country")) { + JsonParseUtil.setValue(object, strings[1], getGeoIpCountry(JsonParseUtil.getValue(object, strings[0]).toString())); + } else if (strings[0].equals("http_host") && strings[2].equals("sub_domain")) { + JsonParseUtil.setValue(object,strings[1],getTopDomain(null,JsonParseUtil.getValue(object,strings[0]).toString())); + } else if (strings[0].equals("ssl_sni") && strings[2].equals("sub_domain")) { + if (StringUtil.isBlank(JsonParseUtil.getValue(object, strings[1]).toString())) { + JsonParseUtil.setValue(object,strings[1],getTopDomain(JsonParseUtil.getValue(object,strings[0]).toString(),null)); + } + + } + } + + + return JSONObject.toJSONString(object); +// System.out.println("补全之后 ===》 "+JSON.toJSONString(object)); + } catch (Exception e) { logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常"); e.printStackTrace(); @@ -128,37 +101,9 @@ public class TransFormUtils { } } - /** - * 解析日志，并补全 - * 补domain,补subscriber_id - * - * @param message CollectProtocol原始日志 - * @return 补全后的日志 - */ - public static String getCollectProtocolMessage(String message) { - ConnectionRecordLog collectProtocolRecordLog = JSONObject.parseObject(message, ConnectionRecordLog.class); - String serverIp = collectProtocolRecordLog.getCommon_server_ip(); - String clientIp = collectProtocolRecordLog.getCommon_client_ip(); - try { - collectProtocolRecordLog.setCommon_log_id(SnowflakeId.generateId()); - collectProtocolRecordLog.setCommon_recv_time((System.currentTimeMillis() / 1000)); - collectProtocolRecordLog.setCommon_server_location(ipLookup.countryLookup(serverIp)); - collectProtocolRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp)); - collectProtocolRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true)); - collectProtocolRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true)); - //TODO 集成AAA数据subscribe_id -数据端补全 - collectProtocolRecordLog.setCommon_subscriber_id(HBaseUtils.getAccount(clientIp)); - collectProtocolRecordLog.setHttp_domain(getTopDomain(collectProtocolRecordLog.getSsl_sni(), collectProtocolRecordLog.getHttp_host())); - return JSONObject.toJSONString(collectProtocolRecordLog); - } catch (Exception e) { - logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常"); - e.printStackTrace(); - return ""; - } - } /** - * 有sni通过sni获取域名，有hots根据host获取域名 + * 有sni通过sni获取域名，有host根据host获取域名 * * @param sni sni * @param host host @@ -191,5 +136,91 @@ public class TransFormUtils { return domain; } + /** + * 生成当前时间戳的操作 + */ + private static long getCurrentTime() { + return (System.currentTimeMillis() / 1000); + } + /** + * 雪花模型生成id + * + * @return + */ + private static long getSnowflakeId() { + + return SnowflakeId.generateId(); + } + + /** + * 根据clientIp获取location信息 + * + * @param ip + * @return + */ + private static String getGeoIpDetail(String ip) { + + return ipLookup.cityLookupDetail(ip); + } + + /** + * 根据ip获取asn信息 + * + * @param ip + * @return + */ + private static String getGeoAsn(String ip) { + + return ipLookup.asnLookup(ip, true); + } + + /** + * 根据ip获取country信息 + * + * @param ip + * @return + */ + private static String getGeoIpCountry(String ip) { + + return ipLookup.countryLookup(ip); + } + + /** + * radius借助hbase补齐 + * + * @param ip + * @return + */ + private static String radiusMatch(String ip) { + return HBaseUtils.getAccount(ip); + } + + /** + * switch 匹配合适的方法 + * current_timestamp + * snowflake_id + * geo_ip_detail + * geo_asn + * radius_match + * geo_ip_country + * geo_asn + * sub_domain + * sub_domain + * @param func + */ +//TODO 行不通的原因是无法统一一个确定的返回值类型 + /* private static String switchFunc(String func){ + switch (func){ + case "current_timestamp": + return String.valueOf(getCurrentTime()); + case "snowflake_id": + case "geo_ip_detail": + case "geo_asn": + case "radius_match": + case "geo_ip_country": + case "sub_domain": + } + return func; + }*/ } diff --git a/src/main/java/cn/ac/iie/utils/general/schema/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/schema/TransFormUtils.java deleted file mode 100644 index 07f8f01..0000000 --- a/src/main/java/cn/ac/iie/utils/general/schema/TransFormUtils.java +++ /dev/null @@ -1,227 +0,0 @@ -package cn.ac.iie.utils.general.schema; - - -import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HBaseUtils; -import cn.ac.iie.utils.json.JsonParseUtil; -import cn.ac.iie.utils.system.SnowflakeId; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import com.google.common.net.InternetDomainName; -import com.zdjizhi.utils.IpLookup; -import com.zdjizhi.utils.StringUtil; -import org.apache.log4j.Logger; - -import java.util.*; -import java.util.regex.Pattern; - - -/** - * 描述:转换或补全工具类 - * - * @author qidaijie - * @create 2018-08-13 15:11 - */ - -public class TransFormUtils { - private static Logger logger = Logger.getLogger(TransFormUtils.class); - private final static Set PUBLIC_SUFFIX_SET = new HashSet( - Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se" - .split("\\|"))); - private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})"); - private static IpLookup ipLookup = new IpLookup.Builder(false) - .loadDataFileV4(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") - .loadDataFileV6(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") - .loadAsnDataFileV4(FlowWriteConfig.IP_LIBRARY + "asn_v4.mmdb") - .loadAsnDataFileV6(FlowWriteConfig.IP_LIBRARY + "asn_v6.mmdb") - .build(); - - //在内存中加载反射类用的map - private static HashMap map = JsonParseUtil.getMapFromhttp(FlowWriteConfig.SCHEMA_HTTP); - //反射成一个类 - private static Object mapObject = JsonParseUtil.generateObject(map); - //获取任务列表 - private static ArrayList jobList = JsonParseUtil.getJobListFromHttp(FlowWriteConfig.SCHEMA_HTTP); - - - /** - * 解析日志，并补全 - * 补domain,补subscriber_id - * - * @param message Security原始日志 - * @return 补全后的日志 - *

- * current_timestamp - * snowflake_id - * geo_ip_detail - * geo_asn - * radius_match - * geo_ip_country - * geo_asn - * sub_domain - * sub_domain - */ - public static String dealCommonMessage(String message) { - - Object object = JSONObject.parseObject(message, mapObject.getClass()); -// System.out.println("补全之前 ===》 "+JSON.toJSONString(object)); - try { - for (String[] strings : jobList) { - - if (strings[2].equals("current_timestamp")) { - JsonParseUtil.setValue(object, strings[1], getCurrentTime()); - } else if (strings[2].equals("snowflake_id")) { - JsonParseUtil.setValue(object, strings[1], getSnowflakeId()); - } else if (strings[2].equals("geo_ip_detail")) { - JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(JsonParseUtil.getValue(object, strings[0]).toString())); - } else if (strings[2].equals("geo_asn")) { - JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(JsonParseUtil.getValue(object, strings[0]).toString())); - } else if (strings[2].equals("radius_match")) { - JsonParseUtil.setValue(object,strings[1],HBaseUtils.getAccount(JsonParseUtil.getValue(object,strings[0]).toString())); -// JsonParseUtil.setValue(object, strings[1], "aaaaaaaaa"); - } else if (strings[2].equals("geo_ip_country")) { - JsonParseUtil.setValue(object, strings[1], getGeoIpCountry(JsonParseUtil.getValue(object, strings[0]).toString())); - } else if (strings[0].equals("http_host") && strings[2].equals("sub_domain")) { - JsonParseUtil.setValue(object,strings[1],getTopDomain(null,JsonParseUtil.getValue(object,strings[0]).toString())); - } else if (strings[0].equals("ssl_sni") && strings[2].equals("sub_domain")) { - if (StringUtil.isBlank(JsonParseUtil.getValue(object, strings[1]).toString())) { - JsonParseUtil.setValue(object,strings[1],getTopDomain(JsonParseUtil.getValue(object,strings[0]).toString(),null)); - } - - } - } - - - return JSONObject.toJSONString(object); -// System.out.println("补全之后 ===》 "+JSON.toJSONString(object)); - - } catch (Exception e) { - logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常"); - e.printStackTrace(); - return ""; - } - } - - - /** - * 有sni通过sni获取域名，有host根据host获取域名 - * - * @param sni sni - * @param host host - * @return 顶级域名 - */ - private static String getTopDomain(String sni, String host) { - if (StringUtil.isNotBlank(host)) { - return getDomainName(host); - } else if (StringUtil.isNotBlank(sni)) { - return getDomainName(sni); - } else { - return ""; - } - } - - - /** - * 根据url截取顶级域名 - * - * @param host 网站url - * @return 顶级域名 - */ - private static String getDomainName(String host) { - String domain = ""; - try { - domain = InternetDomainName.from(host).topPrivateDomain().toString(); - } catch (Exception e) { - logger.error("host解析顶级域名异常: " + e.getMessage()); - } - return domain; - } - - /** - * 生成当前时间戳的操作 - */ - private static long getCurrentTime() { - return (System.currentTimeMillis() / 1000); - } - - /** - * 雪花模型生成id - * - * @return - */ - private static long getSnowflakeId() { - - return SnowflakeId.generateId(); - } - - /** - * 根据clientIp获取location信息 - * - * @param ip - * @return - */ - private static String getGeoIpDetail(String ip) { - - return ipLookup.cityLookupDetail(ip); - } - - /** - * 根据ip获取asn信息 - * - * @param ip - * @return - */ - private static String getGeoAsn(String ip) { - - return ipLookup.asnLookup(ip, true); - } - - /** - * 根据ip获取country信息 - * - * @param ip - * @return - */ - private static String getGeoIpCountry(String ip) { - - return ipLookup.countryLookup(ip); - } - - /** - * radius借助hbase补齐 - * - * @param ip - * @return - */ - private static String radiusMatch(String ip) { - return HBaseUtils.getAccount(ip); - } - - /** - * switch 匹配合适的方法 - * current_timestamp - * snowflake_id - * geo_ip_detail - * geo_asn - * radius_match - * geo_ip_country - * geo_asn - * sub_domain - * sub_domain - * @param func - */ -//TODO 行不通的原因是无法统一一个确定的返回值类型 - /* private static String switchFunc(String func){ - switch (func){ - case "current_timestamp": - return String.valueOf(getCurrentTime()); - case "snowflake_id": - case "geo_ip_detail": - case "geo_asn": - case "radius_match": - case "geo_ip_country": - case "sub_domain": - } - return func; - }*/ -} From 3245ac75078378967a3a9bb28c5e39dd5fc8ef7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Thu, 19 Dec 2019 15:45:51 +0800 Subject: [PATCH 18/27] =?UTF-8?q?=E5=8A=A8=E6=80=81=E8=8E=B7=E5=8F=96schem?= =?UTF-8?q?a=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/cn/ac/iie/utils/general/TransFormUtils.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index 5cd1ab5..e5f9ea6 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -81,9 +81,12 @@ public class TransFormUtils { } else if (strings[2].equals("geo_ip_country")) { JsonParseUtil.setValue(object, strings[1], getGeoIpCountry(JsonParseUtil.getValue(object, strings[0]).toString())); } else if (strings[0].equals("http_host") && strings[2].equals("sub_domain")) { - JsonParseUtil.setValue(object,strings[1],getTopDomain(null,JsonParseUtil.getValue(object,strings[0]).toString())); + if (JsonParseUtil.getValue(object, strings[1]) == null || StringUtil.isBlank(JsonParseUtil.getValue(object, strings[1]).toString())) { +// JsonParseUtil.setValue(object, strings[1], "baidu.com"); + JsonParseUtil.setValue(object, strings[1], getTopDomain(null, JsonParseUtil.getValue(object, strings[0]).toString())); + } } else if (strings[0].equals("ssl_sni") && strings[2].equals("sub_domain")) { - if (StringUtil.isBlank(JsonParseUtil.getValue(object, strings[1]).toString())) { + if (JsonParseUtil.getValue(object, strings[1]) == null || StringUtil.isBlank(JsonParseUtil.getValue(object, strings[1]).toString())) { JsonParseUtil.setValue(object,strings[1],getTopDomain(JsonParseUtil.getValue(object,strings[0]).toString(),null)); } From b1cbb95ed16b9600d8d54d3db5b614193ccf207f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Thu, 19 Dec 2019 17:38:20 +0800 Subject: [PATCH 19/27] =?UTF-8?q?=E5=8A=A8=E6=80=81=E8=8E=B7=E5=8F=96schem?= =?UTF-8?q?a=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ac/iie/utils/general/TransFormUtils.java | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index e5f9ea6..b28bdcc 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -41,7 +41,9 @@ public class TransFormUtils { private static Object mapObject = JsonParseUtil.generateObject(map); //获取任务列表 private static ArrayList jobList = JsonParseUtil.getJobListFromHttp(FlowWriteConfig.SCHEMA_HTTP); - + //需要用到的值 + public static Object use; + public static Object appendTo; /** * 解析日志，并补全 @@ -66,33 +68,36 @@ public class TransFormUtils { // System.out.println("补全之前 ===》 "+JSON.toJSONString(object)); try { for (String[] strings : jobList) { - + //参数的值 + use = JsonParseUtil.getValue(object,strings[0]); + //补全的字段的值 + appendTo = JsonParseUtil.getValue(object,strings[1]); if (strings[2].equals("current_timestamp")) { JsonParseUtil.setValue(object, strings[1], getCurrentTime()); } else if (strings[2].equals("snowflake_id")) { JsonParseUtil.setValue(object, strings[1], getSnowflakeId()); } else if (strings[2].equals("geo_ip_detail")) { - JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(JsonParseUtil.getValue(object, strings[0]).toString())); + JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(use.toString())); } else if (strings[2].equals("geo_asn")) { - JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(JsonParseUtil.getValue(object, strings[0]).toString())); + JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(use.toString())); } else if (strings[2].equals("radius_match")) { - JsonParseUtil.setValue(object,strings[1],HBaseUtils.getAccount(JsonParseUtil.getValue(object,strings[0]).toString())); -// JsonParseUtil.setValue(object, strings[1], "aaaaaaaaa"); + JsonParseUtil.setValue(object,strings[1],HBaseUtils.getAccount(use.toString())); } else if (strings[2].equals("geo_ip_country")) { - JsonParseUtil.setValue(object, strings[1], getGeoIpCountry(JsonParseUtil.getValue(object, strings[0]).toString())); - } else if (strings[0].equals("http_host") && strings[2].equals("sub_domain")) { - if (JsonParseUtil.getValue(object, strings[1]) == null || StringUtil.isBlank(JsonParseUtil.getValue(object, strings[1]).toString())) { -// JsonParseUtil.setValue(object, strings[1], "baidu.com"); - JsonParseUtil.setValue(object, strings[1], getTopDomain(null, JsonParseUtil.getValue(object, strings[0]).toString())); + JsonParseUtil.setValue(object, strings[1], getGeoIpCountry(use.toString())); + } else if (strings[0].equals("http_host") && strings[2].equals("sub_domain") && use != null) { + if (appendTo == null || StringUtil.isBlank(appendTo.toString())) { + JsonParseUtil.setValue(object, strings[1], getTopDomain(null, use.toString())); } - } else if (strings[0].equals("ssl_sni") && strings[2].equals("sub_domain")) { - if (JsonParseUtil.getValue(object, strings[1]) == null || StringUtil.isBlank(JsonParseUtil.getValue(object, strings[1]).toString())) { - JsonParseUtil.setValue(object,strings[1],getTopDomain(JsonParseUtil.getValue(object,strings[0]).toString(),null)); + } else if (strings[0].equals("ssl_sni") && strings[2].equals("sub_domain") && use != null) { + if (appendTo == null || StringUtil.isBlank(appendTo.toString())) { + JsonParseUtil.setValue(object,strings[1],getTopDomain(use.toString(),null)); } } } - + //初始化一下 + use = null; + appendTo = null; return JSONObject.toJSONString(object); // System.out.println("补全之后 ===》 "+JSON.toJSONString(object)); From a12f82ae8a922f24fc39a0ea8f5c66a71114cd30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Fri, 20 Dec 2019 12:49:30 +0800 Subject: [PATCH 20/27] =?UTF-8?q?=E5=8A=A8=E6=80=81=E8=8E=B7=E5=8F=96schem?= =?UTF-8?q?a=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/cn/ac/iie/utils/general/TransFormUtils.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index b28bdcc..8c3bf1c 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -41,9 +41,7 @@ public class TransFormUtils { private static Object mapObject = JsonParseUtil.generateObject(map); //获取任务列表 private static ArrayList jobList = JsonParseUtil.getJobListFromHttp(FlowWriteConfig.SCHEMA_HTTP); - //需要用到的值 - public static Object use; - public static Object appendTo; + /** * 解析日志，并补全 @@ -69,9 +67,9 @@ public class TransFormUtils { try { for (String[] strings : jobList) { //参数的值 - use = JsonParseUtil.getValue(object,strings[0]); + Object use = JsonParseUtil.getValue(object,strings[0]); //补全的字段的值 - appendTo = JsonParseUtil.getValue(object,strings[1]); + Object appendTo = JsonParseUtil.getValue(object,strings[1]); if (strings[2].equals("current_timestamp")) { JsonParseUtil.setValue(object, strings[1], getCurrentTime()); } else if (strings[2].equals("snowflake_id")) { @@ -95,9 +93,6 @@ public class TransFormUtils { } } - //初始化一下 - use = null; - appendTo = null; return JSONObject.toJSONString(object); // System.out.println("补全之后 ===》 "+JSON.toJSONString(object)); From fcbf3824baf957745930d624b31b9883e74ddfda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Fri, 20 Dec 2019 17:42:51 +0800 Subject: [PATCH 21/27] =?UTF-8?q?=E5=8A=A8=E6=80=81=E8=8E=B7=E5=8F=96schem?= =?UTF-8?q?a=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ac/iie/utils/general/TransFormUtils.java | 31 ++++++++++++++----- .../cn/ac/iie/utils/json/JsonParseUtil.java | 29 ++++++++++++++--- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index 8c3bf1c..0f66dc5 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -10,6 +10,7 @@ import com.google.common.net.InternetDomainName; import com.zdjizhi.utils.IpLookup; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; +import org.junit.jupiter.api.Test; import java.util.*; import java.util.regex.Pattern; @@ -42,7 +43,6 @@ public class TransFormUtils { //获取任务列表 private static ArrayList jobList = JsonParseUtil.getJobListFromHttp(FlowWriteConfig.SCHEMA_HTTP); - /** * 解析日志，并补全 * 补domain,补subscriber_id @@ -62,14 +62,19 @@ public class TransFormUtils { */ public static String dealCommonMessage(String message) { + + +// message="{\"ssl_sni\":\"pos.baidu.com\",\"ssl_version\":\"v3\",\"ssl_cn\":\"baidu.com\",\"ssl_san\":\"baidu.com;click.hm.baidu.com;cm.pos.baidu.com;log.hm.baidu.com;update.pan.baidu.com;wn.pos.baidu.com;*.91.com;*.aipage.cn;*.aipage.com;*.apollo.auto;*.baidu.com;*.baidubce.com;*.baiducontent.com;*.baidupcs.com;*.baidustatic.com;*.baifubao.com;*.bce.baidu.com;*.bcehost.com;*.bdimg.com;*.bdstatic.com;*.bdtjrcv.com;*.bj.baidubce.com;*.chuanke.com;*.dlnel.com;*.dlnel.org;*.dueros.baidu.com;*.eyun.baidu.com;*.fanyi.baidu.com;*.gz.baidubce.com;*.hao123.baidu.com;*.hao123.com;*.hao222.com;*.haokan.com;*.im.baidu.com;*.map.baidu.com;*.mbd.baidu.com;*.mipcdn.com;*.news.baidu.com;*.nuomi.com;*.safe.baidu.com;*.smartapps.cn;*.su.baidu.com;*.trustgo.com;*.xueshu.baidu.com;apollo.auto;baifubao.com;dwz.cn;mct.y.nuomi.com;www.baidu.cn;www.baidu.com.cn\",\"common_schema_type\":\"SSL\",\"common_server_ip\":\"182.61.200.109\",\"common_client_ip\":\"192.168.50.144\",\"common_server_port\":443,\"common_client_port\":50529,\"common_stream_dir\":3,\"common_address_type\":4,\"common_s2c_pkt_num\":46,\"common_s2c_byte_num\":33149,\"common_c2s_pkt_num\":23,\"common_c2s_byte_num\":6147,\"common_start_time\":1576744784,\"common_end_time\":1576744799,\"common_con_duration_ms\":15000,\"common_stream_trace_id\":7686307990192,\"common_l4_protocol\":\"IPv4_TCP\",\"common_address_list\":\"50529-443-192.168.50.144-182.61.200.109\",\"common_sled_ip\":\"192.168.40.21\",\"common_policy_id\":172,\"common_service\":0,\"common_action\":2,\"common_user_region\":\"{\\\"protocol\\\":\\\"SSL\\\",\\\"protocol_version\\\":{\\\"allow_http2\\\":1,\\\"min\\\":\\\"ssl3\\\",\\\"max\\\":\\\"tls13\\\",\\\"mirror_client\\\":1},\\\"dynamic_bypass\\\":{\\\"mutual_authentication\\\":1,\\\"cert_pinning\\\":1,\\\"cert_transparency\\\":0,\\\"protocol_errors\\\":1,\\\"ev_cert\\\":0},\\\"decrypt_mirror\\\":{\\\"enable\\\":0},\\\"certificate_checks\\\":{\\\"fail_action\\\":\\\"pass-through\\\",\\\"approach\\\":{\\\"self-signed\\\":1,\\\"expiration\\\":1,\\\"cn\\\":1,\\\"issuer\\\":1}},\\\"keyring\\\":1}\"}"; Object object = JSONObject.parseObject(message, mapObject.getClass()); + // System.out.println("补全之前 ===》 "+JSON.toJSONString(object)); try { for (String[] strings : jobList) { //参数的值 - Object use = JsonParseUtil.getValue(object,strings[0]); + Object use = JsonParseUtil.getValue(object, strings[0]); //补全的字段的值 - Object appendTo = JsonParseUtil.getValue(object,strings[1]); + Object appendTo = JsonParseUtil.getValue(object, strings[1]); + if (strings[2].equals("current_timestamp")) { JsonParseUtil.setValue(object, strings[1], getCurrentTime()); } else if (strings[2].equals("snowflake_id")) { @@ -77,9 +82,9 @@ public class TransFormUtils { } else if (strings[2].equals("geo_ip_detail")) { JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(use.toString())); } else if (strings[2].equals("geo_asn")) { - JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(use.toString())); + JsonParseUtil.setValue(object, strings[1], getGeoAsn(use.toString())); } else if (strings[2].equals("radius_match")) { - JsonParseUtil.setValue(object,strings[1],HBaseUtils.getAccount(use.toString())); + JsonParseUtil.setValue(object, strings[1], radiusMatch(use.toString())); } else if (strings[2].equals("geo_ip_country")) { JsonParseUtil.setValue(object, strings[1], getGeoIpCountry(use.toString())); } else if (strings[0].equals("http_host") && strings[2].equals("sub_domain") && use != null) { @@ -88,12 +93,14 @@ public class TransFormUtils { } } else if (strings[0].equals("ssl_sni") && strings[2].equals("sub_domain") && use != null) { if (appendTo == null || StringUtil.isBlank(appendTo.toString())) { - JsonParseUtil.setValue(object,strings[1],getTopDomain(use.toString(),null)); + JsonParseUtil.setValue(object, strings[1], getTopDomain(use.toString(), null)); } } + } + return JSONObject.toJSONString(object); // System.out.println("补全之后 ===》 "+JSON.toJSONString(object)); @@ -105,6 +112,14 @@ public class TransFormUtils { } + @Test + public void aaa() { + String sni = "203.187.160.131:9011"; + System.out.println(getTopDomain(sni, null)); + System.out.println(getTopDomain(null,sni)); + + } + /** * 有sni通过sni获取域名，有host根据host获取域名 * @@ -142,8 +157,8 @@ public class TransFormUtils { /** * 生成当前时间戳的操作 */ - private static long getCurrentTime() { - return (System.currentTimeMillis() / 1000); + private static int getCurrentTime() { + return (int)(System.currentTimeMillis() / 1000); } /** diff --git a/src/main/java/cn/ac/iie/utils/json/JsonParseUtil.java b/src/main/java/cn/ac/iie/utils/json/JsonParseUtil.java index ec8b7f5..3d9ede5 100644 --- a/src/main/java/cn/ac/iie/utils/json/JsonParseUtil.java +++ b/src/main/java/cn/ac/iie/utils/json/JsonParseUtil.java @@ -22,11 +22,20 @@ public class JsonParseUtil { */ public static Class getClassName(String type) { - Class clazz = int.class; + Class clazz; switch (type) { case "int": - clazz = int.class; + clazz = Integer.class; + break; + case "String": + clazz = String.class; + break; + case "long": + clazz = long.class; + break; + case "Integer": + clazz = Integer.class; break; case "double": clazz = double.class; @@ -34,9 +43,6 @@ public class JsonParseUtil { case "float": clazz = float.class; break; - case "long": - clazz = long.class; - break; case "char": clazz = char.class; break; @@ -114,12 +120,25 @@ public class JsonParseUtil { for (Object field : fields) { String name = JSON.parseObject(field.toString()).get("name").toString(); String type = JSON.parseObject(field.toString()).get("type").toString(); +// if( +// name.equals("dns_qr") || +// name.equals("dns_opcode") || +// name.equals("ssl_pinningst") || +// name.equals("ssl_intercept_state") || +// name.equals("ssl_cert_verify") +// +// ){ +// type="Integer"; +// } + //组合用来生成实体类的map map.put(name, getClassName(type)); } + + return map; } From 6e452dab29d38c37ed46c3074b6bba96f1660cc3 Mon Sep 17 00:00:00 2001 From: lee Date: Tue, 4 Feb 2020 10:57:43 +0800 Subject: [PATCH 22/27] =?UTF-8?q?=E6=B7=BB=E5=8A=A0http=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 83ac80e..e8a81ee 100644 --- a/pom.xml +++ b/pom.xml @@ -85,7 +85,7 @@ UTF-8 1.0.0 1.0.2 - 1.4.9 + 2.2.1 2.7.1 @@ -107,7 +107,7 @@ org.apache.storm storm-core ${storm.version} - provided + slf4j-log4j12 @@ -274,5 +274,19 @@ compile + + + org.apache.httpcomponents + httpclient + 4.5.2 + + + + org.apache.httpcomponents + httpcore + 4.4.1 + + + From 7e9eb8d4b9d2940ad2976832e10aa7a822075e14 Mon Sep 17 00:00:00 2001 From: lee Date: Thu, 6 Feb 2020 14:28:17 +0800 Subject: [PATCH 23/27] =?UTF-8?q?=E5=B7=A5=E5=85=B7=E7=B1=BB=E6=89=93?= =?UTF-8?q?=E6=88=90jar=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- out/artifacts/completeUtil/completeUtil.jar | Bin 0 -> 8827 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 out/artifacts/completeUtil/completeUtil.jar diff --git a/out/artifacts/completeUtil/completeUtil.jar b/out/artifacts/completeUtil/completeUtil.jar new file mode 100644 index 0000000000000000000000000000000000000000..866f81531cac1aa1de0370af015718366addd1e5 GIT binary patch literal 8827 zcmZu%WlUY$w%$0!p}4!dODXPdWn&vJ?(Xhx#a)ZL6nA%bcXtW}UVC2hPVde8)~}VB zF|#sPGCmnoQ3e7M9RLdp3m}shR|Nb8Q~)GE#Ky|j(%8{h&C$$~(OlnI|7`>Ss8Cl@ zz>`4nme>)ne040k%Rv)Xqc8BMsZbMCvej&0GWBX|#M)fXdRRayA>~k;jprm+6$(oe z2b&8i_?}i`o&*Ilgyc1hX>EKQ-345u{&kz=3ERIt3q1j1OHmHZZ5r$0=oDnL_87FHQE# zpKlUaaq$qZftJCGVP+~5ejOgIW2qWuT{F@b{&|uXqbK36d`k_V^k`;sILy|{b*Lhl z-aLoZsX7P&-h*(2ZM7j3wnb1^iUcxd^q3YC(hVG5Q4x<==2cegorF<1s-AQ(nrpC` zK)EzzYFS&oe91y|w;SDlNYeq7{as}ga*Va{Q@k4XH3X8O119_@35J{Hzy(aFc3U^M z4rT}A=Mk(Uoy4E$Hd~q)4^ED&oHD6ddI>gQmMad=oJbxMHz%S#K&)j<13BkGS}EQJ zOGc#gr6Eaj7Jq+TT0DMy}xz)o7X=j_O`H!2RG+MK}AS& zNR05WYGRv4KPzu^Pi4~zlExZqFQ-*Vq_vkmFDyLftY$8id^t)J(iTb$kd#;rx_n5&}sTe_O*=ypzSrRq&Pn1 zQ?v;(M7Za$25%4%5lueS-kc?y*mys&x&F8mcvI4l9iI?Y!F#A*HlTGui{g^cLNrAU zY!1X)J8-Mownr2Z%NZz9=E$`?2$PNWZvi&dRWyFP;szsJX9`@SY7qi&CaA?iK3T)T zg!<`UW?s}!JWSjmuX_o@1eC zb~Re;_OQGYIUU4P$D2f^K3=WhUFMLVBZY@c62LyO0N4F!*{ZqXHi%J2=#t(<3ii@F z?)Q^?4zsVqJ>0KCd|s~34!*tbObM*`*-tOI;aYY`&s{9Fa;x*L`1So{{fw%k9?@(R zhh&|`M^Xj4L6DaTWNIoEVyfikG+Wn-rQkWB;jEh9R&g#Y776%~)VvU-yKrDeMy_sQ z(4l7Qqil?W2#k5S|ceZz5-->q9xun|AC7Z;{BsA=-pl z)4N;}=vWo1TbX;mD0=^eLhHEOl<$rG1+zChWi!x8j&b&D!IP>k`ZIl;CVuMlrD_{- zYe$;Q{W7NQ<-^9Vcv9o{e5p|wqrA(ol6i@P;}dn6r!)eqN7m$lQ*Tr}&*ZWBoiz_4 zBi^l6(ACVZUooR$vDpU3Pv{XUY@{6sQW9e>n%l|h3~p*tdz7BiePP9ztF@>Ly*g8c zigoMx5f>#_&&Y~0Ft9xE^vHxz0KhIX0HFAP4OS&a^gkNx??r&5u)f1T-PPfr_Bs#L zvccy-_pVBO-@2H!{9(swB}8hL2bzly1r?C*g9XOAEG#>*@v+^z##@(37M_R|<_QVc zmk8z}hc-=xzy;>qO;)Gx>DpROjzu5VE(gM52#ocAew*kYJj7q6dZ0B^GHOj(N?{2X z>CON&;gRRC8A=R9S4Usk`F+~x>7#N^-{CZK&x)~{)agFl>eeKo@^`x!LotU6om3jc zi8`STt>N1LrQv#^5^OwUdt^M5>yfa6lwaAk>!BiDwq2(q6d3v(R5N(e@73q5B|w9e#8Fw{ z7i=1cCN}+SVj-$n-=n>gBj_7sRnQ<9?L993-YgdMRtwDy1XDMWt%rpn7ib{ocF^J| z*mR(V#ekdZ@2P)7W)Wg_3b-FS)GEy)46L-I%4PmCN?0RRI za|>IG{kYoR7AHgUG7301e0>e;ihrNIx!(S2c-ejWw#Y>FtN@b7uTp5|QWNkdElP=; zn|({#akte`mH{6z0(@2Gr9u(>x>Hy}N##QLzOd?9dwbyaeytGjw)8YpTwcnO`+h_s zdnC}d{(h0mvK_f=BkC(wCt|`1uZzr2=uF#ir)-j%)FWDxreaHutH*W5P7PB&5;ih~ z9W|xl^l9c3^@t#bycP0VZCUJ(LV(>;np}T1d>&%CV>=(IcZxfLCfsoDSO>Rj!IAGRDSb z6Hc+ewyTk42BWY&HZd`=UYcWwhMbiYr81SkMvh>x#9awQQ=Fi8Fi4kAljM%Q+?v@p z#v@%Ov`37nrggsK>|FM$*1%VtCF!yL*d#W-wKb#)L)X-XJQ)juY$gzB?!pIa(QcWC zOenZm;(}ZU71gV0>GFKe?6Z>bSL)8PNLqW0P)jah0Fm9FE6T&OFXkLGZWQ{tU5wls3 z2lC7s=OoTQg1;!DA%+;z4TYB(y4|4#wsgbPTlkgFD|_vvN+e;YfRI06({}q|%-n{6 z4}Fup7U3u?!k|m|(ipXqizrGk#3=G~o}dnvEvtUOwQ&;?$H*k|Rdx?bBe^S24Zk{_ z6}wIkcR1>cI5TZRS*dj(IW$FnxM^&-e>K@U-B*1nFG|O@(h*NpK01Ly@iajRRvuf> zC1@RSR+=`&T*s85LIV{982mky0im9G9^u*{*mjQ$N>D z4lfTv0R0g4a4g(je;a~Uu@0%t`Lv}(E_JX$#X}{UN#^D!PkD?3TsB9a!b@n7+rZhLN23d%}d*10r3D?n?1>K=|ZnaCJ*>2R3% zW^4>3B$b@5Dc4h@Xf`NI3!*|WOo?S>Vbf}|T+fS6M4S~5<_X-dEl-49h|GPZvfIGM zHPDL30|$GmL-b}EX^ez#v<#`@rHL5!m_G_aXK-423SaoOWeSFy~m_$=+Z9%8d9vf zR^_vY>T11@Y1q&t?qaj7(*#hfs`n&DxWRHzPB>!mLb;O{>!x1YvgZ?SkKL=!g>?;e zHbI9R?q3MQvhF7qy>-hg$L4giUDOW*DnZ{FJ~>?qR5JY9lb__9JZw=k@os!LjaRHc zG$_@fWL(i`Js2DF_wPwCtfl2Xpt<)PaPt8>^Y0~6FBXz^ax4Au3(q4XEIfh~j7CY1 z+3p?-dHqyo;=GtR0`F+e<3!sXu#+L-GpEb%Y}299c|YOSJcMOIM`W(RL}jh)gs0J0 z3ur`F67LT#IbVgyj;wNh9fyh)>tFZW55fEPXt2karJL;z;8jh|=9_iNbfFtWuG00m z1bHEV4&iqc?VH5`6mEOWt_Zu&pck$-Y>x(@uj~N`W~h)D-Xu6p*%Mq zQ2RCR>Ggy--tnWjV0wC8OUoO;_z)9s5aOLZR! zla68Y z9qyfrjRnka$2e>_(lwJTJHw|NuBUHqnUwQEW+unpulr6N6A(HAO$QWjcUBXu^ZPPj zEs6szuERR6M8oZ1y;Gs(Y-=*8r)Iu5xM1(}EGi!kU5jFdiwK)M1D7cBk0SPqAn=cD z_=|D@T`$kiD^v1wT6vXSgo`&+G-9KOaw6QV9~&~c?ZAiockG7*1@7+^a%Z%tp#N2+)O zJL?oZ9tvtsrZdzQRMYaTUgz>(t#E`gJWul0{TRxAQ2cC7h^vz+Vi}mZqE19-CLRL& z&Y&kEw&1cflpO4AYsMD~TPbS_p9cxB_x5L1)A1IAA8yAnF`m%P2s_1!u<;0$iN8ad z-X!QEo${e_(Mq(ACl?b)-_Nm}TwEqw&L{Z!Cj5|YC@9tv?fFsIG!7%`$;ZEMOa59^ z(Z-Tgs1hNT1jE);5c~>+VVEI_i`&+McQJ;Jn6$|x)-+}P03SUaAQ>{sKu9xHTUTJN zICp2%oTi$1<<#bIOE#Qdh{iu)qf~ax-n-fTLZa?ws4kSSEMPkAJMuCnXkIAu}i3fxXYfZWTWhXUT^56lFl> z=moq!M?5w(V6~k;c-8cb8gE5Jojtmb1b+Bp++~oSu+W`iba+R~GtW=$7i0Q-A=eC* zc2z4?;@F5$2|MU2IB)FGCTOHW!(osaP4`W>b!sEhSK;+$zo|vYv&TiedEKUH$4faO z_~f!tAs6TdX51GWrmkQkem!bZ&pIexS+AEERPdB*R2?~g@pd#Ou! zDc3`cN-4!mVO>OlEF;IZ#dSW^)NnMURI^X0jy|OFG!)!6H{J#i&xyl1_aE5Xh|H>!-OFMg0ldN z79_91CVj#JV)kx+YLl!kuRE0x$FGal4Xh?{6F0LmNKcEmkNAW`SLsXHghClb`r`<5 zho=8}clsfHY@6 z(BNFYiQJK8aKNx;x-3Qx#VTS9ZtMxt7neOu0Gt@LRP+g~H6oV(O%S8TBRO^%QYF>b zJiS_>H8J@S<h?PGu&`Lrg)eGiI;-hHCe-X|@RNWhO%)IMUKWa-|zkiEJPz~be=Ht74d)@Hp zbUQ}zh^nWq#i8VM)(LN&x%d2qdT5d8Tv8Gl9U0wxj)a9~H{DYpa;6q;wPEdWU>H-q z;O3dc9MY65RIBW`vRkoOwF|XjZWiX@RcwM|$11nVD9f0q&+Y>D zV)_1wtTByAB{3TisI*gq`DBv(1ltdx1w-l+v}O=!H)GkUwozKDXqX4 zU4zT+F>u?w?5gH^r*?}<(9LpQo2=M8^I5_ON>2naSN?-mq%uBG+Qg%CxDsq^sP({6 zeF)!;fhf7JlXe3o4Br!GpcHrk_)HA8yu9_speEVcZ6|vJ(zPr>1>}e4%?l#07O&>^ z_q(m+w_@bHil-sV5ZD=U0#fxj9@o1gp{vR9qb?&jWJ$?%K5GYuIKDY^$?e*2n2h&zEs|( zo2aqhup~M8Sw9rOJnR3ggM7teu|@_;UPuo>0f`;fO6N0k)Ebu3v}UPe?^i2EZr6#i zeIj*5Xw~lr*9KAyq1$ucqwbcjl3W_@^YnniaC1~mNg(v|l;SGfhFBp{9QdC_vBFW- zqe=a`P0~by4Eh}rQu;w!XnjQsu{$!_-YL_FAMUxlaw8ctnDmfc*|*()e!Ff~@XHSe zQu(d$qK?r9A&rqOzCRAP$J6=_6&BRz)Y1Xl)T9)#+sHqh9c>-HO;p(t;17bz){?28 zj`eNd#aV332qznSfP=)D!mYl%23S22eFb>y<(t*!}_L++a*c*I-|JalNMYG(9RyBHWJz26s&lm9Fk}9EhN^ zH{l*0COz%)ma8YCei>aE@wxuO)}w~zrXSd&h-(I^)ENK7O8gSP8c+srj!-N z4)qv@$X_~VZZ30C+Kx%vJ=nS~47WMo%0J>uNmX^9ZVK$I=qqPZcpipEYx%HPI(Dw) zr@@)x(hnb_C$hrKu_gB5Nt#~`uO4+n9<9I9^<+5eX>w4#=&r#!cOp0XsE53`5I%IS zs4!7tY0wyyg|&CyYT3R_zP(On{Z7KSFV=hcL|UDg?FykAm>0Liak}3zY)Yg~^EhpH z&>vZgXxT(Fttw}Fov672WznATBCELgYsLAChg8j2KP%F5rgEk$yPUN27Z4jF7>$yw zIXrh6Cy{v;x;H3U(5qo3%d=>pr?Fh6wq*&uUK=W%*e`xWy-pN2VH?@TPCH~qW058x zcLUnD6ul4~=2T#%895r3Wgq@&tZev5eAvH}LwA(F8|!~ttlx_O6>A%pua^22#!^QA z7_17Qj?L^B?2g|C3!l`=!IxJ~OS+-a6fd*LGzk8P-3&`|fJdsG+xaVBA}ep9qG_%! z0$kyg5TDYBo&{gGsg&lkiG9YMxsU#3jp5-^SOT*?)XCCe8`0!T#v>Vykn9JkNYNlH zQBXW{HV3)J;l0v&*YQSg2oh2J42~L``4?@adGrCC@@s*Tlct6B9NHpxtyR5EF9}^$ zmlDl*ELMN+NH+~R=hq?R(qZxidO9Sc5hJFN_)t-eDJMsek_1|9Y)=jM^Uo_1NOkc9 z#t+cqKS;2vCpIFCUKD)x`Lg?hy#`rGU1dy{ltsnFq6r&prj$gT9gMtIv^_nlp9V7A z!<$p>7#D7cf#T6P4-64g?FZT&0H26o930qqVvkYi2lxa1Oinjn9$Bn;|c2& zx`h4l^mh>{ZRWsPAPcsb4p=xQ=Xk0eiWTGGP@U6Yg^!DgbdbkEg7NM4R~M?Ckr$c% z%;ds*aG>G}kX?6^>NdT<2iHwN>_NBt;bwsL6IFs6YYVtV_|MdvH$&F@Z@l(&>Kz^G zATK5NZ!Q>d%XTh8OAQAIC#NEC;0}{rP0{eu4(cR*o9e_tR#wwOj$n3o$dU#%l?r5? zFN8}PD0x(xtj4}z9Rv72s^BS^*;FbKMmTwjYd=qWcPe+YYk{bC6bI8eIa*55h_QT~ z@#lvq^QJJSrkSmWA%%fpHyTb1Q_ZCp3skGEP^w_dYBJBAu|n zVB$wOH|PbD6tWmqJAOr>io`7Ut$H(ESYez+N zesOke$$w=6$*yTa&e*C6ey;^jy?dcrn_}e zHuJ1we;XY2F-cO%jC$@Grwevyt3Pw~AiK;NU;u&c8A2YlCoIMqjL*6xsSZcdTCaSz zWKZ3H&OVPD_!|n7UhX zLr+1j+!LD?*SFgc$CEA>l9?vYG?*Q37Fm`d!AUpARJDl7^@Zz5ffC7{&d8pL`?`i# z;G0%f<%e$(-)SG_u|QZNs6XV9yLGf@jy?Es?KA4CrHP{%w~u|am-46FtFVV8O;UId z5CyWo>?#}FmE#U-Q}6sB_#l*~-)>lLFwaL>O)KvPcQ{{0`07>CJ|OOn)0@U?kc*mN zoPoZ_)L_JkDCzB`(T5|VoFt9^6j)ja6#%~+2JR=Tr1+U_N-Q!SH+aj8SXOb0%FdK2 zE;84pzM}~uG*Vx`6VI>#^fS-QvUA#mnr9IZf# z>HUHR&SN9V=gU{V`e}DXVBk0{r}688Up{pGh;}60d@T;dpiW?H;Hd#T8BB(DKR2>( zVKBQO1JGIx=%q0(w#*3Zg#wnCYn#Hx3SB9wLKxrEC(57p$B{`qzq}}9@`@>Z4ED+G z==ki4W{whH107cOsYz~w3Dc!0f{*7Y)evHe-=;DZc|&LxAc(+JqBImy;OBo%r^FDUr|i9q~)bjeZLX$z+vK zmCVCMoK~GP1G(01|1W%WXfb9+l@`7b-VT!$OSe z8`<@P(5{m9GUT0r+VRzxf?0rEUWF9Nh zr)KwPUCyYm?4+i4nG#3%@WQX)pm*gA>^)n(gsBn9%UTU-lni_U0t_w)FkVKa;3vU3{+&B5Ilp|EU*IsG3=RreEcW3G^O- z3&sjPfN9)G%^%iwl{g>*Kl!YSOlczHfj7?pT(E=Cu7GAE-jZvTZT`hycv^YsyRysW z_V_^j2#b`=bDz`Z6C%Gm+ltxPawD)>*aK;La*^looFBF%=;T9DVLK*d;oyA>JIj*m zJn&&Lc&D^!>v?bmdUfrjAK{>Tl)x3QAylY7&*5d4>#7a4JtZ(5m7U}~LiqN$OK#3` zz-X^^Sd*jLlpAM7Vlyo+?|$x_9vA1E(V0~C>-dey$1p>V0(~4D?C6^}aPl_;{qbmU zges$lj&Tvpye|Ji#vMzX2C^*VvGD8i5c!|=-n9GuoptYFmFRU#tY> zaQ08y=>|LSF8ABxG+_RrVlZ%Yz`yq-{vPoCxhL_T_YqvAztI29 z|Nk`?{JZ#%82H Date: Thu, 6 Feb 2020 14:28:54 +0800 Subject: [PATCH 24/27] =?UTF-8?q?=E5=B7=A5=E5=85=B7=E7=B1=BB=E6=89=93?= =?UTF-8?q?=E6=88=90jar=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/artifacts/completeUtil.xml | 12 ++ .idea/compiler.xml | 16 ++ .idea/encodings.xml | 6 + .idea/vcs.xml | 6 + properties/service_flow_config.properties | 13 +- .../iie/bolt/proxy/ProxyCompletionBolt.java | 1 + .../bolt/security/SecurityCompletionBolt.java | 3 +- .../cn/ac/iie/utils/general/CompleteUtil.java | 143 +++++++++++++ .../ac/iie/utils/general/TransFormUtils.java | 198 +++--------------- .../cn/ac/iie/utils/hbase/HBaseUtils.java | 110 +++++++--- .../cn/ac/iie/utils/json/JsonParseUtil.java | 48 ++++- .../cn/ac/iie/utils/system/SnowflakeId.java | 60 ++---- .../iie/utils/zookeeper/DistributedLock.java | 26 +-- .../ac/iie/utils/zookeeper/ZooKeeperLock.java | 50 +---- .../iie/utils/zookeeper/ZookeeperUtils.java | 38 ++-- 15 files changed, 394 insertions(+), 336 deletions(-) create mode 100644 .idea/artifacts/completeUtil.xml create mode 100644 .idea/compiler.xml create mode 100644 .idea/encodings.xml create mode 100644 .idea/vcs.xml create mode 100644 src/main/java/cn/ac/iie/utils/general/CompleteUtil.java diff --git a/.idea/artifacts/completeUtil.xml b/.idea/artifacts/completeUtil.xml new file mode 100644 index 0000000..f6f9995 --- /dev/null +++ b/.idea/artifacts/completeUtil.xml @@ -0,0 +1,12 @@ + + + $PROJECT_DIR$/out/artifacts/completeUtil + + + + + + + + + \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..7603751 --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..b26911b --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index 1e1bf00..90b80e1 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -1,13 +1,14 @@ #管理kafka地址 -bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092 -#bootstrap.servers=192.168.40.186:9092 +#bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092 +bootstrap.servers=192.168.40.151:9092 #zookeeper 地址 -zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 +zookeeper.servers=192.168.40.151:2181 +#zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 #hbase zookeeper地址 #hbase.zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181 -hbase.zookeeper.servers=192.168.40.224:2182 +hbase.zookeeper.servers=192.168.40.151:2181 #hbase tablename hbase.table.name=subscriber_info @@ -19,7 +20,7 @@ auto.offset.reset=latest kafka.topic=SECURITY-EVENT-LOG #读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； -group.id=security-policy-191216 +group.id=security-policy-200204 #输出topic results.output.topic=SECURITY-EVENT-COMPLETED-LOG @@ -43,7 +44,7 @@ ip.library=D:\\dat\\ #kafka批量条数 batch.insert.num=2000 #网关的schema位置 -schema.http=http://192.168.40.224:9999/metadata/schema/v1/fields/security_event_log +schema.http=http://192.168.40.151:9999/metadata/schema/v1/fields/security_event_log #数据中心（UID） data.center.id.num=15 diff --git a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java index f89801a..56f8f8b 100644 --- a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java @@ -1,6 +1,7 @@ package cn.ac.iie.bolt.proxy; import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.system.TupleUtils; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; diff --git a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java index 9a6b303..7a4182e 100644 --- a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java @@ -39,7 +39,8 @@ public class SecurityCompletionBolt extends BaseBasicBolt { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { try { if (TupleUtils.isTick(tuple)) { - HBaseUtils.change(); + HBaseUtils hBaseUtils = new HBaseUtils(FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS, FlowWriteConfig.HBASE_TABLE_NAME); + hBaseUtils.change(); } else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { diff --git a/src/main/java/cn/ac/iie/utils/general/CompleteUtil.java b/src/main/java/cn/ac/iie/utils/general/CompleteUtil.java new file mode 100644 index 0000000..b786a4a --- /dev/null +++ b/src/main/java/cn/ac/iie/utils/general/CompleteUtil.java @@ -0,0 +1,143 @@ +package cn.ac.iie.utils.general; + +import cn.ac.iie.common.FlowWriteConfig; +import cn.ac.iie.utils.hbase.HBaseUtils; +import cn.ac.iie.utils.system.SnowflakeId; +import com.google.common.net.InternetDomainName; +import com.zdjizhi.utils.IpLookup; +import com.zdjizhi.utils.StringUtil; +import org.apache.log4j.Logger; + +import java.util.Base64; + + +public final class CompleteUtil { + + private static Logger logger = Logger.getLogger(TransFormUtils.class); + private static IpLookup ipLookup = new IpLookup.Builder(false) + .loadDataFileV4(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") + .loadDataFileV6(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") + .loadAsnDataFileV4(FlowWriteConfig.IP_LIBRARY + "asn_v4.mmdb") + .loadAsnDataFileV6(FlowWriteConfig.IP_LIBRARY + "asn_v6.mmdb") + .build(); + + /** + * 有sni通过sni获取域名，有host根据host获取域名 + * + * @param sni sni + * @param host host + * @return 顶级域名 + */ + public static String getTopDomain(String sni, String host) { + if (StringUtil.isNotBlank(host)) { + return getDomainName(host); + } else if (StringUtil.isNotBlank(sni)) { + return getDomainName(sni); + } else { + return ""; + } + } + + + /** + * 根据url截取顶级域名 + * + * @param host 网站url + * @return 顶级域名 + */ + public static String getDomainName(String host) { + String domain = ""; + try { + domain = InternetDomainName.from(host).topPrivateDomain().toString(); + } catch (Exception e) { + logger.error("host解析顶级域名异常: " + e.getMessage()); + } + return domain; + } + + /** + * 生成当前时间戳的操作 + */ + public static int getCurrentTime() { + return (int)(System.currentTimeMillis() / 1000); + } + + /** + * 雪花模型生成id + * + * @return + */ + public static long getSnowflakeId(String zookeeperIp, String kafkaTopic, long dataCenterIdNum) { + + return SnowflakeId.generateId(zookeeperIp,kafkaTopic,dataCenterIdNum); + } + + /** + * 根据clientIp获取location信息 + * + * @param ip + * @return + */ + public static String getGeoIpDetail(String ip) { + + return ipLookup.cityLookupDetail(ip); + } + + /** + * 根据ip获取asn信息 + * + * @param ip + * @return asn + */ + public static String getGeoAsn(String ip) { + + return ipLookup.asnLookup(ip, true); + } + + /** + * 根据ip获取country信息 + * + * @param ip + * @return country + */ + public static String getGeoIpCountry(String ip) { + + return ipLookup.countryLookup(ip); + } + + /** + * 根据ip去hbase中匹配对应的用户名 + * @param clientIp + * @param hbaseZookeeper + * @param hbaseTable + * @return 用户名 subscriber_id + */ + public static String radiusMatch(String clientIp, String hbaseZookeeper, String hbaseTable) { + return HBaseUtils.getAccount(clientIp,hbaseZookeeper,hbaseTable); + } + + + /** + * base64 解码 + * + * @param encodedText mail subject + * @param subjectCharset 编码格式 + * @return 解码内容 / 空 + */ + public static String base64Str(String encodedText, String subjectCharset) { + Base64.Decoder decoder = Base64.getDecoder(); + String sub; + try { + if (StringUtil.isNotBlank(subjectCharset)) { + sub = new String(decoder.decode(encodedText), subjectCharset); + } else { + sub = new String(decoder.decode(encodedText), "UTF-8"); + } + return sub; + } catch (Exception e) { + logger.error("transform base64 String failed! base64Str = " + encodedText + "Charset = " + subjectCharset + "error :" + e); + return ""; + } + } + +} diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index 0f66dc5..fb028eb 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -15,6 +15,8 @@ import org.junit.jupiter.api.Test; import java.util.*; import java.util.regex.Pattern; +import static cn.ac.iie.utils.general.CompleteUtil.*; + /** * 描述:转换或补全工具类 @@ -25,16 +27,6 @@ import java.util.regex.Pattern; public class TransFormUtils { private static Logger logger = Logger.getLogger(TransFormUtils.class); - private final static Set PUBLIC_SUFFIX_SET = new HashSet( - Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se" - .split("\\|"))); - private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})"); - private static IpLookup ipLookup = new IpLookup.Builder(false) - .loadDataFileV4(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") - .loadDataFileV6(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") - .loadAsnDataFileV4(FlowWriteConfig.IP_LIBRARY + "asn_v4.mmdb") - .loadAsnDataFileV6(FlowWriteConfig.IP_LIBRARY + "asn_v6.mmdb") - .build(); //在内存中加载反射类用的map private static HashMap map = JsonParseUtil.getMapFromhttp(FlowWriteConfig.SCHEMA_HTTP); @@ -45,55 +37,53 @@ public class TransFormUtils { /** * 解析日志，并补全 - * 补domain,补subscriber_id * - * @param message Security原始日志 + * @param message kafka Topic原始日志 * @return 补全后的日志 - *

- * current_timestamp - * snowflake_id - * geo_ip_detail - * geo_asn - * radius_match - * geo_ip_country - * geo_asn - * sub_domain - * sub_domain */ public static String dealCommonMessage(String message) { -// message="{\"ssl_sni\":\"pos.baidu.com\",\"ssl_version\":\"v3\",\"ssl_cn\":\"baidu.com\",\"ssl_san\":\"baidu.com;click.hm.baidu.com;cm.pos.baidu.com;log.hm.baidu.com;update.pan.baidu.com;wn.pos.baidu.com;*.91.com;*.aipage.cn;*.aipage.com;*.apollo.auto;*.baidu.com;*.baidubce.com;*.baiducontent.com;*.baidupcs.com;*.baidustatic.com;*.baifubao.com;*.bce.baidu.com;*.bcehost.com;*.bdimg.com;*.bdstatic.com;*.bdtjrcv.com;*.bj.baidubce.com;*.chuanke.com;*.dlnel.com;*.dlnel.org;*.dueros.baidu.com;*.eyun.baidu.com;*.fanyi.baidu.com;*.gz.baidubce.com;*.hao123.baidu.com;*.hao123.com;*.hao222.com;*.haokan.com;*.im.baidu.com;*.map.baidu.com;*.mbd.baidu.com;*.mipcdn.com;*.news.baidu.com;*.nuomi.com;*.safe.baidu.com;*.smartapps.cn;*.su.baidu.com;*.trustgo.com;*.xueshu.baidu.com;apollo.auto;baifubao.com;dwz.cn;mct.y.nuomi.com;www.baidu.cn;www.baidu.com.cn\",\"common_schema_type\":\"SSL\",\"common_server_ip\":\"182.61.200.109\",\"common_client_ip\":\"192.168.50.144\",\"common_server_port\":443,\"common_client_port\":50529,\"common_stream_dir\":3,\"common_address_type\":4,\"common_s2c_pkt_num\":46,\"common_s2c_byte_num\":33149,\"common_c2s_pkt_num\":23,\"common_c2s_byte_num\":6147,\"common_start_time\":1576744784,\"common_end_time\":1576744799,\"common_con_duration_ms\":15000,\"common_stream_trace_id\":7686307990192,\"common_l4_protocol\":\"IPv4_TCP\",\"common_address_list\":\"50529-443-192.168.50.144-182.61.200.109\",\"common_sled_ip\":\"192.168.40.21\",\"common_policy_id\":172,\"common_service\":0,\"common_action\":2,\"common_user_region\":\"{\\\"protocol\\\":\\\"SSL\\\",\\\"protocol_version\\\":{\\\"allow_http2\\\":1,\\\"min\\\":\\\"ssl3\\\",\\\"max\\\":\\\"tls13\\\",\\\"mirror_client\\\":1},\\\"dynamic_bypass\\\":{\\\"mutual_authentication\\\":1,\\\"cert_pinning\\\":1,\\\"cert_transparency\\\":0,\\\"protocol_errors\\\":1,\\\"ev_cert\\\":0},\\\"decrypt_mirror\\\":{\\\"enable\\\":0},\\\"certificate_checks\\\":{\\\"fail_action\\\":\\\"pass-through\\\",\\\"approach\\\":{\\\"self-signed\\\":1,\\\"expiration\\\":1,\\\"cn\\\":1,\\\"issuer\\\":1}},\\\"keyring\\\":1}\"}"; Object object = JSONObject.parseObject(message, mapObject.getClass()); // System.out.println("补全之前 ===》 "+JSON.toJSONString(object)); try { for (String[] strings : jobList) { - //参数的值 - Object use = JsonParseUtil.getValue(object, strings[0]); - //补全的字段的值 + //用到的参数的值 + Object name = JsonParseUtil.getValue(object, strings[0]); + //需要补全的字段的值 Object appendTo = JsonParseUtil.getValue(object, strings[1]); + //匹配操作函数的字段 + String function=strings[2]; + //额外的参数的值 + Object param = null; + if (strings[3] != null){ + param=JsonParseUtil.getValue(object, strings[3]); + } - if (strings[2].equals("current_timestamp")) { + + if (function.equals("current_timestamp")) { JsonParseUtil.setValue(object, strings[1], getCurrentTime()); - } else if (strings[2].equals("snowflake_id")) { - JsonParseUtil.setValue(object, strings[1], getSnowflakeId()); - } else if (strings[2].equals("geo_ip_detail")) { - JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(use.toString())); - } else if (strings[2].equals("geo_asn")) { - JsonParseUtil.setValue(object, strings[1], getGeoAsn(use.toString())); - } else if (strings[2].equals("radius_match")) { - JsonParseUtil.setValue(object, strings[1], radiusMatch(use.toString())); - } else if (strings[2].equals("geo_ip_country")) { - JsonParseUtil.setValue(object, strings[1], getGeoIpCountry(use.toString())); - } else if (strings[0].equals("http_host") && strings[2].equals("sub_domain") && use != null) { + } else if (function.equals("snowflake_id")) { + JsonParseUtil.setValue(object, strings[1], getSnowflakeId(FlowWriteConfig.ZOOKEEPER_SERVERS,FlowWriteConfig.KAFKA_TOPIC,FlowWriteConfig.DATA_CENTER_ID_NUM)); + } else if (function.equals("geo_ip_detail")) { + JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(name.toString())); + } else if (function.equals("geo_asn")) { + JsonParseUtil.setValue(object, strings[1], getGeoAsn(name.toString())); + } else if (function.equals("radius_match")) { + JsonParseUtil.setValue(object, strings[1], radiusMatch(name.toString(),FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS,FlowWriteConfig.HBASE_TABLE_NAME)); + } else if (function.equals("geo_ip_country")) { + JsonParseUtil.setValue(object, strings[1], getGeoIpCountry(name.toString())); + } else if (function.equals("decode_of_base64") && param != null){ + JsonParseUtil.setValue(object, strings[1], base64Str(name.toString(),param.toString())); + } else if (name.equals("http_host") && function.equals("sub_domain")) { if (appendTo == null || StringUtil.isBlank(appendTo.toString())) { - JsonParseUtil.setValue(object, strings[1], getTopDomain(null, use.toString())); + JsonParseUtil.setValue(object, strings[1], getTopDomain(null, name.toString())); } - } else if (strings[0].equals("ssl_sni") && strings[2].equals("sub_domain") && use != null) { + } else if (name.equals("ssl_sni") && strings[2].equals("sub_domain")) { if (appendTo == null || StringUtil.isBlank(appendTo.toString())) { - JsonParseUtil.setValue(object, strings[1], getTopDomain(use.toString(), null)); + JsonParseUtil.setValue(object, strings[1], getTopDomain(name.toString(), null)); } } @@ -114,131 +104,9 @@ public class TransFormUtils { @Test public void aaa() { - String sni = "203.187.160.131:9011"; + String sni = "www.baidu.com"; System.out.println(getTopDomain(sni, null)); System.out.println(getTopDomain(null,sni)); } - - /** - * 有sni通过sni获取域名，有host根据host获取域名 - * - * @param sni sni - * @param host host - * @return 顶级域名 - */ - private static String getTopDomain(String sni, String host) { - if (StringUtil.isNotBlank(host)) { - return getDomainName(host); - } else if (StringUtil.isNotBlank(sni)) { - return getDomainName(sni); - } else { - return ""; - } - } - - - /** - * 根据url截取顶级域名 - * - * @param host 网站url - * @return 顶级域名 - */ - private static String getDomainName(String host) { - String domain = ""; - try { - domain = InternetDomainName.from(host).topPrivateDomain().toString(); - } catch (Exception e) { - logger.error("host解析顶级域名异常: " + e.getMessage()); - } - return domain; - } - - /** - * 生成当前时间戳的操作 - */ - private static int getCurrentTime() { - return (int)(System.currentTimeMillis() / 1000); - } - - /** - * 雪花模型生成id - * - * @return - */ - private static long getSnowflakeId() { - - return SnowflakeId.generateId(); - } - - /** - * 根据clientIp获取location信息 - * - * @param ip - * @return - */ - private static String getGeoIpDetail(String ip) { - - return ipLookup.cityLookupDetail(ip); - } - - /** - * 根据ip获取asn信息 - * - * @param ip - * @return - */ - private static String getGeoAsn(String ip) { - - return ipLookup.asnLookup(ip, true); - } - - /** - * 根据ip获取country信息 - * - * @param ip - * @return - */ - private static String getGeoIpCountry(String ip) { - - return ipLookup.countryLookup(ip); - } - - /** - * radius借助hbase补齐 - * - * @param ip - * @return - */ - private static String radiusMatch(String ip) { - return HBaseUtils.getAccount(ip); - } - - /** - * switch 匹配合适的方法 - * current_timestamp - * snowflake_id - * geo_ip_detail - * geo_asn - * radius_match - * geo_ip_country - * geo_asn - * sub_domain - * sub_domain - * @param func - */ -//TODO 行不通的原因是无法统一一个确定的返回值类型 - /* private static String switchFunc(String func){ - switch (func){ - case "current_timestamp": - return String.valueOf(getCurrentTime()); - case "snowflake_id": - case "geo_ip_detail": - case "geo_asn": - case "radius_match": - case "geo_ip_country": - case "sub_domain": - } - return func; - }*/ -} +} \ No newline at end of file diff --git a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java index f386003..5c0e4c9 100644 --- a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java +++ b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java @@ -1,8 +1,5 @@ package cn.ac.iie.utils.hbase; -import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.system.IpUtils; -import io.netty.util.collection.IntObjectHashMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; @@ -15,36 +12,55 @@ import org.apache.log4j.Logger; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentSkipListMap; - - -/** - * HBase 工具类 - * - * @author qidaijie - */ +import java.util.Timer; +import java.util.TimerTask; public class HBaseUtils { private final static Logger logger = Logger.getLogger(HBaseUtils.class); private static Map subIdMap = new HashMap<>(333334); - // private static Map subIdMap = new ConcurrentSkipListMap<>(); private static Connection connection; private static Long time; - static { - // 管理Hbase的配置信息 - Configuration configuration = HBaseConfiguration.create(); - // 设置zookeeper节点 - configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); - configuration.set("hbase.client.retries.number", "3"); - configuration.set("hbase.bulkload.retries.number", "3"); - configuration.set("zookeeper.recovery.retry", "3"); + private static String zookeeperIp; + private static String hbaseTable; + + private static HBaseUtils hBaseUtils; + + private static void getHbaseInstance(String zookeeperServer, String hbaseTableName) { + hBaseUtils = new HBaseUtils(zookeeperServer, hbaseTableName); + } + + /** + * 构造函数-新-20191023 + */ + public HBaseUtils(String zookeeperServer, String hbaseTableName) { + zookeeperIp = zookeeperServer; + hbaseTable = hbaseTableName; + //获取连接 + getHbaseConn(); + //拉取所有 + getAll(); + //定时更新 + updateHabaseCache(); + } + + private static void getHbaseConn() { try { + // 管理Hbase的配置信息 + Configuration configuration = HBaseConfiguration.create(); + // 设置zookeeper节点 + configuration.set("hbase.zookeeper.quorum", zookeeperIp); + configuration.set("hbase.client.retries.number", "3"); + configuration.set("hbase.bulkload.retries.number", "3"); + configuration.set("zookeeper.recovery.retry", "3"); connection = ConnectionFactory.createConnection(configuration); time = System.currentTimeMillis(); - getAll(); - } catch (IOException e) { - logger.error("获取HBase连接失败"); + logger.warn("HBaseUtils get HBase connection,now to getAll()."); + } catch (IOException ioe) { + logger.error("HBaseUtils getHbaseConn() IOException===>{" + ioe + "}<==="); + ioe.printStackTrace(); + } catch (Exception e) { + logger.error("HBaseUtils getHbaseConn() Exception===>{" + e + "}<==="); e.printStackTrace(); } } @@ -53,7 +69,7 @@ public class HBaseUtils { * 更新变量 */ public static void change() { - Long nowTime = System.currentTimeMillis(); + long nowTime = System.currentTimeMillis(); timestampsFilter(time - 1000, nowTime + 500); } @@ -69,7 +85,7 @@ public class HBaseUtils { ResultScanner scanner = null; Scan scan2 = new Scan(); try { - table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); + table = connection.getTable(TableName.valueOf("sub:" + hbaseTable)); scan2.setTimeRange(startTime, endTime); scanner = table.getScanner(scan2); for (Result result : scanner) { @@ -87,10 +103,14 @@ public class HBaseUtils { } } Long end = System.currentTimeMillis(); - logger.warn("当前集合长度" + subIdMap.keySet().size()); - logger.warn("更新缓存耗时：" + (end - begin) + "开始时间：" + begin + "结束时间：" + end); + logger.warn("HBaseUtils Now subIdMap.keySet().size() is: " + subIdMap.keySet().size()); + logger.warn("HBaseUtils Update cache timeConsuming is: " + (end - begin) + ",BeginTime: " + begin + ",EndTime: " + end); time = endTime; - } catch (IOException e) { + } catch (IOException ioe) { + logger.error("HBaseUtils timestampsFilter is IOException===>{" + ioe + "}<==="); + ioe.printStackTrace(); + } catch (Exception e) { + logger.error("HBaseUtils timestampsFilter is Exception===>{" + e + "}<==="); e.printStackTrace(); } finally { if (scanner != null) { @@ -110,9 +130,9 @@ public class HBaseUtils { * 获取所有的 key value */ private static void getAll() { - Long begin = System.currentTimeMillis(); + long begin = System.currentTimeMillis(); try { - Table table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); + Table table = connection.getTable(TableName.valueOf("sub:" + hbaseTable)); Scan scan2 = new Scan(); ResultScanner scanner = table.getScanner(scan2); for (Result result : scanner) { @@ -121,10 +141,14 @@ public class HBaseUtils { subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); } } - logger.warn("获取全量后集合长度：" + subIdMap.size()); - logger.warn("获取全量耗时：" + (System.currentTimeMillis() - begin)); + logger.warn("HBaseUtils Get fullAmount List size->subIdMap.size(): " + subIdMap.size()); + logger.warn("HBaseUtils Get fullAmount List size->subIdMap.size() timeConsuming is: " + (System.currentTimeMillis() - begin)); scanner.close(); - } catch (IOException e) { + } catch (IOException ioe) { + logger.error("HBaseUtils getAll() is IOException===>{" + ioe + "}<==="); + ioe.printStackTrace(); + } catch (Exception e) { + logger.error("HBaseUtils getAll() is Exception===>{" + e + "}<==="); e.printStackTrace(); } } @@ -135,8 +159,26 @@ public class HBaseUtils { * @param clientIp client_ip * @return account */ - public static String getAccount(String clientIp) { + public static String getAccount(String clientIp, String hbaseZookeeper, String hbaseTable) { + if (hBaseUtils == null) { + getHbaseInstance(hbaseZookeeper, hbaseTable); + } return subIdMap.get(clientIp); } + private void updateHabaseCache() { + Timer timer = new Timer(); + timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + try { + change(); + } catch (Exception e) { + logger.error("HBaseUtils update hbaseCache is error===>{" + e + "}<==="); + e.printStackTrace(); + } + } + }, 1, 1000 * 60); + } + } diff --git a/src/main/java/cn/ac/iie/utils/json/JsonParseUtil.java b/src/main/java/cn/ac/iie/utils/json/JsonParseUtil.java index 3d9ede5..9ef09fc 100644 --- a/src/main/java/cn/ac/iie/utils/json/JsonParseUtil.java +++ b/src/main/java/cn/ac/iie/utils/json/JsonParseUtil.java @@ -145,6 +145,7 @@ public class JsonParseUtil { /** * 根据http链接获取schema，解析之后返回一个任务列表 (useList toList funcList) + * * @param http * @return */ @@ -167,25 +168,56 @@ public class JsonParseUtil { Object format = JSON.parseObject(doc.toString()).get("format"); if (format != null) { - Object functions = JSON.parseObject(format.toString()).get("functions"); - Object appendTo = JSON.parseObject(format.toString()).get("appendTo"); - if (appendTo != null) { - String[] functionArray = functions.toString().split(","); - String[] appendToArray = appendTo.toString().split(","); + String functions = null; + String appendTo = null; + String params = null; + Object functionsObj = JSON.parseObject(format.toString()).get("functions"); + Object appendToObj = JSON.parseObject(format.toString()).get("appendTo"); + Object paramObj = JSON.parseObject(format.toString()).get("param"); + + if (functionsObj != null) { + functions = functionsObj.toString(); + } + + if (appendToObj != null) { + appendTo = appendToObj.toString(); + } + if (paramObj != null) { + params = paramObj.toString(); + } + + + if (appendTo != null && params == null) { + String[] functionArray = functions.split(","); + String[] appendToArray = appendTo.split(","); for (int i = 0; i < functionArray.length; i++) { // useList.add(name); // toList.add(appendToArray[i]); // funcList.add(functionArray[i]); - list.add(new String[]{name, appendToArray[i], functionArray[i]}); + list.add(new String[]{name, appendToArray[i], functionArray[i],null}); } - } else { + }else if (appendTo != null && params != null){ + String[] functionArray = functions.split(","); + String[] appendToArray = appendTo.split(","); + String[] paramArray = params.split(","); + + for (int i = 0; i < functionArray.length; i++) { +// useList.add(name); +// toList.add(appendToArray[i]); +// funcList.add(functionArray[i]); + list.add(new String[]{name, appendToArray[i], functionArray[i],paramArray[i]}); + + } + } + + else { // useList.add(name); // funcList.add(functions.toString()); // toList.add(name); - list.add(new String[]{name, name, functions.toString()}); + list.add(new String[]{name, name, functions,params}); } } diff --git a/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java b/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java index 5c3462d..8d67348 100644 --- a/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java +++ b/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java @@ -1,20 +1,13 @@ package cn.ac.iie.utils.system; -import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.zookeeper.DistributedLock; import cn.ac.iie.utils.zookeeper.ZooKeeperLock; import cn.ac.iie.utils.zookeeper.ZookeeperUtils; import org.apache.log4j.Logger; -/** - * 雪花算法 - * - * @author qidaijie - */ + public class SnowflakeId { private static Logger logger = Logger.getLogger(SnowflakeId.class); - // ==============================Fields=========================================== /** * 开始时间截 (2018-08-01 00:00:00) max 17years */ @@ -31,7 +24,8 @@ public class SnowflakeId { private final long dataCenterIdBits = 4L; /** - * 支持的最大机器id，结果是3 (这个移位算法可以很快的计算出几位二进制数所能表示的最大十进制数) + * 支持的最大机器id，结果是63 (这个移位算法可以很快的计算出几位二进制数所能表示的最大十进制数) + * M << n = M * 2^n */ private final long maxWorkerId = -1L ^ (-1L << workerIdBits); @@ -90,53 +84,36 @@ public class SnowflakeId { private static ZookeeperUtils zookeeperUtils = new ZookeeperUtils(); - static { - idWorker = new SnowflakeId(); + private static void getSnowflakeldInstance(String zookeeperIp, String kafkaTopic, long dataCenterIdNum) { + idWorker = new SnowflakeId(zookeeperIp, kafkaTopic, dataCenterIdNum); } - //==============================Constructors===================================== - /** * 构造函数 */ -// private SnowflakeId() { -// DistributedLock lock = new DistributedLock(FlowWriteConfig.ZOOKEEPER_SERVERS, "disLocks1"); -// lock.lock(); -// int tmpWorkerId = zookeeperUtils.modifyNode("/Snowflake/" + FlowWriteConfig.KAFKA_TOPIC); -// if (tmpWorkerId > maxWorkerId || tmpWorkerId < 0) { -// throw new IllegalArgumentException(String.format("worker Id can't be greater than %d or less than 0", maxWorkerId)); -// } -// int dataCenterId = FlowWriteConfig.DATA_CENTER_ID_NUM; -// if (dataCenterId > maxDataCenterId || dataCenterId < 0) { -// throw new IllegalArgumentException(String.format("datacenter Id can't be greater than %d or less than 0", maxDataCenterId)); -// } -// this.workerId = tmpWorkerId; -// this.dataCenterId = dataCenterId; -// } - - private SnowflakeId() { - ZooKeeperLock lock = new ZooKeeperLock(FlowWriteConfig.ZOOKEEPER_SERVERS, "/locks", "disLocks"); + private SnowflakeId(String zookeeperIp, String kafkaTopic, long dataCenterIdNum) { + ZooKeeperLock lock = new ZooKeeperLock(zookeeperIp, "/locks", "disLocks1"); if (lock.lock()) { - int tmpWorkerId = zookeeperUtils.modifyNode("/Snowflake/" + FlowWriteConfig.KAFKA_TOPIC); + int tmpWorkerId = zookeeperUtils.modifyNode("/Snowflake/" + kafkaTopic, zookeeperIp); if (tmpWorkerId > maxWorkerId || tmpWorkerId < 0) { throw new IllegalArgumentException(String.format("worker Id can't be greater than %d or less than 0", maxWorkerId)); } - int dataCenterId = FlowWriteConfig.DATA_CENTER_ID_NUM; - if (dataCenterId > maxDataCenterId || dataCenterId < 0) { - throw new IllegalArgumentException(String.format("datacenter Id can't be greater than %d or less than 0", maxDataCenterId)); + if (dataCenterIdNum > maxDataCenterId || dataCenterIdNum < 0) { + throw new IllegalArgumentException(String.format("datacenter Id can't be greater than %d or less than ", maxDataCenterId)); } this.workerId = tmpWorkerId; - this.dataCenterId = dataCenterId; + this.dataCenterId = dataCenterIdNum; try { lock.unlock(); - } catch (InterruptedException e) { + } catch (InterruptedException ie) { + ie.printStackTrace(); + } catch (Exception e) { e.printStackTrace(); + logger.error("This is not usual error!!!===>>>" + e + "<<<==="); } } } - // ==============================Methods========================================== - /** * 获得下一个ID (该方法是线程安全的) * @@ -204,9 +181,10 @@ public class SnowflakeId { * * @return */ - public static Long generateId() { + public static Long generateId(String zookeeperIp, String kafkaTopic, long dataCenterIdNum) { + if (idWorker == null) { + getSnowflakeldInstance(zookeeperIp, kafkaTopic, dataCenterIdNum); + } return idWorker.nextId(); } - - } \ No newline at end of file diff --git a/src/main/java/cn/ac/iie/utils/zookeeper/DistributedLock.java b/src/main/java/cn/ac/iie/utils/zookeeper/DistributedLock.java index f85a84e..46a7ff2 100644 --- a/src/main/java/cn/ac/iie/utils/zookeeper/DistributedLock.java +++ b/src/main/java/cn/ac/iie/utils/zookeeper/DistributedLock.java @@ -1,7 +1,5 @@ package cn.ac.iie.utils.zookeeper; -import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.system.SnowflakeId; import org.apache.log4j.Logger; import org.apache.zookeeper.*; import org.apache.zookeeper.data.Stat; @@ -15,9 +13,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; -/** - * @author qidaijie - */ + public class DistributedLock implements Lock, Watcher { private static Logger logger = Logger.getLogger(DistributedLock.class); @@ -83,7 +79,7 @@ public class DistributedLock implements Lock, Watcher { } try { if (this.tryLock()) { - System.out.println(Thread.currentThread().getName() + " " + lockName + "获得了锁"); + logger.warn(Thread.currentThread().getName() + " " + lockName + " is being locked......"); } else { // 等待锁 waitForLock(waitLock, sessionTimeout); @@ -98,7 +94,7 @@ public class DistributedLock implements Lock, Watcher { try { String splitStr = "_lock_"; if (lockName.contains(splitStr)) { - throw new LockException("锁名有误"); + throw new LockException("locked name is error!!!"); } // 创建临时有序节点 currentLock = zk.create(ROOT_LOCK + "/" + lockName + splitStr, new byte[0], @@ -187,20 +183,4 @@ public class DistributedLock implements Lock, Watcher { super(e); } } - - public static void main(String[] args) { - ZookeeperUtils zookeeperUtils = new ZookeeperUtils(); - Runnable runnable = new Runnable() { - @Override - public void run() { - DistributedLock lock = new DistributedLock(FlowWriteConfig.ZOOKEEPER_SERVERS, "disLocks1"); - lock.lock(); - lock.unlock(); - } - }; - for (int i = 0; i < 10; i++) { - Thread t = new Thread(runnable); - t.start(); - } - } } \ No newline at end of file diff --git a/src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java b/src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java index 6cff343..8d0a4d5 100644 --- a/src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java +++ b/src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java @@ -1,5 +1,6 @@ package cn.ac.iie.utils.zookeeper; +import org.apache.log4j.Logger; import org.apache.zookeeper.*; import org.apache.zookeeper.data.Stat; @@ -11,6 +12,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; public class ZooKeeperLock implements Watcher { + private static Logger logger = Logger.getLogger(ZooKeeperLock.class); + private ZooKeeper zk = null; private String rootLockNode; // 锁的根节点 private String lockName; // 竞争资源，用来生成子节点名称 @@ -33,13 +36,14 @@ public class ZooKeeperLock implements Watcher { } } catch (IOException | InterruptedException | KeeperException e) { e.printStackTrace(); + logger.error("ZooKeeperLock Constructors ===>>>Node already exists!"); } } // 2. 加锁方法，先尝试加锁，不能加锁则等待上一个锁的释放 public boolean lock() { if (this.tryLock()) { - System.out.println("线程【" + Thread.currentThread().getName() + "】加锁（" + this.currentLock + "）成功！"); + logger.warn("ZooKeeperLock method lock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] addZkLock(" + this.currentLock + ")success！"); return true; } else { return waitOtherLock(this.waitLock, this.sessionTimeout); @@ -56,8 +60,7 @@ public class ZooKeeperLock implements Watcher { // 创建锁节点（临时有序节点） this.currentLock = zk.create(this.rootLockNode + "/" + this.lockName + split, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL); - System.out.println("线程【" + Thread.currentThread().getName() - + "】创建锁节点（" + this.currentLock + "）成功，开始竞争..."); + logger.warn("ZooKeeperLock method tryLock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] create zkLockNode(" + this.currentLock + ")success，begin to election..."); // 取所有子节点 List nodes = zk.getChildren(this.rootLockNode, false); // 取所有竞争lockName的锁 @@ -90,18 +93,15 @@ public class ZooKeeperLock implements Watcher { String waitLockNode = this.rootLockNode + "/" + waitLock; Stat stat = zk.exists(waitLockNode, true); if (null != stat) { - System.out.println("线程【" + Thread.currentThread().getName() - + "】锁（" + this.currentLock + "）加锁失败，等待锁（" + waitLockNode + "）释放..."); + logger.error("ZooKeeperLock method waitOtherLock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] zkLock(" + this.currentLock + ")addZkLock fail,wait lock(" + waitLockNode + ")release..."); // 设置计数器，使用计数器阻塞线程 this.countDownLatch = new CountDownLatch(1); islock = this.countDownLatch.await(sessionTimeout, TimeUnit.MILLISECONDS); this.countDownLatch = null; if (islock) { - System.out.println("线程【" + Thread.currentThread().getName() + "】锁（" - + this.currentLock + "）加锁成功，锁（" + waitLockNode + "）已经释放"); + logger.warn("ZooKeeperLock method waitOtherLock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] zkLock(" + this.currentLock + ")addZkLock success,lock(" + waitLockNode + ")release over."); } else { - System.out.println("线程【" + Thread.currentThread().getName() + "】锁（" - + this.currentLock + "）加锁失败..."); + logger.error("ZooKeeperLock method waitOtherLock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] zkLock(" + this.currentLock + "addZkLock fail..."); } } else { islock = true; @@ -117,7 +117,7 @@ public class ZooKeeperLock implements Watcher { try { Stat stat = zk.exists(this.currentLock, false); if (null != stat) { - System.out.println("线程【" + Thread.currentThread().getName() + "】释放锁 " + this.currentLock); + logger.warn("ZooKeeperLock method unlock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] free zkLock " + this.currentLock); zk.delete(this.currentLock, -1); this.currentLock = null; } @@ -137,34 +137,4 @@ public class ZooKeeperLock implements Watcher { } } - - public static void doSomething() { - - } - - public static void main(String[] args) { - Runnable runnable = new Runnable() { - @Override - public void run() { - ZooKeeperLock lock = null; - lock = new ZooKeeperLock("192.168.40.119:2181", "/locks", "test1"); - if (lock.lock()) { - doSomething(); - try { -// Thread.sleep(1000); - lock.unlock(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - }; - - for (int i = 0; i < 10; i++) { - Thread t = new Thread(runnable); - t.start(); - } - } - - } diff --git a/src/main/java/cn/ac/iie/utils/zookeeper/ZookeeperUtils.java b/src/main/java/cn/ac/iie/utils/zookeeper/ZookeeperUtils.java index 639b50c..4a022e9 100644 --- a/src/main/java/cn/ac/iie/utils/zookeeper/ZookeeperUtils.java +++ b/src/main/java/cn/ac/iie/utils/zookeeper/ZookeeperUtils.java @@ -1,6 +1,5 @@ package cn.ac.iie.utils.zookeeper; -import cn.ac.iie.common.FlowWriteConfig; import org.apache.commons.lang3.RandomUtils; import org.apache.log4j.Logger; import org.apache.zookeeper.*; @@ -11,9 +10,7 @@ import java.io.IOException; import java.util.List; import java.util.concurrent.CountDownLatch; -/** - * @author qidaijie - */ + public class ZookeeperUtils implements Watcher { private static Logger logger = Logger.getLogger(ZookeeperUtils.class); @@ -36,12 +33,11 @@ public class ZookeeperUtils implements Watcher { * * @param path 节点路径 */ - public int modifyNode(String path) { - createNode("/Snowflake", null, ZooDefs.Ids.OPEN_ACL_UNSAFE); - createNode("/Snowflake/" + FlowWriteConfig.KAFKA_TOPIC, "0".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE); - int workerId; + public int modifyNode(String path, String zookeeperIp) { + createNode(path, "0".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, zookeeperIp); + int workerId = 0; try { - connectZookeeper(); + connectZookeeper(zookeeperIp); Stat stat = zookeeper.exists(path, true); workerId = Integer.parseInt(getNodeDate(path)); if (workerId > 55) { @@ -58,20 +54,22 @@ public class ZookeeperUtils implements Watcher { } catch (KeeperException | InterruptedException e) { e.printStackTrace(); workerId = RandomUtils.nextInt(56, 63); - } finally { + } + finally { closeConn(); } - logger.error("工作ID是：" + workerId); + logger.warn("workerID is：" + workerId); return workerId; } /** * 连接zookeeper * + * @param host 地址 */ - private void connectZookeeper() { + public void connectZookeeper(String host) { try { - zookeeper = new ZooKeeper(FlowWriteConfig.ZOOKEEPER_SERVERS, SESSION_TIME_OUT, this); + zookeeper = new ZooKeeper(host, SESSION_TIME_OUT, this); countDownLatch.await(); } catch (IOException | InterruptedException e) { e.printStackTrace(); @@ -81,7 +79,7 @@ public class ZookeeperUtils implements Watcher { /** * 关闭连接 */ - private void closeConn() { + public void closeConn() { try { if (zookeeper != null) { zookeeper.close(); @@ -97,7 +95,7 @@ public class ZookeeperUtils implements Watcher { * @param path 节点路径 * @return 内容/异常null */ - private String getNodeDate(String path) { + public String getNodeDate(String path) { String result = null; Stat stat = new Stat(); try { @@ -115,14 +113,18 @@ public class ZookeeperUtils implements Watcher { * @param date 节点所存储的数据的byte[] * @param acls 控制权限策略 */ - private void createNode(String path, byte[] date, List acls) { + public void createNode(String path, byte[] date, List acls, String zookeeperIp) { try { - connectZookeeper(); + connectZookeeper(zookeeperIp); Stat exists = zookeeper.exists(path, true); if (exists == null) { + Stat existsSnowflakeld = zookeeper.exists("/Snowflake", true); + if (existsSnowflakeld == null) { + zookeeper.create("/Snowflake", null, acls, CreateMode.PERSISTENT); + } zookeeper.create(path, date, acls, CreateMode.PERSISTENT); } else { - logger.warn("Node already exists!,Don't need to create"); + logger.warn("Node already exists ! Don't need to create"); } } catch (KeeperException | InterruptedException e) { e.printStackTrace(); From 475f28cabc12c91289a924267c25f6d8bd6a1346 Mon Sep 17 00:00:00 2001 From: lee Date: Wed, 12 Feb 2020 16:32:26 +0800 Subject: [PATCH 25/27] =?UTF-8?q?=E5=B7=A5=E5=85=B7=E7=B1=BB=E6=89=93?= =?UTF-8?q?=E5=8C=85=E6=9B=BF=E6=8D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- properties/service_flow_config.properties | 4 +- .../bolt/collect/CollectCompletedBolt.java | 2 +- .../iie/bolt/proxy/ProxyCompletionBolt.java | 5 +- .../bolt/security/SecurityCompletionBolt.java | 2 +- .../cn/ac/iie/utils/general/CompleteUtil.java | 143 ------------- .../ac/iie/utils/general/TransFormUtils.java | 40 ++-- .../cn/ac/iie/utils/hbase/HBaseUtils.java | 184 ----------------- .../cn/ac/iie/utils/system/SnowflakeId.java | 190 ------------------ .../ac/iie/utils/zookeeper/ZooKeeperLock.java | 140 ------------- .../iie/utils/zookeeper/ZookeeperUtils.java | 136 ------------- .../java/cn/ac/iie/test/DistributedLock.java | 4 +- 12 files changed, 31 insertions(+), 821 deletions(-) delete mode 100644 src/main/java/cn/ac/iie/utils/general/CompleteUtil.java delete mode 100644 src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java delete mode 100644 src/main/java/cn/ac/iie/utils/system/SnowflakeId.java delete mode 100644 src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java delete mode 100644 src/main/java/cn/ac/iie/utils/zookeeper/ZookeeperUtils.java diff --git a/pom.xml b/pom.xml index e8a81ee..273a18c 100644 --- a/pom.xml +++ b/pom.xml @@ -153,7 +153,7 @@ com.zdjizhi galaxy - 1.0.1 + 1.0.2 slf4j-log4j12 diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index 90b80e1..9336ca9 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -29,13 +29,13 @@ results.output.topic=SECURITY-EVENT-COMPLETED-LOG topology.workers=1 #spout并行度 建议与kafka分区数相同 -spout.parallelism=3 +spout.parallelism=1 #处理补全操作的bolt并行度-worker的倍数 datacenter.bolt.parallelism=1 #写入kafka的并行度10 -kafka.bolt.parallelism=3 +kafka.bolt.parallelism=1 #定位库地址 #ip.library=/home/ceiec/topology/dat/ diff --git a/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java b/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java index 5f6ab55..a7c4162 100644 --- a/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java +++ b/src/main/java/cn/ac/iie/bolt/collect/CollectCompletedBolt.java @@ -1,8 +1,8 @@ package cn.ac.iie.bolt.collect; import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.system.TupleUtils; +import com.zdjizhi.utils.HBaseUtils; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; import org.apache.storm.task.TopologyContext; diff --git a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java index 56f8f8b..a2783ee 100644 --- a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java @@ -1,8 +1,8 @@ package cn.ac.iie.bolt.proxy; import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.system.TupleUtils; +import com.zdjizhi.utils.HBaseUtils; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; import org.apache.storm.task.TopologyContext; @@ -17,7 +17,6 @@ import java.util.HashMap; import java.util.Map; import static cn.ac.iie.utils.general.TransFormUtils.dealCommonMessage; -import static cn.ac.iie.utils.hbase.HBaseUtils.change; /** * 通联关系日志补全 @@ -38,7 +37,7 @@ public class ProxyCompletionBolt extends BaseBasicBolt { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { try { if (TupleUtils.isTick(tuple)) { - change(); + HBaseUtils.change(); } else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { diff --git a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java index 7a4182e..09cd305 100644 --- a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java @@ -1,8 +1,8 @@ package cn.ac.iie.bolt.security; import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.system.TupleUtils; +import com.zdjizhi.utils.HBaseUtils; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; import org.apache.storm.task.TopologyContext; diff --git a/src/main/java/cn/ac/iie/utils/general/CompleteUtil.java b/src/main/java/cn/ac/iie/utils/general/CompleteUtil.java deleted file mode 100644 index b786a4a..0000000 --- a/src/main/java/cn/ac/iie/utils/general/CompleteUtil.java +++ /dev/null @@ -1,143 +0,0 @@ -package cn.ac.iie.utils.general; - -import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HBaseUtils; -import cn.ac.iie.utils.system.SnowflakeId; -import com.google.common.net.InternetDomainName; -import com.zdjizhi.utils.IpLookup; -import com.zdjizhi.utils.StringUtil; -import org.apache.log4j.Logger; - -import java.util.Base64; - - -public final class CompleteUtil { - - private static Logger logger = Logger.getLogger(TransFormUtils.class); - private static IpLookup ipLookup = new IpLookup.Builder(false) - .loadDataFileV4(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") - .loadDataFileV6(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") - .loadAsnDataFileV4(FlowWriteConfig.IP_LIBRARY + "asn_v4.mmdb") - .loadAsnDataFileV6(FlowWriteConfig.IP_LIBRARY + "asn_v6.mmdb") - .build(); - - /** - * 有sni通过sni获取域名，有host根据host获取域名 - * - * @param sni sni - * @param host host - * @return 顶级域名 - */ - public static String getTopDomain(String sni, String host) { - if (StringUtil.isNotBlank(host)) { - return getDomainName(host); - } else if (StringUtil.isNotBlank(sni)) { - return getDomainName(sni); - } else { - return ""; - } - } - - - /** - * 根据url截取顶级域名 - * - * @param host 网站url - * @return 顶级域名 - */ - public static String getDomainName(String host) { - String domain = ""; - try { - domain = InternetDomainName.from(host).topPrivateDomain().toString(); - } catch (Exception e) { - logger.error("host解析顶级域名异常: " + e.getMessage()); - } - return domain; - } - - /** - * 生成当前时间戳的操作 - */ - public static int getCurrentTime() { - return (int)(System.currentTimeMillis() / 1000); - } - - /** - * 雪花模型生成id - * - * @return - */ - public static long getSnowflakeId(String zookeeperIp, String kafkaTopic, long dataCenterIdNum) { - - return SnowflakeId.generateId(zookeeperIp,kafkaTopic,dataCenterIdNum); - } - - /** - * 根据clientIp获取location信息 - * - * @param ip - * @return - */ - public static String getGeoIpDetail(String ip) { - - return ipLookup.cityLookupDetail(ip); - } - - /** - * 根据ip获取asn信息 - * - * @param ip - * @return asn - */ - public static String getGeoAsn(String ip) { - - return ipLookup.asnLookup(ip, true); - } - - /** - * 根据ip获取country信息 - * - * @param ip - * @return country - */ - public static String getGeoIpCountry(String ip) { - - return ipLookup.countryLookup(ip); - } - - /** - * 根据ip去hbase中匹配对应的用户名 - * @param clientIp - * @param hbaseZookeeper - * @param hbaseTable - * @return 用户名 subscriber_id - */ - public static String radiusMatch(String clientIp, String hbaseZookeeper, String hbaseTable) { - return HBaseUtils.getAccount(clientIp,hbaseZookeeper,hbaseTable); - } - - - /** - * base64 解码 - * - * @param encodedText mail subject - * @param subjectCharset 编码格式 - * @return 解码内容 / 空 - */ - public static String base64Str(String encodedText, String subjectCharset) { - Base64.Decoder decoder = Base64.getDecoder(); - String sub; - try { - if (StringUtil.isNotBlank(subjectCharset)) { - sub = new String(decoder.decode(encodedText), subjectCharset); - } else { - sub = new String(decoder.decode(encodedText), "UTF-8"); - } - return sub; - } catch (Exception e) { - logger.error("transform base64 String failed! base64Str = " + encodedText + "Charset = " + subjectCharset + "error :" + e); - return ""; - } - } - -} diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index fb028eb..572cf78 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -2,20 +2,16 @@ package cn.ac.iie.utils.general; import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.json.JsonParseUtil; -import cn.ac.iie.utils.system.SnowflakeId; import com.alibaba.fastjson.JSONObject; -import com.google.common.net.InternetDomainName; +import com.zdjizhi.utils.FormatUtils; import com.zdjizhi.utils.IpLookup; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; import org.junit.jupiter.api.Test; import java.util.*; -import java.util.regex.Pattern; -import static cn.ac.iie.utils.general.CompleteUtil.*; /** @@ -28,6 +24,16 @@ import static cn.ac.iie.utils.general.CompleteUtil.*; public class TransFormUtils { private static Logger logger = Logger.getLogger(TransFormUtils.class); +// private static IpLookup ipLookup = FormatUtils.getIpLookup(FlowWriteConfig.IP_LIBRARY); + + private static IpLookup ipLookup = new IpLookup.Builder(false) + .loadDataFileV4(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") + .loadDataFileV6(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") + .loadAsnDataFileV4(FlowWriteConfig.IP_LIBRARY + "asn_v4.mmdb") + .loadAsnDataFileV6(FlowWriteConfig.IP_LIBRARY + "asn_v6.mmdb") + .build(); + + //在内存中加载反射类用的map private static HashMap map = JsonParseUtil.getMapFromhttp(FlowWriteConfig.SCHEMA_HTTP); //反射成一个类 @@ -43,8 +49,6 @@ public class TransFormUtils { */ public static String dealCommonMessage(String message) { - - Object object = JSONObject.parseObject(message, mapObject.getClass()); // System.out.println("补全之前 ===》 "+JSON.toJSONString(object)); @@ -64,26 +68,26 @@ public class TransFormUtils { if (function.equals("current_timestamp")) { - JsonParseUtil.setValue(object, strings[1], getCurrentTime()); + JsonParseUtil.setValue(object, strings[1], FormatUtils.getCurrentTime()); } else if (function.equals("snowflake_id")) { - JsonParseUtil.setValue(object, strings[1], getSnowflakeId(FlowWriteConfig.ZOOKEEPER_SERVERS,FlowWriteConfig.KAFKA_TOPIC,FlowWriteConfig.DATA_CENTER_ID_NUM)); + JsonParseUtil.setValue(object, strings[1], FormatUtils.getSnowflakeId(FlowWriteConfig.ZOOKEEPER_SERVERS,FlowWriteConfig.KAFKA_TOPIC,FlowWriteConfig.DATA_CENTER_ID_NUM)); } else if (function.equals("geo_ip_detail")) { - JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(name.toString())); + JsonParseUtil.setValue(object, strings[1], FormatUtils.getGeoIpDetail(name.toString(),ipLookup)); } else if (function.equals("geo_asn")) { - JsonParseUtil.setValue(object, strings[1], getGeoAsn(name.toString())); + JsonParseUtil.setValue(object, strings[1], FormatUtils.getGeoAsn(name.toString(),ipLookup)); } else if (function.equals("radius_match")) { - JsonParseUtil.setValue(object, strings[1], radiusMatch(name.toString(),FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS,FlowWriteConfig.HBASE_TABLE_NAME)); + JsonParseUtil.setValue(object, strings[1], FormatUtils.radiusMatch(name.toString(),FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS,FlowWriteConfig.HBASE_TABLE_NAME)); } else if (function.equals("geo_ip_country")) { - JsonParseUtil.setValue(object, strings[1], getGeoIpCountry(name.toString())); + JsonParseUtil.setValue(object, strings[1], FormatUtils.getGeoIpCountry(name.toString(),ipLookup)); } else if (function.equals("decode_of_base64") && param != null){ - JsonParseUtil.setValue(object, strings[1], base64Str(name.toString(),param.toString())); + JsonParseUtil.setValue(object, strings[1], FormatUtils.base64Str(name.toString(),param.toString())); } else if (name.equals("http_host") && function.equals("sub_domain")) { if (appendTo == null || StringUtil.isBlank(appendTo.toString())) { - JsonParseUtil.setValue(object, strings[1], getTopDomain(null, name.toString())); + JsonParseUtil.setValue(object, strings[1], FormatUtils.getTopDomain(null, name.toString())); } } else if (name.equals("ssl_sni") && strings[2].equals("sub_domain")) { if (appendTo == null || StringUtil.isBlank(appendTo.toString())) { - JsonParseUtil.setValue(object, strings[1], getTopDomain(name.toString(), null)); + JsonParseUtil.setValue(object, strings[1], FormatUtils.getTopDomain(name.toString(), null)); } } @@ -105,8 +109,8 @@ public class TransFormUtils { @Test public void aaa() { String sni = "www.baidu.com"; - System.out.println(getTopDomain(sni, null)); - System.out.println(getTopDomain(null,sni)); + System.out.println(FormatUtils.getTopDomain(sni, null)); + System.out.println(FormatUtils.getTopDomain(null,sni)); } } \ No newline at end of file diff --git a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java deleted file mode 100644 index 5c0e4c9..0000000 --- a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java +++ /dev/null @@ -1,184 +0,0 @@ -package cn.ac.iie.utils.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.log4j.Logger; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; - -public class HBaseUtils { - private final static Logger logger = Logger.getLogger(HBaseUtils.class); - private static Map subIdMap = new HashMap<>(333334); - private static Connection connection; - private static Long time; - - private static String zookeeperIp; - private static String hbaseTable; - - private static HBaseUtils hBaseUtils; - - private static void getHbaseInstance(String zookeeperServer, String hbaseTableName) { - hBaseUtils = new HBaseUtils(zookeeperServer, hbaseTableName); - } - - /** - * 构造函数-新-20191023 - */ - public HBaseUtils(String zookeeperServer, String hbaseTableName) { - zookeeperIp = zookeeperServer; - hbaseTable = hbaseTableName; - //获取连接 - getHbaseConn(); - //拉取所有 - getAll(); - //定时更新 - updateHabaseCache(); - } - - private static void getHbaseConn() { - try { - // 管理Hbase的配置信息 - Configuration configuration = HBaseConfiguration.create(); - // 设置zookeeper节点 - configuration.set("hbase.zookeeper.quorum", zookeeperIp); - configuration.set("hbase.client.retries.number", "3"); - configuration.set("hbase.bulkload.retries.number", "3"); - configuration.set("zookeeper.recovery.retry", "3"); - connection = ConnectionFactory.createConnection(configuration); - time = System.currentTimeMillis(); - logger.warn("HBaseUtils get HBase connection,now to getAll()."); - } catch (IOException ioe) { - logger.error("HBaseUtils getHbaseConn() IOException===>{" + ioe + "}<==="); - ioe.printStackTrace(); - } catch (Exception e) { - logger.error("HBaseUtils getHbaseConn() Exception===>{" + e + "}<==="); - e.printStackTrace(); - } - } - - /** - * 更新变量 - */ - public static void change() { - long nowTime = System.currentTimeMillis(); - timestampsFilter(time - 1000, nowTime + 500); - } - - /** - * 获取变更内容 - * - * @param startTime 开始时间 - * @param endTime 结束时间 - */ - private static void timestampsFilter(Long startTime, Long endTime) { - Long begin = System.currentTimeMillis(); - Table table = null; - ResultScanner scanner = null; - Scan scan2 = new Scan(); - try { - table = connection.getTable(TableName.valueOf("sub:" + hbaseTable)); - scan2.setTimeRange(startTime, endTime); - scanner = table.getScanner(scan2); - for (Result result : scanner) { - Cell[] cells = result.rawCells(); - for (Cell cell : cells) { - String key = Bytes.toString(CellUtil.cloneRow(cell)); - String value = Bytes.toString(CellUtil.cloneValue(cell)); - if (subIdMap.containsKey(key)) { - if (!value.equals(subIdMap.get(key))) { - subIdMap.put(key, value); - } - } else { - subIdMap.put(key, value); - } - } - } - Long end = System.currentTimeMillis(); - logger.warn("HBaseUtils Now subIdMap.keySet().size() is: " + subIdMap.keySet().size()); - logger.warn("HBaseUtils Update cache timeConsuming is: " + (end - begin) + ",BeginTime: " + begin + ",EndTime: " + end); - time = endTime; - } catch (IOException ioe) { - logger.error("HBaseUtils timestampsFilter is IOException===>{" + ioe + "}<==="); - ioe.printStackTrace(); - } catch (Exception e) { - logger.error("HBaseUtils timestampsFilter is Exception===>{" + e + "}<==="); - e.printStackTrace(); - } finally { - if (scanner != null) { - scanner.close(); - } - if (table != null) { - try { - table.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - } - - /** - * 获取所有的 key value - */ - private static void getAll() { - long begin = System.currentTimeMillis(); - try { - Table table = connection.getTable(TableName.valueOf("sub:" + hbaseTable)); - Scan scan2 = new Scan(); - ResultScanner scanner = table.getScanner(scan2); - for (Result result : scanner) { - Cell[] cells = result.rawCells(); - for (Cell cell : cells) { - subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); - } - } - logger.warn("HBaseUtils Get fullAmount List size->subIdMap.size(): " + subIdMap.size()); - logger.warn("HBaseUtils Get fullAmount List size->subIdMap.size() timeConsuming is: " + (System.currentTimeMillis() - begin)); - scanner.close(); - } catch (IOException ioe) { - logger.error("HBaseUtils getAll() is IOException===>{" + ioe + "}<==="); - ioe.printStackTrace(); - } catch (Exception e) { - logger.error("HBaseUtils getAll() is Exception===>{" + e + "}<==="); - e.printStackTrace(); - } - } - - /** - * 获取 account - * - * @param clientIp client_ip - * @return account - */ - public static String getAccount(String clientIp, String hbaseZookeeper, String hbaseTable) { - if (hBaseUtils == null) { - getHbaseInstance(hbaseZookeeper, hbaseTable); - } - return subIdMap.get(clientIp); - } - - private void updateHabaseCache() { - Timer timer = new Timer(); - timer.scheduleAtFixedRate(new TimerTask() { - @Override - public void run() { - try { - change(); - } catch (Exception e) { - logger.error("HBaseUtils update hbaseCache is error===>{" + e + "}<==="); - e.printStackTrace(); - } - } - }, 1, 1000 * 60); - } - -} diff --git a/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java b/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java deleted file mode 100644 index 8d67348..0000000 --- a/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java +++ /dev/null @@ -1,190 +0,0 @@ -package cn.ac.iie.utils.system; - -import cn.ac.iie.utils.zookeeper.ZooKeeperLock; -import cn.ac.iie.utils.zookeeper.ZookeeperUtils; -import org.apache.log4j.Logger; - - -public class SnowflakeId { - private static Logger logger = Logger.getLogger(SnowflakeId.class); - - /** - * 开始时间截 (2018-08-01 00:00:00) max 17years - */ - private final long twepoch = 1564588800000L; - - /** - * 机器id所占的位数 - */ - private final long workerIdBits = 6L; - - /** - * 数据标识id所占的位数 - */ - private final long dataCenterIdBits = 4L; - - /** - * 支持的最大机器id，结果是63 (这个移位算法可以很快的计算出几位二进制数所能表示的最大十进制数) - * M << n = M * 2^n - */ - private final long maxWorkerId = -1L ^ (-1L << workerIdBits); - - /** - * 支持的最大数据标识id，结果是15 - */ - private final long maxDataCenterId = -1L ^ (-1L << dataCenterIdBits); - - /** - * 序列在id中占的位数 - */ - private final long sequenceBits = 14L; - - /** - * 机器ID向左移12位 - */ - private final long workerIdShift = sequenceBits; - - /** - * 数据标识id向左移17位(14+6) - */ - private final long dataCenterIdShift = sequenceBits + workerIdBits; - - /** - * 时间截向左移22位(4+6+14) - */ - private final long timestampLeftShift = sequenceBits + workerIdBits + dataCenterIdBits; - - /** - * 生成序列的掩码，这里为16383 - */ - private final long sequenceMask = -1L ^ (-1L << sequenceBits); - - /** - * 工作机器ID(0~63) - */ - private long workerId; - - /** - * 数据中心ID(0~15) - */ - private long dataCenterId; - - /** - * 毫秒内序列(0~16383) - */ - private long sequence = 0L; - - /** - * 上次生成ID的时间截 - */ - private long lastTimestamp = -1L; - - - private static SnowflakeId idWorker; - - private static ZookeeperUtils zookeeperUtils = new ZookeeperUtils(); - - private static void getSnowflakeldInstance(String zookeeperIp, String kafkaTopic, long dataCenterIdNum) { - idWorker = new SnowflakeId(zookeeperIp, kafkaTopic, dataCenterIdNum); - } - - /** - * 构造函数 - */ - private SnowflakeId(String zookeeperIp, String kafkaTopic, long dataCenterIdNum) { - ZooKeeperLock lock = new ZooKeeperLock(zookeeperIp, "/locks", "disLocks1"); - if (lock.lock()) { - int tmpWorkerId = zookeeperUtils.modifyNode("/Snowflake/" + kafkaTopic, zookeeperIp); - if (tmpWorkerId > maxWorkerId || tmpWorkerId < 0) { - throw new IllegalArgumentException(String.format("worker Id can't be greater than %d or less than 0", maxWorkerId)); - } - if (dataCenterIdNum > maxDataCenterId || dataCenterIdNum < 0) { - throw new IllegalArgumentException(String.format("datacenter Id can't be greater than %d or less than ", maxDataCenterId)); - } - this.workerId = tmpWorkerId; - this.dataCenterId = dataCenterIdNum; - try { - lock.unlock(); - } catch (InterruptedException ie) { - ie.printStackTrace(); - } catch (Exception e) { - e.printStackTrace(); - logger.error("This is not usual error!!!===>>>" + e + "<<<==="); - } - } - } - - /** - * 获得下一个ID (该方法是线程安全的) - * - * @return SnowflakeId - */ - private synchronized long nextId() { - long timestamp = timeGen(); - - //如果当前时间小于上一次ID生成的时间戳，说明系统时钟回退过这个时候应当抛出异常 - if (timestamp < lastTimestamp) { - throw new RuntimeException( - String.format("Clock moved backwards. Refusing to generate id for %d milliseconds", lastTimestamp - timestamp)); - } - - //如果是同一时间生成的，则进行毫秒内序列 - if (lastTimestamp == timestamp) { - sequence = (sequence + 1) & sequenceMask; - //毫秒内序列溢出 - if (sequence == 0) { - //阻塞到下一个毫秒,获得新的时间戳 - timestamp = tilNextMillis(lastTimestamp); - } - } - //时间戳改变，毫秒内序列重置 - else { - sequence = 0L; - } - - //上次生成ID的时间截 - lastTimestamp = timestamp; - - //移位并通过或运算拼到一起组成64位的ID - return ((timestamp - twepoch) << timestampLeftShift) - | (dataCenterId << dataCenterIdShift) - | (workerId << workerIdShift) - | sequence; - } - - /** - * 阻塞到下一个毫秒，直到获得新的时间戳 - * - * @param lastTimestamp 上次生成ID的时间截 - * @return 当前时间戳 - */ - protected long tilNextMillis(long lastTimestamp) { - long timestamp = timeGen(); - while (timestamp <= lastTimestamp) { - timestamp = timeGen(); - } - return timestamp; - } - - /** - * 返回以毫秒为单位的当前时间 - * - * @return 当前时间(毫秒) - */ - protected long timeGen() { - return System.currentTimeMillis(); - } - - - /** - * 静态工具类 - * - * @return - */ - public static Long generateId(String zookeeperIp, String kafkaTopic, long dataCenterIdNum) { - if (idWorker == null) { - getSnowflakeldInstance(zookeeperIp, kafkaTopic, dataCenterIdNum); - } - return idWorker.nextId(); - } -} \ No newline at end of file diff --git a/src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java b/src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java deleted file mode 100644 index 8d0a4d5..0000000 --- a/src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java +++ /dev/null @@ -1,140 +0,0 @@ -package cn.ac.iie.utils.zookeeper; - -import org.apache.log4j.Logger; -import org.apache.zookeeper.*; -import org.apache.zookeeper.data.Stat; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -public class ZooKeeperLock implements Watcher { - private static Logger logger = Logger.getLogger(ZooKeeperLock.class); - - private ZooKeeper zk = null; - private String rootLockNode; // 锁的根节点 - private String lockName; // 竞争资源，用来生成子节点名称 - private String currentLock; // 当前锁 - private String waitLock; // 等待的锁（前一个锁） - private CountDownLatch countDownLatch; // 计数器（用来在加锁失败时阻塞加锁线程） - private int sessionTimeout = 30000; // 超时时间 - - // 1. 构造器中创建ZK链接，创建锁的根节点 - public ZooKeeperLock(String zkAddress, String rootLockNode, String lockName) { - this.rootLockNode = rootLockNode; - this.lockName = lockName; - try { - // 创建连接，zkAddress格式为：IP:PORT - zk = new ZooKeeper(zkAddress, this.sessionTimeout, this); - // 检测锁的根节点是否存在，不存在则创建 - Stat stat = zk.exists(rootLockNode, false); - if (null == stat) { - zk.create(rootLockNode, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } - } catch (IOException | InterruptedException | KeeperException e) { - e.printStackTrace(); - logger.error("ZooKeeperLock Constructors ===>>>Node already exists!"); - } - } - - // 2. 加锁方法，先尝试加锁，不能加锁则等待上一个锁的释放 - public boolean lock() { - if (this.tryLock()) { - logger.warn("ZooKeeperLock method lock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] addZkLock(" + this.currentLock + ")success！"); - return true; - } else { - return waitOtherLock(this.waitLock, this.sessionTimeout); - } - } - - public boolean tryLock() { - // 分隔符 - String split = "_lock_"; - if (this.lockName.contains("_lock_")) { - throw new RuntimeException("lockName can't contains '_lock_' "); - } - try { - // 创建锁节点（临时有序节点） - this.currentLock = zk.create(this.rootLockNode + "/" + this.lockName + split, new byte[0], - ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL); - logger.warn("ZooKeeperLock method tryLock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] create zkLockNode(" + this.currentLock + ")success，begin to election..."); - // 取所有子节点 - List nodes = zk.getChildren(this.rootLockNode, false); - // 取所有竞争lockName的锁 - List lockNodes = new ArrayList(); - for (String nodeName : nodes) { - if (nodeName.split(split)[0].equals(this.lockName)) { - lockNodes.add(nodeName); - } - } - Collections.sort(lockNodes); - // 取最小节点与当前锁节点比对加锁 - String currentLockPath = this.rootLockNode + "/" + lockNodes.get(0); - if (this.currentLock.equals(currentLockPath)) { - return true; - } - // 加锁失败，设置前一节点为等待锁节点 - String currentLockNode = this.currentLock.substring(this.currentLock.lastIndexOf("/") + 1); - int preNodeIndex = Collections.binarySearch(lockNodes, currentLockNode) - 1; - this.waitLock = lockNodes.get(preNodeIndex); - } catch (KeeperException | InterruptedException e) { - e.printStackTrace(); - } - return false; - } - - private boolean waitOtherLock(String waitLock, int sessionTimeout) { - boolean islock = false; - try { - // 监听等待锁节点 - String waitLockNode = this.rootLockNode + "/" + waitLock; - Stat stat = zk.exists(waitLockNode, true); - if (null != stat) { - logger.error("ZooKeeperLock method waitOtherLock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] zkLock(" + this.currentLock + ")addZkLock fail,wait lock(" + waitLockNode + ")release..."); - // 设置计数器，使用计数器阻塞线程 - this.countDownLatch = new CountDownLatch(1); - islock = this.countDownLatch.await(sessionTimeout, TimeUnit.MILLISECONDS); - this.countDownLatch = null; - if (islock) { - logger.warn("ZooKeeperLock method waitOtherLock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] zkLock(" + this.currentLock + ")addZkLock success,lock(" + waitLockNode + ")release over."); - } else { - logger.error("ZooKeeperLock method waitOtherLock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] zkLock(" + this.currentLock + "addZkLock fail..."); - } - } else { - islock = true; - } - } catch (KeeperException | InterruptedException e) { - e.printStackTrace(); - } - return islock; - } - - // 3. 释放锁 - public void unlock() throws InterruptedException { - try { - Stat stat = zk.exists(this.currentLock, false); - if (null != stat) { - logger.warn("ZooKeeperLock method unlock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] free zkLock " + this.currentLock); - zk.delete(this.currentLock, -1); - this.currentLock = null; - } - } catch (InterruptedException | KeeperException e) { - e.printStackTrace(); - } finally { - zk.close(); - } - } - - // 4. 监听器回调 - @Override - public void process(WatchedEvent watchedEvent) { - if (null != this.countDownLatch && watchedEvent.getType() == Event.EventType.NodeDeleted) { - // 计数器减一，恢复线程操作 - this.countDownLatch.countDown(); - } - } - -} diff --git a/src/main/java/cn/ac/iie/utils/zookeeper/ZookeeperUtils.java b/src/main/java/cn/ac/iie/utils/zookeeper/ZookeeperUtils.java deleted file mode 100644 index 4a022e9..0000000 --- a/src/main/java/cn/ac/iie/utils/zookeeper/ZookeeperUtils.java +++ /dev/null @@ -1,136 +0,0 @@ -package cn.ac.iie.utils.zookeeper; - -import org.apache.commons.lang3.RandomUtils; -import org.apache.log4j.Logger; -import org.apache.zookeeper.*; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.CountDownLatch; - - -public class ZookeeperUtils implements Watcher { - private static Logger logger = Logger.getLogger(ZookeeperUtils.class); - - private ZooKeeper zookeeper; - - private static final int SESSION_TIME_OUT = 20000; - - private CountDownLatch countDownLatch = new CountDownLatch(1); - - @Override - public void process(WatchedEvent event) { - if (event.getState() == Watcher.Event.KeeperState.SyncConnected) { - countDownLatch.countDown(); - } - } - - - /** - * 修改节点信息 - * - * @param path 节点路径 - */ - public int modifyNode(String path, String zookeeperIp) { - createNode(path, "0".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, zookeeperIp); - int workerId = 0; - try { - connectZookeeper(zookeeperIp); - Stat stat = zookeeper.exists(path, true); - workerId = Integer.parseInt(getNodeDate(path)); - if (workerId > 55) { - workerId = 0; - zookeeper.setData(path, "1".getBytes(), stat.getVersion()); - } else { - String result = String.valueOf(workerId + 1); - if (stat != null) { - zookeeper.setData(path, result.getBytes(), stat.getVersion()); - } else { - logger.error("Node does not exist!,Can't modify"); - } - } - } catch (KeeperException | InterruptedException e) { - e.printStackTrace(); - workerId = RandomUtils.nextInt(56, 63); - } - finally { - closeConn(); - } - logger.warn("workerID is：" + workerId); - return workerId; - } - - /** - * 连接zookeeper - * - * @param host 地址 - */ - public void connectZookeeper(String host) { - try { - zookeeper = new ZooKeeper(host, SESSION_TIME_OUT, this); - countDownLatch.await(); - } catch (IOException | InterruptedException e) { - e.printStackTrace(); - } - } - - /** - * 关闭连接 - */ - public void closeConn() { - try { - if (zookeeper != null) { - zookeeper.close(); - } - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - /** - * 获取节点内容 - * - * @param path 节点路径 - * @return 内容/异常null - */ - public String getNodeDate(String path) { - String result = null; - Stat stat = new Stat(); - try { - byte[] resByte = zookeeper.getData(path, true, stat); - result = new String(resByte); - } catch (KeeperException | InterruptedException e) { - logger.error("Get node information exception"); - e.printStackTrace(); - } - return result; - } - - /** - * @param path 节点创建的路径 - * @param date 节点所存储的数据的byte[] - * @param acls 控制权限策略 - */ - public void createNode(String path, byte[] date, List acls, String zookeeperIp) { - try { - connectZookeeper(zookeeperIp); - Stat exists = zookeeper.exists(path, true); - if (exists == null) { - Stat existsSnowflakeld = zookeeper.exists("/Snowflake", true); - if (existsSnowflakeld == null) { - zookeeper.create("/Snowflake", null, acls, CreateMode.PERSISTENT); - } - zookeeper.create(path, date, acls, CreateMode.PERSISTENT); - } else { - logger.warn("Node already exists ! Don't need to create"); - } - } catch (KeeperException | InterruptedException e) { - e.printStackTrace(); - } finally { - closeConn(); - } - } - -} diff --git a/src/test/java/cn/ac/iie/test/DistributedLock.java b/src/test/java/cn/ac/iie/test/DistributedLock.java index 030bf4a..0d45cb8 100644 --- a/src/test/java/cn/ac/iie/test/DistributedLock.java +++ b/src/test/java/cn/ac/iie/test/DistributedLock.java @@ -10,7 +10,7 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.zookeeper.ZookeeperUtils; +import com.zdjizhi.utils.ZookeeperUtils; import org.apache.log4j.Logger; import org.apache.zookeeper.*; import org.apache.zookeeper.data.Stat; @@ -183,7 +183,7 @@ public class DistributedLock implements Lock, Watcher { try { lock = new DistributedLock(FlowWriteConfig.ZOOKEEPER_SERVERS, "disLocks1"); lock.lock(); - zookeeperUtils.modifyNode("/Snowflake/" + FlowWriteConfig.KAFKA_TOPIC); + zookeeperUtils.modifyNode("/Snowflake/" + FlowWriteConfig.KAFKA_TOPIC,FlowWriteConfig.ZOOKEEPER_SERVERS); } finally { if (lock != null) { lock.unlock(); From 28e0b9e38c2bbfc164e63318c404090c544e2d1e Mon Sep 17 00:00:00 2001 From: lee Date: Tue, 25 Feb 2020 14:16:16 +0800 Subject: [PATCH 26/27] =?UTF-8?q?=E5=8A=A8=E6=80=81=E8=8E=B7=E5=8F=96schem?= =?UTF-8?q?a=E4=BB=A3=E7=A0=81=E6=9B=B4=E6=96=B00225?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/vcs.xml | 2 +- pom.xml | 4 +- properties/service_flow_config.properties | 6 +- .../iie/bolt/proxy/ProxyCompletionBolt.java | 3 +- .../bolt/security/SecurityCompletionBolt.java | 3 +- .../cn/ac/iie/spout/CustomizedKafkaSpout.java | 1 + .../cn/ac/iie/utils/general/CompleteUtil.java | 143 ------------- .../ac/iie/utils/general/TransFormUtils.java | 119 +++++++++-- .../cn/ac/iie/utils/hbase/HBaseUtils.java | 106 +++------- .../cn/ac/iie/utils/system/SnowflakeId.java | 190 ------------------ .../ac/iie/utils/zookeeper/ZooKeeperLock.java | 140 ------------- .../iie/utils/zookeeper/ZookeeperUtils.java | 136 ------------- .../java/cn/ac/iie/test/DistributedLock.java | 4 +- src/test/java/cn/ac/iie/test/test.java | 45 ++++- 14 files changed, 182 insertions(+), 720 deletions(-) delete mode 100644 src/main/java/cn/ac/iie/utils/general/CompleteUtil.java delete mode 100644 src/main/java/cn/ac/iie/utils/system/SnowflakeId.java delete mode 100644 src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java delete mode 100644 src/main/java/cn/ac/iie/utils/zookeeper/ZookeeperUtils.java diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 35eb1dd..94a25f7 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -1,6 +1,6 @@ - + \ No newline at end of file diff --git a/pom.xml b/pom.xml index e8a81ee..4cda26b 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ org.apache.storm storm-core ${storm.version} - + provided slf4j-log4j12 @@ -153,7 +153,7 @@ com.zdjizhi galaxy - 1.0.1 + 1.0.2 slf4j-log4j12 diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index 90b80e1..04a60a3 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -20,7 +20,7 @@ auto.offset.reset=latest kafka.topic=SECURITY-EVENT-LOG #读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； -group.id=security-policy-200204 +group.id=security-policy-200224 #输出topic results.output.topic=SECURITY-EVENT-COMPLETED-LOG @@ -29,13 +29,13 @@ results.output.topic=SECURITY-EVENT-COMPLETED-LOG topology.workers=1 #spout并行度 建议与kafka分区数相同 -spout.parallelism=3 +spout.parallelism=1 #处理补全操作的bolt并行度-worker的倍数 datacenter.bolt.parallelism=1 #写入kafka的并行度10 -kafka.bolt.parallelism=3 +kafka.bolt.parallelism=1 #定位库地址 #ip.library=/home/ceiec/topology/dat/ diff --git a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java index 56f8f8b..fcac2db 100644 --- a/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/proxy/ProxyCompletionBolt.java @@ -17,7 +17,6 @@ import java.util.HashMap; import java.util.Map; import static cn.ac.iie.utils.general.TransFormUtils.dealCommonMessage; -import static cn.ac.iie.utils.hbase.HBaseUtils.change; /** * 通联关系日志补全 @@ -38,7 +37,7 @@ public class ProxyCompletionBolt extends BaseBasicBolt { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { try { if (TupleUtils.isTick(tuple)) { - change(); + HBaseUtils.change(); } else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { diff --git a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java index 7a4182e..9a6b303 100644 --- a/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java +++ b/src/main/java/cn/ac/iie/bolt/security/SecurityCompletionBolt.java @@ -39,8 +39,7 @@ public class SecurityCompletionBolt extends BaseBasicBolt { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { try { if (TupleUtils.isTick(tuple)) { - HBaseUtils hBaseUtils = new HBaseUtils(FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS, FlowWriteConfig.HBASE_TABLE_NAME); - hBaseUtils.change(); + HBaseUtils.change(); } else { String message = tuple.getString(0); if (StringUtil.isNotBlank(message)) { diff --git a/src/main/java/cn/ac/iie/spout/CustomizedKafkaSpout.java b/src/main/java/cn/ac/iie/spout/CustomizedKafkaSpout.java index 0f806fa..d89ec41 100644 --- a/src/main/java/cn/ac/iie/spout/CustomizedKafkaSpout.java +++ b/src/main/java/cn/ac/iie/spout/CustomizedKafkaSpout.java @@ -37,6 +37,7 @@ public class CustomizedKafkaSpout extends BaseRichSpout { props.put("max.poll.records", 3000); props.put("max.partition.fetch.bytes", 31457280); props.put("auto.offset.reset", FlowWriteConfig.AUTO_OFFSET_RESET); + props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); return props; diff --git a/src/main/java/cn/ac/iie/utils/general/CompleteUtil.java b/src/main/java/cn/ac/iie/utils/general/CompleteUtil.java deleted file mode 100644 index b786a4a..0000000 --- a/src/main/java/cn/ac/iie/utils/general/CompleteUtil.java +++ /dev/null @@ -1,143 +0,0 @@ -package cn.ac.iie.utils.general; - -import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.hbase.HBaseUtils; -import cn.ac.iie.utils.system.SnowflakeId; -import com.google.common.net.InternetDomainName; -import com.zdjizhi.utils.IpLookup; -import com.zdjizhi.utils.StringUtil; -import org.apache.log4j.Logger; - -import java.util.Base64; - - -public final class CompleteUtil { - - private static Logger logger = Logger.getLogger(TransFormUtils.class); - private static IpLookup ipLookup = new IpLookup.Builder(false) - .loadDataFileV4(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") - .loadDataFileV6(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") - .loadAsnDataFileV4(FlowWriteConfig.IP_LIBRARY + "asn_v4.mmdb") - .loadAsnDataFileV6(FlowWriteConfig.IP_LIBRARY + "asn_v6.mmdb") - .build(); - - /** - * 有sni通过sni获取域名，有host根据host获取域名 - * - * @param sni sni - * @param host host - * @return 顶级域名 - */ - public static String getTopDomain(String sni, String host) { - if (StringUtil.isNotBlank(host)) { - return getDomainName(host); - } else if (StringUtil.isNotBlank(sni)) { - return getDomainName(sni); - } else { - return ""; - } - } - - - /** - * 根据url截取顶级域名 - * - * @param host 网站url - * @return 顶级域名 - */ - public static String getDomainName(String host) { - String domain = ""; - try { - domain = InternetDomainName.from(host).topPrivateDomain().toString(); - } catch (Exception e) { - logger.error("host解析顶级域名异常: " + e.getMessage()); - } - return domain; - } - - /** - * 生成当前时间戳的操作 - */ - public static int getCurrentTime() { - return (int)(System.currentTimeMillis() / 1000); - } - - /** - * 雪花模型生成id - * - * @return - */ - public static long getSnowflakeId(String zookeeperIp, String kafkaTopic, long dataCenterIdNum) { - - return SnowflakeId.generateId(zookeeperIp,kafkaTopic,dataCenterIdNum); - } - - /** - * 根据clientIp获取location信息 - * - * @param ip - * @return - */ - public static String getGeoIpDetail(String ip) { - - return ipLookup.cityLookupDetail(ip); - } - - /** - * 根据ip获取asn信息 - * - * @param ip - * @return asn - */ - public static String getGeoAsn(String ip) { - - return ipLookup.asnLookup(ip, true); - } - - /** - * 根据ip获取country信息 - * - * @param ip - * @return country - */ - public static String getGeoIpCountry(String ip) { - - return ipLookup.countryLookup(ip); - } - - /** - * 根据ip去hbase中匹配对应的用户名 - * @param clientIp - * @param hbaseZookeeper - * @param hbaseTable - * @return 用户名 subscriber_id - */ - public static String radiusMatch(String clientIp, String hbaseZookeeper, String hbaseTable) { - return HBaseUtils.getAccount(clientIp,hbaseZookeeper,hbaseTable); - } - - - /** - * base64 解码 - * - * @param encodedText mail subject - * @param subjectCharset 编码格式 - * @return 解码内容 / 空 - */ - public static String base64Str(String encodedText, String subjectCharset) { - Base64.Decoder decoder = Base64.getDecoder(); - String sub; - try { - if (StringUtil.isNotBlank(subjectCharset)) { - sub = new String(decoder.decode(encodedText), subjectCharset); - } else { - sub = new String(decoder.decode(encodedText), "UTF-8"); - } - return sub; - } catch (Exception e) { - logger.error("transform base64 String failed! base64Str = " + encodedText + "Charset = " + subjectCharset + "error :" + e); - return ""; - } - } - -} diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java index fb028eb..3c7b318 100644 --- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java @@ -4,18 +4,15 @@ package cn.ac.iie.utils.general; import cn.ac.iie.common.FlowWriteConfig; import cn.ac.iie.utils.hbase.HBaseUtils; import cn.ac.iie.utils.json.JsonParseUtil; -import cn.ac.iie.utils.system.SnowflakeId; import com.alibaba.fastjson.JSONObject; import com.google.common.net.InternetDomainName; +import com.zdjizhi.utils.FormatUtils; import com.zdjizhi.utils.IpLookup; import com.zdjizhi.utils.StringUtil; import org.apache.log4j.Logger; -import org.junit.jupiter.api.Test; import java.util.*; -import java.util.regex.Pattern; -import static cn.ac.iie.utils.general.CompleteUtil.*; /** @@ -34,7 +31,15 @@ public class TransFormUtils { private static Object mapObject = JsonParseUtil.generateObject(map); //获取任务列表 private static ArrayList jobList = JsonParseUtil.getJobListFromHttp(FlowWriteConfig.SCHEMA_HTTP); - + //补全工具类 + private static FormatUtils build = new FormatUtils.Builder(false).build(); + //IP定位库工具类 + private static IpLookup ipLookup = new IpLookup.Builder(false) + .loadDataFileV4(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") + .loadDataFileV6(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb") + .loadAsnDataFileV4(FlowWriteConfig.IP_LIBRARY + "asn_v4.mmdb") + .loadAsnDataFileV6(FlowWriteConfig.IP_LIBRARY + "asn_v6.mmdb") + .build(); /** * 解析日志，并补全 * @@ -43,10 +48,7 @@ public class TransFormUtils { */ public static String dealCommonMessage(String message) { - - Object object = JSONObject.parseObject(message, mapObject.getClass()); - // System.out.println("补全之前 ===》 "+JSON.toJSONString(object)); try { for (String[] strings : jobList) { @@ -66,17 +68,17 @@ public class TransFormUtils { if (function.equals("current_timestamp")) { JsonParseUtil.setValue(object, strings[1], getCurrentTime()); } else if (function.equals("snowflake_id")) { - JsonParseUtil.setValue(object, strings[1], getSnowflakeId(FlowWriteConfig.ZOOKEEPER_SERVERS,FlowWriteConfig.KAFKA_TOPIC,FlowWriteConfig.DATA_CENTER_ID_NUM)); + JsonParseUtil.setValue(object, strings[1], build.getSnowflakeId(FlowWriteConfig.ZOOKEEPER_SERVERS,FlowWriteConfig.KAFKA_TOPIC,FlowWriteConfig.DATA_CENTER_ID_NUM)); } else if (function.equals("geo_ip_detail")) { JsonParseUtil.setValue(object, strings[1], getGeoIpDetail(name.toString())); } else if (function.equals("geo_asn")) { JsonParseUtil.setValue(object, strings[1], getGeoAsn(name.toString())); } else if (function.equals("radius_match")) { - JsonParseUtil.setValue(object, strings[1], radiusMatch(name.toString(),FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS,FlowWriteConfig.HBASE_TABLE_NAME)); + JsonParseUtil.setValue(object, strings[1], radiusMatch(name.toString())); } else if (function.equals("geo_ip_country")) { JsonParseUtil.setValue(object, strings[1], getGeoIpCountry(name.toString())); } else if (function.equals("decode_of_base64") && param != null){ - JsonParseUtil.setValue(object, strings[1], base64Str(name.toString(),param.toString())); + JsonParseUtil.setValue(object, strings[1], FormatUtils.base64Str(name.toString(),param.toString())); } else if (name.equals("http_host") && function.equals("sub_domain")) { if (appendTo == null || StringUtil.isBlank(appendTo.toString())) { JsonParseUtil.setValue(object, strings[1], getTopDomain(null, name.toString())); @@ -102,11 +104,96 @@ public class TransFormUtils { } - @Test - public void aaa() { - String sni = "www.baidu.com"; - System.out.println(getTopDomain(sni, null)); - System.out.println(getTopDomain(null,sni)); +// @Test +// public void aaa() { +// String sni = "www.baidu.com"; +// System.out.println(getTopDomain(sni, null)); +// System.out.println(getTopDomain(null,sni)); +// +// } + /** + * 有sni通过sni获取域名，有host根据host获取域名 + * + * @param sni sni + * @param host host + * @return 顶级域名 + */ + private static String getTopDomain(String sni, String host) { + if (StringUtil.isNotBlank(host)) { + return getDomainName(host); + } else if (StringUtil.isNotBlank(sni)) { + return getDomainName(sni); + } else { + return ""; + } + } + + + /** + * 根据url截取顶级域名 + * + * @param host 网站url + * @return 顶级域名 + */ + private static String getDomainName(String host) { + String domain = ""; + try { + domain = InternetDomainName.from(host).topPrivateDomain().toString(); + } catch (Exception e) { + logger.error("host解析顶级域名异常: " + e.getMessage()); + } + return domain; + } + + /** + * 生成当前时间戳的操作 + */ + private static int getCurrentTime() { + return (int)(System.currentTimeMillis() / 1000); + } + + + /** + * 根据clientIp获取location信息 + * + * @param ip + * @return + */ + private static String getGeoIpDetail(String ip) { + + return ipLookup.cityLookupDetail(ip); + } + + /** + * 根据ip获取asn信息 + * + * @param ip + * @return + */ + private static String getGeoAsn(String ip) { + + return ipLookup.asnLookup(ip, true); + } + + /** + * 根据ip获取country信息 + * + * @param ip + * @return + */ + private static String getGeoIpCountry(String ip) { + + return ipLookup.countryLookup(ip); + } + + /** + * radius借助hbase补齐 + * + * @param ip + * @return + */ + private static String radiusMatch(String ip) { + return HBaseUtils.getAccount(ip); } } \ No newline at end of file diff --git a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java index 5c0e4c9..5341e9d 100644 --- a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java +++ b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java @@ -1,5 +1,6 @@ package cn.ac.iie.utils.hbase; +import cn.ac.iie.common.FlowWriteConfig; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; @@ -12,55 +13,34 @@ import org.apache.log4j.Logger; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; + +/** + * HBase 工具类 + * + * @author qidaijie + */ public class HBaseUtils { private final static Logger logger = Logger.getLogger(HBaseUtils.class); private static Map subIdMap = new HashMap<>(333334); + // private static Map subIdMap = new ConcurrentSkipListMap<>(); private static Connection connection; private static Long time; - private static String zookeeperIp; - private static String hbaseTable; - - private static HBaseUtils hBaseUtils; - - private static void getHbaseInstance(String zookeeperServer, String hbaseTableName) { - hBaseUtils = new HBaseUtils(zookeeperServer, hbaseTableName); - } - - /** - * 构造函数-新-20191023 - */ - public HBaseUtils(String zookeeperServer, String hbaseTableName) { - zookeeperIp = zookeeperServer; - hbaseTable = hbaseTableName; - //获取连接 - getHbaseConn(); - //拉取所有 - getAll(); - //定时更新 - updateHabaseCache(); - } - - private static void getHbaseConn() { + static { + // 管理Hbase的配置信息 + Configuration configuration = HBaseConfiguration.create(); + // 设置zookeeper节点 + configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS); + configuration.set("hbase.client.retries.number", "3"); + configuration.set("hbase.bulkload.retries.number", "3"); + configuration.set("zookeeper.recovery.retry", "3"); try { - // 管理Hbase的配置信息 - Configuration configuration = HBaseConfiguration.create(); - // 设置zookeeper节点 - configuration.set("hbase.zookeeper.quorum", zookeeperIp); - configuration.set("hbase.client.retries.number", "3"); - configuration.set("hbase.bulkload.retries.number", "3"); - configuration.set("zookeeper.recovery.retry", "3"); connection = ConnectionFactory.createConnection(configuration); time = System.currentTimeMillis(); - logger.warn("HBaseUtils get HBase connection,now to getAll()."); - } catch (IOException ioe) { - logger.error("HBaseUtils getHbaseConn() IOException===>{" + ioe + "}<==="); - ioe.printStackTrace(); - } catch (Exception e) { - logger.error("HBaseUtils getHbaseConn() Exception===>{" + e + "}<==="); + getAll(); + } catch (IOException e) { + logger.error("获取HBase连接失败"); e.printStackTrace(); } } @@ -69,7 +49,7 @@ public class HBaseUtils { * 更新变量 */ public static void change() { - long nowTime = System.currentTimeMillis(); + Long nowTime = System.currentTimeMillis(); timestampsFilter(time - 1000, nowTime + 500); } @@ -85,7 +65,7 @@ public class HBaseUtils { ResultScanner scanner = null; Scan scan2 = new Scan(); try { - table = connection.getTable(TableName.valueOf("sub:" + hbaseTable)); + table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); scan2.setTimeRange(startTime, endTime); scanner = table.getScanner(scan2); for (Result result : scanner) { @@ -103,14 +83,10 @@ public class HBaseUtils { } } Long end = System.currentTimeMillis(); - logger.warn("HBaseUtils Now subIdMap.keySet().size() is: " + subIdMap.keySet().size()); - logger.warn("HBaseUtils Update cache timeConsuming is: " + (end - begin) + ",BeginTime: " + begin + ",EndTime: " + end); + logger.warn("当前集合长度" + subIdMap.keySet().size()); + logger.warn("更新缓存耗时：" + (end - begin) + "开始时间：" + begin + "结束时间：" + end); time = endTime; - } catch (IOException ioe) { - logger.error("HBaseUtils timestampsFilter is IOException===>{" + ioe + "}<==="); - ioe.printStackTrace(); - } catch (Exception e) { - logger.error("HBaseUtils timestampsFilter is Exception===>{" + e + "}<==="); + } catch (IOException e) { e.printStackTrace(); } finally { if (scanner != null) { @@ -130,9 +106,9 @@ public class HBaseUtils { * 获取所有的 key value */ private static void getAll() { - long begin = System.currentTimeMillis(); + Long begin = System.currentTimeMillis(); try { - Table table = connection.getTable(TableName.valueOf("sub:" + hbaseTable)); + Table table = connection.getTable(TableName.valueOf("sub:" + FlowWriteConfig.HBASE_TABLE_NAME)); Scan scan2 = new Scan(); ResultScanner scanner = table.getScanner(scan2); for (Result result : scanner) { @@ -141,14 +117,10 @@ public class HBaseUtils { subIdMap.put(Bytes.toString(CellUtil.cloneRow(cell)), Bytes.toString(CellUtil.cloneValue(cell))); } } - logger.warn("HBaseUtils Get fullAmount List size->subIdMap.size(): " + subIdMap.size()); - logger.warn("HBaseUtils Get fullAmount List size->subIdMap.size() timeConsuming is: " + (System.currentTimeMillis() - begin)); + logger.warn("获取全量后集合长度：" + subIdMap.size()); + logger.warn("获取全量耗时：" + (System.currentTimeMillis() - begin)); scanner.close(); - } catch (IOException ioe) { - logger.error("HBaseUtils getAll() is IOException===>{" + ioe + "}<==="); - ioe.printStackTrace(); - } catch (Exception e) { - logger.error("HBaseUtils getAll() is Exception===>{" + e + "}<==="); + } catch (IOException e) { e.printStackTrace(); } } @@ -159,26 +131,8 @@ public class HBaseUtils { * @param clientIp client_ip * @return account */ - public static String getAccount(String clientIp, String hbaseZookeeper, String hbaseTable) { - if (hBaseUtils == null) { - getHbaseInstance(hbaseZookeeper, hbaseTable); - } + public static String getAccount(String clientIp) { return subIdMap.get(clientIp); } - private void updateHabaseCache() { - Timer timer = new Timer(); - timer.scheduleAtFixedRate(new TimerTask() { - @Override - public void run() { - try { - change(); - } catch (Exception e) { - logger.error("HBaseUtils update hbaseCache is error===>{" + e + "}<==="); - e.printStackTrace(); - } - } - }, 1, 1000 * 60); - } - } diff --git a/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java b/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java deleted file mode 100644 index 8d67348..0000000 --- a/src/main/java/cn/ac/iie/utils/system/SnowflakeId.java +++ /dev/null @@ -1,190 +0,0 @@ -package cn.ac.iie.utils.system; - -import cn.ac.iie.utils.zookeeper.ZooKeeperLock; -import cn.ac.iie.utils.zookeeper.ZookeeperUtils; -import org.apache.log4j.Logger; - - -public class SnowflakeId { - private static Logger logger = Logger.getLogger(SnowflakeId.class); - - /** - * 开始时间截 (2018-08-01 00:00:00) max 17years - */ - private final long twepoch = 1564588800000L; - - /** - * 机器id所占的位数 - */ - private final long workerIdBits = 6L; - - /** - * 数据标识id所占的位数 - */ - private final long dataCenterIdBits = 4L; - - /** - * 支持的最大机器id，结果是63 (这个移位算法可以很快的计算出几位二进制数所能表示的最大十进制数) - * M << n = M * 2^n - */ - private final long maxWorkerId = -1L ^ (-1L << workerIdBits); - - /** - * 支持的最大数据标识id，结果是15 - */ - private final long maxDataCenterId = -1L ^ (-1L << dataCenterIdBits); - - /** - * 序列在id中占的位数 - */ - private final long sequenceBits = 14L; - - /** - * 机器ID向左移12位 - */ - private final long workerIdShift = sequenceBits; - - /** - * 数据标识id向左移17位(14+6) - */ - private final long dataCenterIdShift = sequenceBits + workerIdBits; - - /** - * 时间截向左移22位(4+6+14) - */ - private final long timestampLeftShift = sequenceBits + workerIdBits + dataCenterIdBits; - - /** - * 生成序列的掩码，这里为16383 - */ - private final long sequenceMask = -1L ^ (-1L << sequenceBits); - - /** - * 工作机器ID(0~63) - */ - private long workerId; - - /** - * 数据中心ID(0~15) - */ - private long dataCenterId; - - /** - * 毫秒内序列(0~16383) - */ - private long sequence = 0L; - - /** - * 上次生成ID的时间截 - */ - private long lastTimestamp = -1L; - - - private static SnowflakeId idWorker; - - private static ZookeeperUtils zookeeperUtils = new ZookeeperUtils(); - - private static void getSnowflakeldInstance(String zookeeperIp, String kafkaTopic, long dataCenterIdNum) { - idWorker = new SnowflakeId(zookeeperIp, kafkaTopic, dataCenterIdNum); - } - - /** - * 构造函数 - */ - private SnowflakeId(String zookeeperIp, String kafkaTopic, long dataCenterIdNum) { - ZooKeeperLock lock = new ZooKeeperLock(zookeeperIp, "/locks", "disLocks1"); - if (lock.lock()) { - int tmpWorkerId = zookeeperUtils.modifyNode("/Snowflake/" + kafkaTopic, zookeeperIp); - if (tmpWorkerId > maxWorkerId || tmpWorkerId < 0) { - throw new IllegalArgumentException(String.format("worker Id can't be greater than %d or less than 0", maxWorkerId)); - } - if (dataCenterIdNum > maxDataCenterId || dataCenterIdNum < 0) { - throw new IllegalArgumentException(String.format("datacenter Id can't be greater than %d or less than ", maxDataCenterId)); - } - this.workerId = tmpWorkerId; - this.dataCenterId = dataCenterIdNum; - try { - lock.unlock(); - } catch (InterruptedException ie) { - ie.printStackTrace(); - } catch (Exception e) { - e.printStackTrace(); - logger.error("This is not usual error!!!===>>>" + e + "<<<==="); - } - } - } - - /** - * 获得下一个ID (该方法是线程安全的) - * - * @return SnowflakeId - */ - private synchronized long nextId() { - long timestamp = timeGen(); - - //如果当前时间小于上一次ID生成的时间戳，说明系统时钟回退过这个时候应当抛出异常 - if (timestamp < lastTimestamp) { - throw new RuntimeException( - String.format("Clock moved backwards. Refusing to generate id for %d milliseconds", lastTimestamp - timestamp)); - } - - //如果是同一时间生成的，则进行毫秒内序列 - if (lastTimestamp == timestamp) { - sequence = (sequence + 1) & sequenceMask; - //毫秒内序列溢出 - if (sequence == 0) { - //阻塞到下一个毫秒,获得新的时间戳 - timestamp = tilNextMillis(lastTimestamp); - } - } - //时间戳改变，毫秒内序列重置 - else { - sequence = 0L; - } - - //上次生成ID的时间截 - lastTimestamp = timestamp; - - //移位并通过或运算拼到一起组成64位的ID - return ((timestamp - twepoch) << timestampLeftShift) - | (dataCenterId << dataCenterIdShift) - | (workerId << workerIdShift) - | sequence; - } - - /** - * 阻塞到下一个毫秒，直到获得新的时间戳 - * - * @param lastTimestamp 上次生成ID的时间截 - * @return 当前时间戳 - */ - protected long tilNextMillis(long lastTimestamp) { - long timestamp = timeGen(); - while (timestamp <= lastTimestamp) { - timestamp = timeGen(); - } - return timestamp; - } - - /** - * 返回以毫秒为单位的当前时间 - * - * @return 当前时间(毫秒) - */ - protected long timeGen() { - return System.currentTimeMillis(); - } - - - /** - * 静态工具类 - * - * @return - */ - public static Long generateId(String zookeeperIp, String kafkaTopic, long dataCenterIdNum) { - if (idWorker == null) { - getSnowflakeldInstance(zookeeperIp, kafkaTopic, dataCenterIdNum); - } - return idWorker.nextId(); - } -} \ No newline at end of file diff --git a/src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java b/src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java deleted file mode 100644 index 8d0a4d5..0000000 --- a/src/main/java/cn/ac/iie/utils/zookeeper/ZooKeeperLock.java +++ /dev/null @@ -1,140 +0,0 @@ -package cn.ac.iie.utils.zookeeper; - -import org.apache.log4j.Logger; -import org.apache.zookeeper.*; -import org.apache.zookeeper.data.Stat; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -public class ZooKeeperLock implements Watcher { - private static Logger logger = Logger.getLogger(ZooKeeperLock.class); - - private ZooKeeper zk = null; - private String rootLockNode; // 锁的根节点 - private String lockName; // 竞争资源，用来生成子节点名称 - private String currentLock; // 当前锁 - private String waitLock; // 等待的锁（前一个锁） - private CountDownLatch countDownLatch; // 计数器（用来在加锁失败时阻塞加锁线程） - private int sessionTimeout = 30000; // 超时时间 - - // 1. 构造器中创建ZK链接，创建锁的根节点 - public ZooKeeperLock(String zkAddress, String rootLockNode, String lockName) { - this.rootLockNode = rootLockNode; - this.lockName = lockName; - try { - // 创建连接，zkAddress格式为：IP:PORT - zk = new ZooKeeper(zkAddress, this.sessionTimeout, this); - // 检测锁的根节点是否存在，不存在则创建 - Stat stat = zk.exists(rootLockNode, false); - if (null == stat) { - zk.create(rootLockNode, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } - } catch (IOException | InterruptedException | KeeperException e) { - e.printStackTrace(); - logger.error("ZooKeeperLock Constructors ===>>>Node already exists!"); - } - } - - // 2. 加锁方法，先尝试加锁，不能加锁则等待上一个锁的释放 - public boolean lock() { - if (this.tryLock()) { - logger.warn("ZooKeeperLock method lock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] addZkLock(" + this.currentLock + ")success！"); - return true; - } else { - return waitOtherLock(this.waitLock, this.sessionTimeout); - } - } - - public boolean tryLock() { - // 分隔符 - String split = "_lock_"; - if (this.lockName.contains("_lock_")) { - throw new RuntimeException("lockName can't contains '_lock_' "); - } - try { - // 创建锁节点（临时有序节点） - this.currentLock = zk.create(this.rootLockNode + "/" + this.lockName + split, new byte[0], - ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL); - logger.warn("ZooKeeperLock method tryLock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] create zkLockNode(" + this.currentLock + ")success，begin to election..."); - // 取所有子节点 - List nodes = zk.getChildren(this.rootLockNode, false); - // 取所有竞争lockName的锁 - List lockNodes = new ArrayList(); - for (String nodeName : nodes) { - if (nodeName.split(split)[0].equals(this.lockName)) { - lockNodes.add(nodeName); - } - } - Collections.sort(lockNodes); - // 取最小节点与当前锁节点比对加锁 - String currentLockPath = this.rootLockNode + "/" + lockNodes.get(0); - if (this.currentLock.equals(currentLockPath)) { - return true; - } - // 加锁失败，设置前一节点为等待锁节点 - String currentLockNode = this.currentLock.substring(this.currentLock.lastIndexOf("/") + 1); - int preNodeIndex = Collections.binarySearch(lockNodes, currentLockNode) - 1; - this.waitLock = lockNodes.get(preNodeIndex); - } catch (KeeperException | InterruptedException e) { - e.printStackTrace(); - } - return false; - } - - private boolean waitOtherLock(String waitLock, int sessionTimeout) { - boolean islock = false; - try { - // 监听等待锁节点 - String waitLockNode = this.rootLockNode + "/" + waitLock; - Stat stat = zk.exists(waitLockNode, true); - if (null != stat) { - logger.error("ZooKeeperLock method waitOtherLock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] zkLock(" + this.currentLock + ")addZkLock fail,wait lock(" + waitLockNode + ")release..."); - // 设置计数器，使用计数器阻塞线程 - this.countDownLatch = new CountDownLatch(1); - islock = this.countDownLatch.await(sessionTimeout, TimeUnit.MILLISECONDS); - this.countDownLatch = null; - if (islock) { - logger.warn("ZooKeeperLock method waitOtherLock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] zkLock(" + this.currentLock + ")addZkLock success,lock(" + waitLockNode + ")release over."); - } else { - logger.error("ZooKeeperLock method waitOtherLock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] zkLock(" + this.currentLock + "addZkLock fail..."); - } - } else { - islock = true; - } - } catch (KeeperException | InterruptedException e) { - e.printStackTrace(); - } - return islock; - } - - // 3. 释放锁 - public void unlock() throws InterruptedException { - try { - Stat stat = zk.exists(this.currentLock, false); - if (null != stat) { - logger.warn("ZooKeeperLock method unlock() ===>>> zkLockProcess[[[" + Thread.currentThread().getName() + "]]] free zkLock " + this.currentLock); - zk.delete(this.currentLock, -1); - this.currentLock = null; - } - } catch (InterruptedException | KeeperException e) { - e.printStackTrace(); - } finally { - zk.close(); - } - } - - // 4. 监听器回调 - @Override - public void process(WatchedEvent watchedEvent) { - if (null != this.countDownLatch && watchedEvent.getType() == Event.EventType.NodeDeleted) { - // 计数器减一，恢复线程操作 - this.countDownLatch.countDown(); - } - } - -} diff --git a/src/main/java/cn/ac/iie/utils/zookeeper/ZookeeperUtils.java b/src/main/java/cn/ac/iie/utils/zookeeper/ZookeeperUtils.java deleted file mode 100644 index 4a022e9..0000000 --- a/src/main/java/cn/ac/iie/utils/zookeeper/ZookeeperUtils.java +++ /dev/null @@ -1,136 +0,0 @@ -package cn.ac.iie.utils.zookeeper; - -import org.apache.commons.lang3.RandomUtils; -import org.apache.log4j.Logger; -import org.apache.zookeeper.*; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.CountDownLatch; - - -public class ZookeeperUtils implements Watcher { - private static Logger logger = Logger.getLogger(ZookeeperUtils.class); - - private ZooKeeper zookeeper; - - private static final int SESSION_TIME_OUT = 20000; - - private CountDownLatch countDownLatch = new CountDownLatch(1); - - @Override - public void process(WatchedEvent event) { - if (event.getState() == Watcher.Event.KeeperState.SyncConnected) { - countDownLatch.countDown(); - } - } - - - /** - * 修改节点信息 - * - * @param path 节点路径 - */ - public int modifyNode(String path, String zookeeperIp) { - createNode(path, "0".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, zookeeperIp); - int workerId = 0; - try { - connectZookeeper(zookeeperIp); - Stat stat = zookeeper.exists(path, true); - workerId = Integer.parseInt(getNodeDate(path)); - if (workerId > 55) { - workerId = 0; - zookeeper.setData(path, "1".getBytes(), stat.getVersion()); - } else { - String result = String.valueOf(workerId + 1); - if (stat != null) { - zookeeper.setData(path, result.getBytes(), stat.getVersion()); - } else { - logger.error("Node does not exist!,Can't modify"); - } - } - } catch (KeeperException | InterruptedException e) { - e.printStackTrace(); - workerId = RandomUtils.nextInt(56, 63); - } - finally { - closeConn(); - } - logger.warn("workerID is：" + workerId); - return workerId; - } - - /** - * 连接zookeeper - * - * @param host 地址 - */ - public void connectZookeeper(String host) { - try { - zookeeper = new ZooKeeper(host, SESSION_TIME_OUT, this); - countDownLatch.await(); - } catch (IOException | InterruptedException e) { - e.printStackTrace(); - } - } - - /** - * 关闭连接 - */ - public void closeConn() { - try { - if (zookeeper != null) { - zookeeper.close(); - } - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - /** - * 获取节点内容 - * - * @param path 节点路径 - * @return 内容/异常null - */ - public String getNodeDate(String path) { - String result = null; - Stat stat = new Stat(); - try { - byte[] resByte = zookeeper.getData(path, true, stat); - result = new String(resByte); - } catch (KeeperException | InterruptedException e) { - logger.error("Get node information exception"); - e.printStackTrace(); - } - return result; - } - - /** - * @param path 节点创建的路径 - * @param date 节点所存储的数据的byte[] - * @param acls 控制权限策略 - */ - public void createNode(String path, byte[] date, List acls, String zookeeperIp) { - try { - connectZookeeper(zookeeperIp); - Stat exists = zookeeper.exists(path, true); - if (exists == null) { - Stat existsSnowflakeld = zookeeper.exists("/Snowflake", true); - if (existsSnowflakeld == null) { - zookeeper.create("/Snowflake", null, acls, CreateMode.PERSISTENT); - } - zookeeper.create(path, date, acls, CreateMode.PERSISTENT); - } else { - logger.warn("Node already exists ! Don't need to create"); - } - } catch (KeeperException | InterruptedException e) { - e.printStackTrace(); - } finally { - closeConn(); - } - } - -} diff --git a/src/test/java/cn/ac/iie/test/DistributedLock.java b/src/test/java/cn/ac/iie/test/DistributedLock.java index 030bf4a..0d45cb8 100644 --- a/src/test/java/cn/ac/iie/test/DistributedLock.java +++ b/src/test/java/cn/ac/iie/test/DistributedLock.java @@ -10,7 +10,7 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import cn.ac.iie.common.FlowWriteConfig; -import cn.ac.iie.utils.zookeeper.ZookeeperUtils; +import com.zdjizhi.utils.ZookeeperUtils; import org.apache.log4j.Logger; import org.apache.zookeeper.*; import org.apache.zookeeper.data.Stat; @@ -183,7 +183,7 @@ public class DistributedLock implements Lock, Watcher { try { lock = new DistributedLock(FlowWriteConfig.ZOOKEEPER_SERVERS, "disLocks1"); lock.lock(); - zookeeperUtils.modifyNode("/Snowflake/" + FlowWriteConfig.KAFKA_TOPIC); + zookeeperUtils.modifyNode("/Snowflake/" + FlowWriteConfig.KAFKA_TOPIC,FlowWriteConfig.ZOOKEEPER_SERVERS); } finally { if (lock != null) { lock.unlock(); diff --git a/src/test/java/cn/ac/iie/test/test.java b/src/test/java/cn/ac/iie/test/test.java index e361e31..d4ba3c2 100644 --- a/src/test/java/cn/ac/iie/test/test.java +++ b/src/test/java/cn/ac/iie/test/test.java @@ -1,22 +1,53 @@ package cn.ac.iie.test; +import cn.ac.iie.common.FlowWriteConfig; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.google.common.net.InternetDomainName; +import com.zdjizhi.utils.*; import org.apache.log4j.Logger; +import org.apache.storm.shade.com.google.common.collect.Lists; +import org.junit.Test; + +import java.io.File; +import java.util.ArrayList; public class test { private static Logger logger = Logger.getLogger(test.class); - public static void main(String[] args) { - String message = "{\"common_schema_type\":\"SSL\",\"ssl_sni\":\"www.myssl.cn\",\"common_server_ip\":\"101.37.81.250\",\"common_client_ip\":\"192.168.50.12\",\"common_server_port\":443,\"common_client_port\":51398,\"common_stream_dir\":3,\"common_address_type\":4,\"common_s2c_pkt_num\":1,\"common_s2c_byte_num\":0,\"common_c2s_pkt_num\":3,\"common_c2s_byte_num\":517,\"common_stream_trace_id\":7629423224647,\"common_l4_protocol\":\"IPv4_TCP\",\"common_address_list\":\"51398-443-192.168.50.12-101.37.81.250\",\"common_sled_ip\":\"192.168.40.161\",\"common_start_time\":1575008799,\"common_end_time\":1575008799,\"common_con_duration_ms\":0,\"common_policy_id\":282,\"common_service\":0,\"common_action\":128,\"common_user_region\":\"{\\\"protocol\\\":\\\"SSL\\\"}\"}"; - JSONObject jsonObject = JSON.parseObject(message); - String common_server_ip = jsonObject.getString("common_server_ip"); - String common_client_ip = jsonObject.getString("common_client_ip"); + static ZookeeperUtils zookeeperUtils = new ZookeeperUtils(); - jsonObject.put("common__recv_time",(System.currentTimeMillis() / 1000)); +// + @Test + public void test() throws InterruptedException { + File file = new File("D:\\123\\test.txt"); + String zookeeperIp ="192.168.40.224:2181"; + String kafkaTopic ="CONNECTION-RECORD-LOG"; + + System.out.println(zookeeperUtils.modifyNode("/Snowflake/" + kafkaTopic, zookeeperIp)); + System.out.println(zookeeperUtils.modifyNode("/Snowflake/" + kafkaTopic, zookeeperIp)); + +// ArrayList list = Lists.newArrayList(); +// for (int i = 1; i <= 500; i++) { +// ZooKeeperLock lock = new ZooKeeperLock(zookeeperIp, "/locks", "disLocks1"); +// if (lock.lock()) { +// int tmpWorkerId = zookeeperUtils.modifyNode("/Snowflake/" + kafkaTopic, zookeeperIp); +// Long generateId = SnowflakeId.generateId(tmpWorkerId, 12); +// System.err.println(generateId); +// list.add(generateId); +// lock.unlock(); +// } +// if(i%5==0) { +//// fileWrite(list, file); +// Thread.sleep(1000); +// } +// } +// System.err.println("第2个进程结束"); + +// FormatUtils build = new FormatUtils.Builder(false).build(); +// long snowflakeId = build.getSnowflakeId("192.168.40.224:2181", "CONNECTION-RECORD-LOG", 12); +// System.err.println(snowflakeId); - System.out.println(jsonObject.toString()); } } From 31456351a3a3077414c1112fda048450a36029c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E7=8E=BA=E5=BA=B7?= Date: Tue, 25 Feb 2020 15:12:03 +0800 Subject: [PATCH 27/27] Deleted .idea/artifacts/completeUtil.xml, .idea/compiler.xml, .idea/encodings.xml, .idea/vcs.xml, out/artifacts/completeUtil/completeUtil.jar files --- .idea/artifacts/completeUtil.xml | 12 ------------ .idea/compiler.xml | 16 ---------------- .idea/encodings.xml | 6 ------ .idea/vcs.xml | 6 ------ out/artifacts/completeUtil/completeUtil.jar | Bin 8827 -> 0 bytes 5 files changed, 40 deletions(-) delete mode 100644 .idea/artifacts/completeUtil.xml delete mode 100644 .idea/compiler.xml delete mode 100644 .idea/encodings.xml delete mode 100644 .idea/vcs.xml delete mode 100644 out/artifacts/completeUtil/completeUtil.jar diff --git a/.idea/artifacts/completeUtil.xml b/.idea/artifacts/completeUtil.xml deleted file mode 100644 index f6f9995..0000000 --- a/.idea/artifacts/completeUtil.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - $PROJECT_DIR$/out/artifacts/completeUtil - - - - - - - - - \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml deleted file mode 100644 index 7603751..0000000 --- a/.idea/compiler.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml deleted file mode 100644 index b26911b..0000000 --- a/.idea/encodings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/out/artifacts/completeUtil/completeUtil.jar b/out/artifacts/completeUtil/completeUtil.jar deleted file mode 100644 index 866f81531cac1aa1de0370af015718366addd1e5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8827 zcmZu%WlUY$w%$0!p}4!dODXPdWn&vJ?(Xhx#a)ZL6nA%bcXtW}UVC2hPVde8)~}VB zF|#sPGCmnoQ3e7M9RLdp3m}shR|Nb8Q~)GE#Ky|j(%8{h&C$$~(OlnI|7`>Ss8Cl@ zz>`4nme>)ne040k%Rv)Xqc8BMsZbMCvej&0GWBX|#M)fXdRRayA>~k;jprm+6$(oe z2b&8i_?}i`o&*Ilgyc1hX>EKQ-345u{&kz=3ERIt3q1j1OHmHZZ5r$0=oDnL_87FHQE# zpKlUaaq$qZftJCGVP+~5ejOgIW2qWuT{F@b{&|uXqbK36d`k_V^k`;sILy|{b*Lhl z-aLoZsX7P&-h*(2ZM7j3wnb1^iUcxd^q3YC(hVG5Q4x<==2cegorF<1s-AQ(nrpC` zK)EzzYFS&oe91y|w;SDlNYeq7{as}ga*Va{Q@k4XH3X8O119_@35J{Hzy(aFc3U^M z4rT}A=Mk(Uoy4E$Hd~q)4^ED&oHD6ddI>gQmMad=oJbxMHz%S#K&)j<13BkGS}EQJ zOGc#gr6Eaj7Jq+TT0DMy}xz)o7X=j_O`H!2RG+MK}AS& zNR05WYGRv4KPzu^Pi4~zlExZqFQ-*Vq_vkmFDyLftY$8id^t)J(iTb$kd#;rx_n5&}sTe_O*=ypzSrRq&Pn1 zQ?v;(M7Za$25%4%5lueS-kc?y*mys&x&F8mcvI4l9iI?Y!F#A*HlTGui{g^cLNrAU zY!1X)J8-Mownr2Z%NZz9=E$`?2$PNWZvi&dRWyFP;szsJX9`@SY7qi&CaA?iK3T)T zg!<`UW?s}!JWSjmuX_o@1eC zb~Re;_OQGYIUU4P$D2f^K3=WhUFMLVBZY@c62LyO0N4F!*{ZqXHi%J2=#t(<3ii@F z?)Q^?4zsVqJ>0KCd|s~34!*tbObM*`*-tOI;aYY`&s{9Fa;x*L`1So{{fw%k9?@(R zhh&|`M^Xj4L6DaTWNIoEVyfikG+Wn-rQkWB;jEh9R&g#Y776%~)VvU-yKrDeMy_sQ z(4l7Qqil?W2#k5S|ceZz5-->q9xun|AC7Z;{BsA=-pl z)4N;}=vWo1TbX;mD0=^eLhHEOl<$rG1+zChWi!x8j&b&D!IP>k`ZIl;CVuMlrD_{- zYe$;Q{W7NQ<-^9Vcv9o{e5p|wqrA(ol6i@P;}dn6r!)eqN7m$lQ*Tr}&*ZWBoiz_4 zBi^l6(ACVZUooR$vDpU3Pv{XUY@{6sQW9e>n%l|h3~p*tdz7BiePP9ztF@>Ly*g8c zigoMx5f>#_&&Y~0Ft9xE^vHxz0KhIX0HFAP4OS&a^gkNx??r&5u)f1T-PPfr_Bs#L zvccy-_pVBO-@2H!{9(swB}8hL2bzly1r?C*g9XOAEG#>*@v+^z##@(37M_R|<_QVc zmk8z}hc-=xzy;>qO;)Gx>DpROjzu5VE(gM52#ocAew*kYJj7q6dZ0B^GHOj(N?{2X z>CON&;gRRC8A=R9S4Usk`F+~x>7#N^-{CZK&x)~{)agFl>eeKo@^`x!LotU6om3jc zi8`STt>N1LrQv#^5^OwUdt^M5>yfa6lwaAk>!BiDwq2(q6d3v(R5N(e@73q5B|w9e#8Fw{ z7i=1cCN}+SVj-$n-=n>gBj_7sRnQ<9?L993-YgdMRtwDy1XDMWt%rpn7ib{ocF^J| z*mR(V#ekdZ@2P)7W)Wg_3b-FS)GEy)46L-I%4PmCN?0RRI za|>IG{kYoR7AHgUG7301e0>e;ihrNIx!(S2c-ejWw#Y>FtN@b7uTp5|QWNkdElP=; zn|({#akte`mH{6z0(@2Gr9u(>x>Hy}N##QLzOd?9dwbyaeytGjw)8YpTwcnO`+h_s zdnC}d{(h0mvK_f=BkC(wCt|`1uZzr2=uF#ir)-j%)FWDxreaHutH*W5P7PB&5;ih~ z9W|xl^l9c3^@t#bycP0VZCUJ(LV(>;np}T1d>&%CV>=(IcZxfLCfsoDSO>Rj!IAGRDSb z6Hc+ewyTk42BWY&HZd`=UYcWwhMbiYr81SkMvh>x#9awQQ=Fi8Fi4kAljM%Q+?v@p z#v@%Ov`37nrggsK>|FM$*1%VtCF!yL*d#W-wKb#)L)X-XJQ)juY$gzB?!pIa(QcWC zOenZm;(}ZU71gV0>GFKe?6Z>bSL)8PNLqW0P)jah0Fm9FE6T&OFXkLGZWQ{tU5wls3 z2lC7s=OoTQg1;!DA%+;z4TYB(y4|4#wsgbPTlkgFD|_vvN+e;YfRI06({}q|%-n{6 z4}Fup7U3u?!k|m|(ipXqizrGk#3=G~o}dnvEvtUOwQ&;?$H*k|Rdx?bBe^S24Zk{_ z6}wIkcR1>cI5TZRS*dj(IW$FnxM^&-e>K@U-B*1nFG|O@(h*NpK01Ly@iajRRvuf> zC1@RSR+=`&T*s85LIV{982mky0im9G9^u*{*mjQ$N>D z4lfTv0R0g4a4g(je;a~Uu@0%t`Lv}(E_JX$#X}{UN#^D!PkD?3TsB9a!b@n7+rZhLN23d%}d*10r3D?n?1>K=|ZnaCJ*>2R3% zW^4>3B$b@5Dc4h@Xf`NI3!*|WOo?S>Vbf}|T+fS6M4S~5<_X-dEl-49h|GPZvfIGM zHPDL30|$GmL-b}EX^ez#v<#`@rHL5!m_G_aXK-423SaoOWeSFy~m_$=+Z9%8d9vf zR^_vY>T11@Y1q&t?qaj7(*#hfs`n&DxWRHzPB>!mLb;O{>!x1YvgZ?SkKL=!g>?;e zHbI9R?q3MQvhF7qy>-hg$L4giUDOW*DnZ{FJ~>?qR5JY9lb__9JZw=k@os!LjaRHc zG$_@fWL(i`Js2DF_wPwCtfl2Xpt<)PaPt8>^Y0~6FBXz^ax4Au3(q4XEIfh~j7CY1 z+3p?-dHqyo;=GtR0`F+e<3!sXu#+L-GpEb%Y}299c|YOSJcMOIM`W(RL}jh)gs0J0 z3ur`F67LT#IbVgyj;wNh9fyh)>tFZW55fEPXt2karJL;z;8jh|=9_iNbfFtWuG00m z1bHEV4&iqc?VH5`6mEOWt_Zu&pck$-Y>x(@uj~N`W~h)D-Xu6p*%Mq zQ2RCR>Ggy--tnWjV0wC8OUoO;_z)9s5aOLZR! zla68Y z9qyfrjRnka$2e>_(lwJTJHw|NuBUHqnUwQEW+unpulr6N6A(HAO$QWjcUBXu^ZPPj zEs6szuERR6M8oZ1y;Gs(Y-=*8r)Iu5xM1(}EGi!kU5jFdiwK)M1D7cBk0SPqAn=cD z_=|D@T`$kiD^v1wT6vXSgo`&+G-9KOaw6QV9~&~c?ZAiockG7*1@7+^a%Z%tp#N2+)O zJL?oZ9tvtsrZdzQRMYaTUgz>(t#E`gJWul0{TRxAQ2cC7h^vz+Vi}mZqE19-CLRL& z&Y&kEw&1cflpO4AYsMD~TPbS_p9cxB_x5L1)A1IAA8yAnF`m%P2s_1!u<;0$iN8ad z-X!QEo${e_(Mq(ACl?b)-_Nm}TwEqw&L{Z!Cj5|YC@9tv?fFsIG!7%`$;ZEMOa59^ z(Z-Tgs1hNT1jE);5c~>+VVEI_i`&+McQJ;Jn6$|x)-+}P03SUaAQ>{sKu9xHTUTJN zICp2%oTi$1<<#bIOE#Qdh{iu)qf~ax-n-fTLZa?ws4kSSEMPkAJMuCnXkIAu}i3fxXYfZWTWhXUT^56lFl> z=moq!M?5w(V6~k;c-8cb8gE5Jojtmb1b+Bp++~oSu+W`iba+R~GtW=$7i0Q-A=eC* zc2z4?;@F5$2|MU2IB)FGCTOHW!(osaP4`W>b!sEhSK;+$zo|vYv&TiedEKUH$4faO z_~f!tAs6TdX51GWrmkQkem!bZ&pIexS+AEERPdB*R2?~g@pd#Ou! zDc3`cN-4!mVO>OlEF;IZ#dSW^)NnMURI^X0jy|OFG!)!6H{J#i&xyl1_aE5Xh|H>!-OFMg0ldN z79_91CVj#JV)kx+YLl!kuRE0x$FGal4Xh?{6F0LmNKcEmkNAW`SLsXHghClb`r`<5 zho=8}clsfHY@6 z(BNFYiQJK8aKNx;x-3Qx#VTS9ZtMxt7neOu0Gt@LRP+g~H6oV(O%S8TBRO^%QYF>b zJiS_>H8J@S<h?PGu&`Lrg)eGiI;-hHCe-X|@RNWhO%)IMUKWa-|zkiEJPz~be=Ht74d)@Hp zbUQ}zh^nWq#i8VM)(LN&x%d2qdT5d8Tv8Gl9U0wxj)a9~H{DYpa;6q;wPEdWU>H-q z;O3dc9MY65RIBW`vRkoOwF|XjZWiX@RcwM|$11nVD9f0q&+Y>D zV)_1wtTByAB{3TisI*gq`DBv(1ltdx1w-l+v}O=!H)GkUwozKDXqX4 zU4zT+F>u?w?5gH^r*?}<(9LpQo2=M8^I5_ON>2naSN?-mq%uBG+Qg%CxDsq^sP({6 zeF)!;fhf7JlXe3o4Br!GpcHrk_)HA8yu9_speEVcZ6|vJ(zPr>1>}e4%?l#07O&>^ z_q(m+w_@bHil-sV5ZD=U0#fxj9@o1gp{vR9qb?&jWJ$?%K5GYuIKDY^$?e*2n2h&zEs|( zo2aqhup~M8Sw9rOJnR3ggM7teu|@_;UPuo>0f`;fO6N0k)Ebu3v}UPe?^i2EZr6#i zeIj*5Xw~lr*9KAyq1$ucqwbcjl3W_@^YnniaC1~mNg(v|l;SGfhFBp{9QdC_vBFW- zqe=a`P0~by4Eh}rQu;w!XnjQsu{$!_-YL_FAMUxlaw8ctnDmfc*|*()e!Ff~@XHSe zQu(d$qK?r9A&rqOzCRAP$J6=_6&BRz)Y1Xl)T9)#+sHqh9c>-HO;p(t;17bz){?28 zj`eNd#aV332qznSfP=)D!mYl%23S22eFb>y<(t*!}_L++a*c*I-|JalNMYG(9RyBHWJz26s&lm9Fk}9EhN^ zH{l*0COz%)ma8YCei>aE@wxuO)}w~zrXSd&h-(I^)ENK7O8gSP8c+srj!-N z4)qv@$X_~VZZ30C+Kx%vJ=nS~47WMo%0J>uNmX^9ZVK$I=qqPZcpipEYx%HPI(Dw) zr@@)x(hnb_C$hrKu_gB5Nt#~`uO4+n9<9I9^<+5eX>w4#=&r#!cOp0XsE53`5I%IS zs4!7tY0wyyg|&CyYT3R_zP(On{Z7KSFV=hcL|UDg?FykAm>0Liak}3zY)Yg~^EhpH z&>vZgXxT(Fttw}Fov672WznATBCELgYsLAChg8j2KP%F5rgEk$yPUN27Z4jF7>$yw zIXrh6Cy{v;x;H3U(5qo3%d=>pr?Fh6wq*&uUK=W%*e`xWy-pN2VH?@TPCH~qW058x zcLUnD6ul4~=2T#%895r3Wgq@&tZev5eAvH}LwA(F8|!~ttlx_O6>A%pua^22#!^QA z7_17Qj?L^B?2g|C3!l`=!IxJ~OS+-a6fd*LGzk8P-3&`|fJdsG+xaVBA}ep9qG_%! z0$kyg5TDYBo&{gGsg&lkiG9YMxsU#3jp5-^SOT*?)XCCe8`0!T#v>Vykn9JkNYNlH zQBXW{HV3)J;l0v&*YQSg2oh2J42~L``4?@adGrCC@@s*Tlct6B9NHpxtyR5EF9}^$ zmlDl*ELMN+NH+~R=hq?R(qZxidO9Sc5hJFN_)t-eDJMsek_1|9Y)=jM^Uo_1NOkc9 z#t+cqKS;2vCpIFCUKD)x`Lg?hy#`rGU1dy{ltsnFq6r&prj$gT9gMtIv^_nlp9V7A z!<$p>7#D7cf#T6P4-64g?FZT&0H26o930qqVvkYi2lxa1Oinjn9$Bn;|c2& zx`h4l^mh>{ZRWsPAPcsb4p=xQ=Xk0eiWTGGP@U6Yg^!DgbdbkEg7NM4R~M?Ckr$c% z%;ds*aG>G}kX?6^>NdT<2iHwN>_NBt;bwsL6IFs6YYVtV_|MdvH$&F@Z@l(&>Kz^G zATK5NZ!Q>d%XTh8OAQAIC#NEC;0}{rP0{eu4(cR*o9e_tR#wwOj$n3o$dU#%l?r5? zFN8}PD0x(xtj4}z9Rv72s^BS^*;FbKMmTwjYd=qWcPe+YYk{bC6bI8eIa*55h_QT~ z@#lvq^QJJSrkSmWA%%fpHyTb1Q_ZCp3skGEP^w_dYBJBAu|n zVB$wOH|PbD6tWmqJAOr>io`7Ut$H(ESYez+N zesOke$$w=6$*yTa&e*C6ey;^jy?dcrn_}e zHuJ1we;XY2F-cO%jC$@Grwevyt3Pw~AiK;NU;u&c8A2YlCoIMqjL*6xsSZcdTCaSz zWKZ3H&OVPD_!|n7UhX zLr+1j+!LD?*SFgc$CEA>l9?vYG?*Q37Fm`d!AUpARJDl7^@Zz5ffC7{&d8pL`?`i# z;G0%f<%e$(-)SG_u|QZNs6XV9yLGf@jy?Es?KA4CrHP{%w~u|am-46FtFVV8O;UId z5CyWo>?#}FmE#U-Q}6sB_#l*~-)>lLFwaL>O)KvPcQ{{0`07>CJ|OOn)0@U?kc*mN zoPoZ_)L_JkDCzB`(T5|VoFt9^6j)ja6#%~+2JR=Tr1+U_N-Q!SH+aj8SXOb0%FdK2 zE;84pzM}~uG*Vx`6VI>#^fS-QvUA#mnr9IZf# z>HUHR&SN9V=gU{V`e}DXVBk0{r}688Up{pGh;}60d@T;dpiW?H;Hd#T8BB(DKR2>( zVKBQO1JGIx=%q0(w#*3Zg#wnCYn#Hx3SB9wLKxrEC(57p$B{`qzq}}9@`@>Z4ED+G z==ki4W{whH107cOsYz~w3Dc!0f{*7Y)evHe-=;DZc|&LxAc(+JqBImy;OBo%r^FDUr|i9q~)bjeZLX$z+vK zmCVCMoK~GP1G(01|1W%WXfb9+l@`7b-VT!$OSe z8`<@P(5{m9GUT0r+VRzxf?0rEUWF9Nh zr)KwPUCyYm?4+i4nG#3%@WQX)pm*gA>^)n(gsBn9%UTU-lni_U0t_w)FkVKa;3vU3{+&B5Ilp|EU*IsG3=RreEcW3G^O- z3&sjPfN9)G%^%iwl{g>*Kl!YSOlczHfj7?pT(E=Cu7GAE-jZvTZT`hycv^YsyRysW z_V_^j2#b`=bDz`Z6C%Gm+ltxPawD)>*aK;La*^looFBF%=;T9DVLK*d;oyA>JIj*m zJn&&Lc&D^!>v?bmdUfrjAK{>Tl)x3QAylY7&*5d4>#7a4JtZ(5m7U}~LiqN$OK#3` zz-X^^Sd*jLlpAM7Vlyo+?|$x_9vA1E(V0~C>-dey$1p>V0(~4D?C6^}aPl_;{qbmU zges$lj&Tvpye|Ji#vMzX2C^*VvGD8i5c!|=-n9GuoptYFmFRU#tY> zaQ08y=>|LSF8ABxG+_RrVlZ%Yz`yq-{vPoCxhL_T_YqvAztI29 z|Nk`?{JZ#%82H