gfwleak/galaxy-tsg-olap-generate-baseline
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

丢弃全零的traffic数据

Browse Source
This commit is contained in:
yinjiangyi
2021-09-01 15:06:31 +08:00
parent b318da0db1
commit a124ac05ea
2 changed files with 23 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/main/java/cn/mesalab/service/BaselineSingleThread.java
View File

@@ -39,7 +39,8 @@ public class BaselineSingleThread extends Thread {
private final ArrayList<Integer> frequencyBinCounter = new ArrayList<>(Collections.nCopies(ApplicationConfig.MONITOR_FREQUENCY_BIN_NUM, 0)); private final ArrayList<Integer> frequencyBinCounter = new ArrayList<>(Collections.nCopies(ApplicationConfig.MONITOR_FREQUENCY_BIN_NUM, 0));
private final ArrayList<Integer> generateTypeCounter = new ArrayList<>(Collections.nCopies(3, 0)); private final ArrayList<Integer> generateTypeCounter = new ArrayList<>(Collections.nCopies(3, 0));
private int discardCounter = 0; private int discardBaselineCounter = 0;
private ArrayList<String> discardIpList = new ArrayList<>();
public BaselineSingleThread( public BaselineSingleThread(
List<String> attackTypeList, List<String> attackTypeList,
@@ -78,12 +79,12 @@ public class BaselineSingleThread extends Thread {
e.printStackTrace(); e.printStackTrace();
} }
try { try {
LOG.info("完成数据读取：获取Server IP：" + batchDruidData.size() + LOG.info("完成数据读取：获取Server IP：" + batchDruidData.size() +
" 运行时间：" + (System.currentTimeMillis() - start)); " 运行时间：" + (System.currentTimeMillis() - start));
} catch (Exception e){ } catch (Exception e){
e.printStackTrace(); e.printStackTrace();
} }
// 基线生成 // 基线生成
@@ -111,15 +112,15 @@ public class BaselineSingleThread extends Thread {
} }
try { try {
hbaseTable.put(putList); hbaseTable.put(putList);
LOG.info("MONITOR-IP频率分段统计: " + frequencyBinCounter); LOG.info("MONITOR-IP频率分段统计：" + frequencyBinCounter);
LOG.info("MONITOR-生成类别统计: " + generateTypeCounter); LOG.info("MONITOR-生成类别统计：" + generateTypeCounter);
LOG.info("MONITOR-无baseline生成的(IP,攻击类型)个数: " + discardCounter); LOG.info("MONITOR-无baseline生成的个数：" + discardBaselineCounter + " 其中包括IP共：" + discardIpList.size());
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} finally { } finally {
hbaseUtils.close(); hbaseUtils.close();
countDownLatch.countDown(); countDownLatch.countDown();
LOG.info("本线程更新字段数:" + putList.size() + " 剩余线程数量：" + countDownLatch.getCount()); LOG.info("本线程更新字段数：" + putList.size() + " 剩余线程数量：" + countDownLatch.getCount());
} }
} }
@@ -151,7 +152,7 @@ public class BaselineSingleThread extends Thread {
private Tuple3<int[], Integer, Integer> generateSingleIpBaseline(String ip, List<Map<String, Object>> ipDruidData){ private Tuple3<int[], Integer, Integer> generateSingleIpBaseline(String ip, List<Map<String, Object>> ipDruidData){
// 无数据（ip-攻击类型）不计算 // 无数据（ip-攻击类型）不计算
if (ipDruidData.size()==0){ if (ipDruidData.size()==0){
discardCounter += 1; updateDiscardCounter(ip);
return null; return null;
} }
List<Integer> originSeries = ipDruidData.stream().map(i -> List<Integer> originSeries = ipDruidData.stream().map(i ->
@@ -160,7 +161,7 @@ public class BaselineSingleThread extends Thread {
// 全零（ip-攻击类型）不计算 // 全零（ip-攻击类型）不计算
if(originNonZeroSeries.size()==0){ if(originNonZeroSeries.size()==0){
discardCounter += 1; updateDiscardCounter(ip);
return null; return null;
} }
@@ -245,4 +246,11 @@ public class BaselineSingleThread extends Thread {
generateTypeCounter.set(index, generateTypeCounter.get(index)+1); generateTypeCounter.set(index, generateTypeCounter.get(index)+1);
} }
private void updateDiscardCounter(String ip){
if(!discardIpList.contains(ip)){
discardIpList.add(ip);
}
discardBaselineCounter += 1;
}
} }

4
src/main/resources/application.properties
View File

@@ -2,12 +2,12 @@
############## 数据库配置 ############### ############## 数据库配置 ###############
########################################## ##########################################
#Druid配置 #Druid配置
druid.url=jdbc:avatica:remote:url=http://192.168.44.10:8082/druid/v2/sql/avatica/ druid.url=jdbc:avatica:remote:url=http://192.168.44.12:8082/druid/v2/sql/avatica/
druid.driver=org.apache.calcite.avatica.remote.Driver druid.driver=org.apache.calcite.avatica.remote.Driver
druid.table=traffic_top_destination_ip_metrics_log druid.table=traffic_top_destination_ip_metrics_log
#HBase配置 #HBase配置
hbase.table=ddos_traffic_baselines hbase.table=dos:ddos_traffic_baselines
hbase.zookeeper.quorum=192.168.44.12 hbase.zookeeper.quorum=192.168.44.12
hbase.zookeeper.client.port=2181 hbase.zookeeper.client.port=2181