gfwleak/galaxy-tsg-olap-generate-baseline
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

default value存在为0的情况，采用非0统计点的百分位数

Browse Source
This commit is contained in:
yinjiangyi
2021-08-25 10:05:13 +08:00
parent 8e79c9da54
commit eb6e7f3e16
2 changed files with 26 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/main/java/cn/mesalab/service/BaselineSingleThread.java
View File

@@ -39,6 +39,7 @@ public class BaselineSingleThread extends Thread {
private final ArrayList<Integer> frequencyBinCounter = new ArrayList<>(Collections.nCopies(ApplicationConfig.MONITOR_FREQUENCY_BIN_NUM, 0)); private final ArrayList<Integer> frequencyBinCounter = new ArrayList<>(Collections.nCopies(ApplicationConfig.MONITOR_FREQUENCY_BIN_NUM, 0));
private final ArrayList<Integer> generateTypeCounter = new ArrayList<>(Collections.nCopies(3, 0)); private final ArrayList<Integer> generateTypeCounter = new ArrayList<>(Collections.nCopies(3, 0));
private int discardCounter = 0;
public BaselineSingleThread( public BaselineSingleThread(
List<String> attackTypeList, List<String> attackTypeList,
@@ -77,8 +78,13 @@ public class BaselineSingleThread extends Thread {
e.printStackTrace(); e.printStackTrace();
} }
try {
LOG.info("完成数据读取：获取Server IP：" + batchDruidData.size() + LOG.info("完成数据读取：获取Server IP：" + batchDruidData.size() +
" 运行时间：" + (System.currentTimeMillis()- start)); " 运行时间：" + (System.currentTimeMillis() - start));
} catch (Exception e){
e.printStackTrace();
}
// 基线生成 // 基线生成
List<Put> putList = new ArrayList<>(); List<Put> putList = new ArrayList<>();
@@ -107,8 +113,9 @@ public class BaselineSingleThread extends Thread {
} }
try { try {
hbaseTable.put(putList); hbaseTable.put(putList);
LOG.info("MONITOR-IP频率分段统计:" + frequencyBinCounter); LOG.info("MONITOR-IP频率分段统计: " + frequencyBinCounter);
LOG.info("MONITOR-生成类别统计:" + generateTypeCounter); LOG.info("MONITOR-生成类别统计: " + generateTypeCounter);
LOG.info("MONITOR-无baseline生成的(IP,攻击类型)个数: " + discardCounter);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} finally { } finally {
@@ -144,20 +151,22 @@ public class BaselineSingleThread extends Thread {
* 3：其他类型IP, 采用百分位阈值基线 * 3：其他类型IP, 采用百分位阈值基线
*/ */
private Tuple3<int[], Integer, Integer> generateSingleIpBaseline(String ip, List<Map<String, Object>> ipDruidData){ private Tuple3<int[], Integer, Integer> generateSingleIpBaseline(String ip, List<Map<String, Object>> ipDruidData){
// 无数据（ip-攻击类型）不计算
if (ipDruidData.size()==0){ if (ipDruidData.size()==0){
discardCounter += 1;
return null; return null;
} }
List<Integer> originSeries = ipDruidData.stream().map(i -> List<Integer> originSeries = ipDruidData.stream().map(i ->
Integer.valueOf(i.get(ApplicationConfig.BASELINE_METRIC_TYPE).toString())).collect(Collectors.toList()); Integer.valueOf(i.get(ApplicationConfig.BASELINE_METRIC_TYPE).toString())).collect(Collectors.toList());
if(Collections.max(originSeries)==0){ List<Integer> originNonZeroSeries = originSeries.stream().filter(i->i>0).collect(Collectors.toList());
// 全零（ip-攻击类型）不计算
if(originNonZeroSeries.size()==0){
discardCounter += 1;
return null; return null;
} }
int ipPercentile = SeriesUtils.percentile(originSeries, ApplicationConfig.BASELINE_RATIONAL_PERCENTILE); int ipPercentile = SeriesUtils.percentile(originSeries, ApplicationConfig.BASELINE_RATIONAL_PERCENTILE);
if(ipPercentile == 0){
LOG.error(ip + "-" + "IP zero-replace value is 0 !");
}
int baselineGenerationType; int baselineGenerationType;
int[] baselineArr = new int[baselinePointNum]; int[] baselineArr = new int[baselinePointNum];
@@ -197,7 +206,13 @@ public class BaselineSingleThread extends Thread {
} }
updateLogGenerateTypeCounter(baselineGenerationType); updateLogGenerateTypeCounter(baselineGenerationType);
return new Tuple3<>(baselineArr, baselineGenerationType, ipPercentile); // 计算默认值-非零数据的百分位数
int defaultValue = SeriesUtils.percentile(originNonZeroSeries, ApplicationConfig.BASELINE_RATIONAL_PERCENTILE);
if(defaultValue == 0){
LOG.error(ip + "-" + "baseline default value is 0 !");
}
return new Tuple3<>(baselineArr, baselineGenerationType, defaultValue);
} }
/** /**

2
src/main/resources/application.properties
View File

@@ -69,7 +69,7 @@ monitor.frequency.bin.num=100
################ 并发参数 ################# ################ 并发参数 #################
########################################## ##########################################
#druid.read.batch.time.grad.hour=4 #druid.read.batch.time.grad.hour=4
thread.pool.num=10 thread.pool.num=20
#druid分区字段partition_num的最大值为9999 #druid分区字段partition_num的最大值为9999
druid.partition.num.max=10000 druid.partition.num.max=10000
druid.connection.retry.time.max=10 druid.connection.retry.time.max=10