gfwleak/galaxy-tsg-olap-generate-baseline
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改异常值替换逻辑

Browse Source
This commit is contained in:
yinjiangyi
2021-08-06 17:51:36 +08:00
parent 5c33957c11
commit ef1f0fd826
4 changed files with 31 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/cn/mesalab/config/ApplicationConfig.java
View File

@@ -31,9 +31,14 @@ public class ApplicationConfig {
public static final String DRUID_ATTACKTYPE_COLUMN_NAME = ConfigUtils.getStringProperty("druid.attacktype.columnname");
public static final String DRUID_RECVTIME_COLUMN_NAME = ConfigUtils.getStringProperty("druid.recvtime.columnname");
// 周期性相关系数阈值
public static final Float BASELINE_PERIOD_CORR_THRE = ConfigUtils.getFloatProperty("baseline.period.correlative.threshold");
public static final Float BASELINE_HISTORICAL_RATIO = ConfigUtils.getFloatProperty("baseline.historical.ratio.threshold");
public static final Float BASELINE_SPARSE_FILL_PERCENTILE = ConfigUtils.getFloatProperty("baseline.historical.sparse.fill.percentile");
// 频率阈值
public static final Float BASELINE_HISTORICAL_FREQUENCY_THREAD = ConfigUtils.getFloatProperty("baseline.historical.frequency.thread");
// 异常值判断分位数
public static final Float BASELINE_EXECEPTION_PERCENTILE = ConfigUtils.getFloatProperty("baseline.exception.percentile");
// 异常值替换百分位
public static final Float BASELINE_EXCECPTION_FILL_PERCENTILE = ConfigUtils.getFloatProperty("baseline.exception.fill.percentile");
public static final String BASELINE_FUNCTION = ConfigUtils.getStringProperty("baseline.function");
public static final Integer BASELINE_RANGE_DAYS = ConfigUtils.getIntProperty("baseline.range.days");
public static final Float BASELINE_RATIONAL_PERCENTILE = ConfigUtils.getFloatProperty("baseline.rational.percentile");

12
src/main/java/cn/mesalab/service/BaselineSingleThread.java
View File

@@ -88,11 +88,15 @@ public class BaselineSingleThread extends Thread {
i -> Integer.valueOf(i.get(ApplicationConfig.BASELINE_METRIC_TYPE).toString())).collect(Collectors.toList());
// 判断ip出现频率
if(ipDruidData.size()/(float)completSeries.size()>ApplicationConfig.BASELINE_HISTORICAL_RATIO){
if(ipDruidData.size()/(float)completSeries.size()>ApplicationConfig.BASELINE_HISTORICAL_FREQUENCY_THREAD){
// 异常值剔除
double percentile = StatUtils.percentile(series.stream().mapToDouble(Double::valueOf).toArray(),
ApplicationConfig.BASELINE_SPARSE_FILL_PERCENTILE);
Arrays.fill(baselineArr, (int)percentile);
double exceptionPercentile = SeriesUtils.percentile(series, ApplicationConfig.BASELINE_EXECEPTION_PERCENTILE);
double exceptionFillPercentile = SeriesUtils.percentile(series, ApplicationConfig.BASELINE_EXCECPTION_FILL_PERCENTILE);
for(int i=0; i<series.size(); i++){
if(series.get(i) > exceptionPercentile){
series.set(i, (int) exceptionFillPercentile);
}
}
// KF
baselineArr = baselineFunction(series);
// System.out.println("高频IP:" + ip + " origin:" + series + "\n baseline:" + Arrays.toString(baselineArr));

13
src/main/java/cn/mesalab/utils/SeriesUtils.java
View File

@@ -184,9 +184,16 @@ public class SeriesUtils {
}
public static int percentile(List<Integer> latencies, double percentile) {
Collections.sort(latencies);
int index = (int) Math.ceil(percentile * latencies.size());
return latencies.get(index-1);
List<Integer> tmp = new ArrayList<>(latencies);
Collections.sort(tmp);
int index = (int) Math.ceil(percentile * tmp.size());
return tmp.get(index-1);
}
public static void main(String[] args) {
List<Integer> a = Arrays.asList(184, 134, 95, 133, 56, 128, 60, 65, 143, 149, 68, 150, 384, 516, 135, 171, 601, 778, 431, 175, 0, 0, 0, 2032, 0, 0, 0, 2978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2354, 2291, 2826, 3775, 6544, 6736, 7748, 7027, 6483, 5670, 3153, 382, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 5, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 9, 0, 0, 0, 53, 0, 20, 0, 0, 0, 0, 0, 0, 44, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 62, 0, 0, 0, 12, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 94, 65, 126, 27, 39, 39, 67, 120, 110, 34, 57, 198, 36, 77, 41, 65, 145, 140, 107, 100, 116, 262, 185, 168, 167, 395, 339, 522, 668, 1048, 2898, 3773, 4979, 6046, 6846, 6232, 6488, 7438, 5554, 6105, 6673, 6256, 6252, 5264, 5305, 5166, 5657, 6098, 5696, 3887, 4591, 5952, 6035, 6541, 8607, 8784, 9042, 10123, 9975, 10430, 10931, 10100, 9908, 8022, 4090, 1911, 404, 832, 304, 10, 22, 54, 0, 118, 80, 17, 25, 17, 30, 0, 22, 5, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 15, 22, 0, 0, 0, 24, 31, 0, 0, 47, 0, 25, 0, 12, 46, 37, 51, 0, 0, 4, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 16, 0, 0, 0, 0, 31, 0, 0, 0, 37, 10, 57, 0, 0, 15, 13, 22, 0, 13, 13, 25, 115, 214, 85, 180, 162, 206, 95, 55, 110, 191, 37, 232, 54, 83, 81, 130, 74, 43, 107, 45, 98, 63, 44, 154, 283, 109, 95, 207, 302, 1016, 1289, 3089, 3847, 757, 4157, 4025, 4783, 5196, 5423, 5503, 3997, 5291, 5957, 3795, 4170, 4649, 5265, 5371, 4950, 1272, 2451, 4251, 4393, 5535, 7303, 7958, 7865, 9039, 8410, 9304, 9496, 7115, 6135, 5335, 1425, 1486, 1124, 935, 905, 58, 40, 19, 0, 11, 49, 0, 28, 15, 25, 16, 0, 0, 0, 0, 0, 42, 0, 0, 0, 11, 0, 53, 0, 6, 0, 9, 117, 31, 70, 56, 33, 21, 11, 0, 102, 0, 39, 15, 46, 58, 102, 66, 17, 36, 0, 31, 0, 0, 24, 0, 245, 50, 50, 0, 0, 0, 0, 75, 12, 55, 0, 18, 0, 47, 14, 92, 0, 0, 10, 27, 3, 33, 0, 0, 159, 369, 453, 264, 115, 230, 286, 92, 674, 295, 64, 201);
System.out.println(percentile(a,0.99));
}
}

11
src/main/resources/application.properties
View File

@@ -43,14 +43,15 @@ time.format=yyyy-MM-dd HH:mm:ss
#算法参数
baseline.period.correlative.threshold=0.5
baseline.historical.ratio.threshold=0.1
baseline.historical.sparse.fill.percentile=0.95
baseline.historical.frequency.thread=0.1
baseline.exception.percentile=0.99
baseline.exception.fill.percentile=0.99
baseline.rational.percentile=0.95
#Kalman Filter
baseline.kalman.q=0.00001
baseline.kalman.q=0.1
baseline.kalman.r=0.1
baseline.kalman.p=1
baseline.kalman.m=1
baseline.kalman.p=8
baseline.kalman.m=2
# FOR TEST
baseline.generate.batch.size=100