gfwleak/galaxy-tsg-olap-generate-baseline
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改判断框架，优先判断周期性，增加p99/p50判断

Browse Source
This commit is contained in:
yinjiangyi
2021-10-27 19:35:07 +08:00
parent 251fefc061
commit 2cfe3cf88a
3 changed files with 20 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/cn/mesalab/config/ApplicationConfig.java
View File

@@ -48,6 +48,8 @@ public class ApplicationConfig {
public static final Integer BASELINE_RANGE_DAYS = ConfigUtils.getIntProperty("baseline.range.days"); public static final Integer BASELINE_RANGE_DAYS = ConfigUtils.getIntProperty("baseline.range.days");
public static final Float BASELINE_RATIONAL_PERCENTILE = ConfigUtils.getFloatProperty("baseline.rational.percentile"); public static final Float BASELINE_RATIONAL_PERCENTILE = ConfigUtils.getFloatProperty("baseline.rational.percentile");
public static final Float BASELINE_TIMES_PERCENT_TIMES = ConfigUtils.getFloatProperty("baseline.times.percent.times");
public static final String HBASE_TABLE = ConfigUtils.getStringProperty("hbase.table"); public static final String HBASE_TABLE = ConfigUtils.getStringProperty("hbase.table");
public static final String HBASE_ZOOKEEPER_QUORUM= ConfigUtils.getStringProperty("hbase.zookeeper.quorum"); public static final String HBASE_ZOOKEEPER_QUORUM= ConfigUtils.getStringProperty("hbase.zookeeper.quorum");

30
src/main/java/cn/mesalab/service/BaselineSingleThread.java
View File

@@ -183,12 +183,10 @@ public class BaselineSingleThread extends Thread {
List<Integer>series = completSeries.stream().map( List<Integer>series = completSeries.stream().map(
i -> Integer.valueOf(i.get(ApplicationConfig.BASELINE_METRIC_TYPE).toString())).collect(Collectors.toList()); i -> Integer.valueOf(i.get(ApplicationConfig.BASELINE_METRIC_TYPE).toString())).collect(Collectors.toList());
// 判断ip出现频率 // 具备周期性
float ipFrequency = ipDruidData.size() / (float) completSeries.size(); if(SeriesUtils.isPeriod(series)){
updateLogFrequencyCounter(ipFrequency);
if(ipFrequency >ApplicationConfig.BASELINE_HISTORICAL_FREQUENCY_THREAD){
// 异常值剔除
baselineGenerationType = 1; baselineGenerationType = 1;
// 异常值剔除
double exceptionPercentile = SeriesUtils.percentile(series, ApplicationConfig.BASELINE_EXECEPTION_PERCENTILE); double exceptionPercentile = SeriesUtils.percentile(series, ApplicationConfig.BASELINE_EXECEPTION_PERCENTILE);
double exceptionFillPercentile = SeriesUtils.percentile(series, ApplicationConfig.BASELINE_EXCECPTION_FILL_PERCENTILE); double exceptionFillPercentile = SeriesUtils.percentile(series, ApplicationConfig.BASELINE_EXCECPTION_FILL_PERCENTILE);
LOG.debug(ip + ": series-" + series); LOG.debug(ip + ": series-" + series);
@@ -197,24 +195,24 @@ public class BaselineSingleThread extends Thread {
series.set(i, (int) exceptionFillPercentile); series.set(i, (int) exceptionFillPercentile);
} }
} }
LOG.debug(ip + ": execpSeries-" + series);
// KF // KF
baselineArr = baselineFunction(series); baselineArr = baselineFunction(series);
// System.out.println("type-01：" + ipPercentile + " " + ip + " " + Arrays.toString(baselineArr));
LOG.debug(ip + ": baseline-" + baselineArr);
} else { } else {
// 判断周期性 double p99 = SeriesUtils.percentile(series, 0.99);
if (SeriesUtils.isPeriod(series)){ double p50 = SeriesUtils.percentile(series, 0.50);
// 无周期性
float ipFrequency = ipDruidData.size() / (float) completSeries.size();
updateLogFrequencyCounter(ipFrequency);
// p99/p50 > 10
if(ipFrequency >ApplicationConfig.BASELINE_HISTORICAL_FREQUENCY_THREAD && p99/p50 > ApplicationConfig.BASELINE_TIMES_PERCENT_TIMES){
baselineGenerationType = 2; baselineGenerationType = 2;
// KF
baselineArr = baselineFunction(series);
// System.out.println("type-02：" + ipPercentile + " " + ip + " " + Arrays.toString(baselineArr));
} else {
baselineGenerationType = 3;
Arrays.fill(baselineArr, ipPercentile); Arrays.fill(baselineArr, ipPercentile);
// System.out.println("type-03：" + ipPercentile + " " + ip + " " + Arrays.toString(baselineArr)); } else {
return null;
} }
} }
updateLogGenerateTypeCounter(baselineGenerationType); updateLogGenerateTypeCounter(baselineGenerationType);
// 计算默认值-非零数据的百分位数 // 计算默认值-非零数据的百分位数

5
src/main/resources/application.properties
View File

@@ -55,7 +55,10 @@ baseline.period.correlative.threshold=0.5
baseline.historical.frequency.thread=0.2 baseline.historical.frequency.thread=0.2
baseline.exception.percentile=0.99 baseline.exception.percentile=0.99
baseline.exception.fill.percentile=0.99 baseline.exception.fill.percentile=0.99
baseline.rational.percentile=0.95 baseline.rational.percentile=0.99
# p99/p50 倍数阈值
baseline.times.percent.times=10
#Kalman Filter #Kalman Filter
baseline.kalman.q=0.1 baseline.kalman.q=0.1