gfwleak/galaxy-platform-galaxy-qgw-service
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into br-384

Browse Source
This commit is contained in:
wangwei
2024-11-19 16:56:52 +08:00
parent 82a8389b3a e4c5d10868
commit 61eb0c23e1
8 changed files with 92 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
docs/release/release-383.md
View File

@@ -19,3 +19,4 @@ Release 383 (TSG-24.10)
* 适配DoS Threat Map 业务相关Datasets及Schema(TSG-22745)
* 完善 TIME_FLOOR_WITH_FILL 补全函数对结果数据的排序
* 完善数据集，策略流量趋势数据集增加 bps 指标
* 完善补全函数，规范TIME_FLOOR_WITH_FILL填充值(TSG-23775)

8
pom.xml
View File

@@ -821,6 +821,14 @@
<skipTests>true</skipTests> <!-- 默认关掉单元测试-->
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>${maven.compiler.source}</source>
<target>${maven.compiler.target}</target>
</configuration>
</plugin>
</plugins>
</build>

128
src/main/java/com/mesalab/common/utils/ConvertUtil.java
View File

@@ -1,12 +1,16 @@
package com.mesalab.common.utils;
import cn.hutool.core.date.DateTime;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.*;
import com.mesalab.common.entity.DataTypeMapping;
import com.mesalab.common.support.IConvertor;
import com.geedgenetworks.utils.DateUtils;
import com.geedgenetworks.utils.StringUtil;
import lombok.Getter;
import org.nutz.lang.Lang;
import java.util.*;
/**
@@ -16,10 +20,12 @@ import java.util.*;
* @date 2018/5/9
*/
public class ConvertUtil {
public static final String GROUP_SEPARATOR = "__GALAXY__";
enum Fill {
public static final String GROUP_SEPARATOR = "__GALAXY__";
NONE(""), NULL("NULL"), ZERO("0"), PREVIOUS("previous"), NEXT("next");
@Getter
enum Fill {
NULL("NULL"), ZERO("zero"), PREVIOUS("previous"), NEXT("next");
private final String value;
@@ -27,10 +33,6 @@ public class ConvertUtil {
this.value = value;
}
public String getValue() {
return value;
}
}
@@ -135,16 +137,16 @@ public class ConvertUtil {
/**
* 将时序特征的数据，转换成层级结构；
* 例如：Map<groupKey, timeseries Map>
* Map<timeGran,Metrics Map>
* Map<metric,value>
* Map<timeGran,Metrics Map>
* Map<metric,value>
*
* @param sources
* @param groupLabels
* @param timeGranLabel
* @return
*/
public static Map<String, Map<String, Map<String, Object>>> convertResultListToTimeSeries(Collection<Object> sources,
List<String> groupLabels, String timeGranLabel) {
public static Map<String, Map<String, Map<String, Object>>> convertResultListToTimeSeries(Collection<Object> sources,
List<String> groupLabels, String timeGranLabel) {
Map<String, Map<String, Map<String, Object>>> resultMap = Maps.newLinkedHashMap();
for (Object source : sources) {
@@ -155,7 +157,7 @@ public class ConvertUtil {
if (StringUtil.isEmpty(groupLabels)) {
groupValues.add("default");
} else {
for(String groupLabel : groupLabels) {
for (String groupLabel : groupLabels) {
groupValues.add(String.valueOf(sourceMap.get(groupLabel)));
sourceMap.remove(groupLabel);
}
@@ -169,7 +171,7 @@ public class ConvertUtil {
if (StringUtil.isEmpty(groupMap)) {
groupMap = Maps.newTreeMap();
groupMap.put(timeGran, sourceMap);
resultMap.put(groupKey,groupMap);
resultMap.put(groupKey, groupMap);
} else {
@@ -184,19 +186,18 @@ public class ConvertUtil {
}
public static List<Map<String, Object>> convertTimeSeriesToList(Map<String, Map<String, Map<String, Object>>> target, List<String> groupLabels, String timeGranLabel) {
List<Map<String, Object>> results = Lists.newArrayList();
List<String> groupValues;
for(String groupKey : target.keySet()) {
for (String groupKey : target.keySet()) {
Map<String, Map<String, Object>> timeSeriesValueMap = target.get(groupKey);
for (Map.Entry<String, Map<String,Object>> entry: timeSeriesValueMap.entrySet()) {
Map<String, Map<String, Object>> timeSeriesValueMap = target.get(groupKey);
for (Map.Entry<String, Map<String, Object>> entry : timeSeriesValueMap.entrySet()) {
Map<String, Object> resultMap = Maps.newLinkedHashMap();
resultMap.put(timeGranLabel, entry.getKey());
if (StringUtil.isNotEmpty(groupLabels)) {
groupValues= Splitter.on(GROUP_SEPARATOR).splitToList(groupKey);
groupValues = Splitter.on(GROUP_SEPARATOR).splitToList(groupKey);
for (int i = 0; i < groupLabels.size(); i++) {
resultMap.put(groupLabels.get(i), StringUtil.isBlank(groupKey) ? StringUtil.EMPTY : groupValues.get(i));
}
@@ -215,35 +216,32 @@ public class ConvertUtil {
}
/**
* 时序数据补全操作
* @param sources TreeMap<timeGran, metricMap>
*
* @param sources TreeMap<timeGran, metricMap>
* @param dateList
* @param fill
* @return
*/
public static Map<String, Map<String, Object>> completeTimeseries(TreeMap<String, Map<String, Object>> sources,
List<Date> dateList, String fill) {
public static Map<String, Map<String, Object>> completeTimeseries(TreeMap<String, Map<String, Object>> sources, List<Date> dateList, List<Map<String, String>> metaList, String fill) {
Map<String, Object> defaultElement = Maps.newHashMap(sources.firstEntry().getValue());
if (fill.equalsIgnoreCase(Fill.ZERO.name())){
for(String key: defaultElement.keySet()){
defaultElement.put(key, Integer.valueOf(Fill.ZERO.getValue()));
if (fill.equalsIgnoreCase(Fill.ZERO.getValue())) {
Map<String, String> nameTypeMap = new HashMap<>();
for (Map<String, String> stringStringMap : metaList) {
nameTypeMap.put(stringStringMap.get("name"), stringStringMap.get("type"));
}
} else if (fill.equalsIgnoreCase(Fill.NULL.name())){
for(String key: defaultElement.keySet()) {
defaultElement.put(key, Fill.NULL.value);
}
} else if (fill.equalsIgnoreCase(Fill.NONE.name())) {
for(String key: defaultElement.keySet()) {
defaultElement.put(key, Fill.NONE.value);
for (String key : defaultElement.keySet()) {
String type = nameTypeMap.get(key);
if (type != null) {
Object zeroValue = getZeroValueByType(type);
defaultElement.put(key, zeroValue);
}
}
} else if (fill.equalsIgnoreCase(Fill.NULL.getValue())) {
defaultElement.replaceAll((k, v) -> null);
} else {
defaultElement.clear();
}
@@ -251,15 +249,14 @@ public class ConvertUtil {
TreeMap<String, Map<String, Object>> target = Maps.newTreeMap(sources);
boolean isTimestamp = StringUtil.isNumeric(sources.firstKey());
for(Date date : dateList) {
for (Date date : dateList) {
String timeKey;
if (isTimestamp) {
timeKey = String.valueOf(date.getTime()/1000);
timeKey = String.valueOf(date.getTime() / 1000);
} else {
timeKey = DateUtils.getFormatDate(date, DateUtils.YYYY_MM_DD_HH24_MM_SS);
}
Map<String, Object> metricsMap = sources.get(timeKey);
if (StringUtil.isEmpty(metricsMap)) {
target.put(timeKey, defaultElement);
@@ -267,76 +264,67 @@ public class ConvertUtil {
}
if (StringUtil.isEmpty(defaultElement)) {
defaultElement = getDefaultElement(sources.get(sources.firstKey()));
for (String key : target.keySet()) {
if (StringUtil.isEmpty(target.get(key))) {
Map<String, Object> element= getPositionElement(target, key, fill);
Map<String, Object> element = getPositionElement(target, key, fill);
if (StringUtil.isEmpty(element)) {
target.put(key, defaultElement);
} else {
target.put(key, element);
}
}
}
}
return target;
}
private static Object getZeroValueByType(String type) {
Object defaultValue = null;
if (type.startsWith(DataTypeMapping.ARRAY)) {
defaultValue = Lists.newArrayList();
} else if (DataTypeMapping.STRING.equalsIgnoreCase(type)) {
defaultValue = "";
} else if (DataTypeMapping.INT.equalsIgnoreCase(type) || DataTypeMapping.LONG.equalsIgnoreCase(type)) {
defaultValue = 0;
} else if (DataTypeMapping.FLOAT.equalsIgnoreCase(type) || DataTypeMapping.DOUBLE.equalsIgnoreCase(type)) {
defaultValue = 0.0;
} else if (DataTypeMapping.BOOLEAN.equalsIgnoreCase(type)) {
defaultValue = false;
} else if (DataTypeMapping.DATE.equalsIgnoreCase(type) || DataTypeMapping.TIMESTAMP.equalsIgnoreCase(type)) {
defaultValue = new DateTime(0);
}
return defaultValue;
}
private static Map<String, Object> getPositionElement(TreeMap<String, Map<String, Object>> target,
String key, String fill) {
String key, String fill) {
Map<String, Object> element = Maps.newHashMap();
if (fill.equalsIgnoreCase(Fill.PREVIOUS.name())) {
if (fill.equalsIgnoreCase(Fill.PREVIOUS.getValue())) {
if (StringUtil.isNotEmpty(target.lowerEntry(key))) {
element = target.lowerEntry(key).getValue();
}
} else if (fill.equalsIgnoreCase(Fill.NEXT.name())) {
} else if (fill.equalsIgnoreCase(Fill.NEXT.getValue())) {
if (StringUtil.isNotEmpty(target.higherEntry(key))) {
element = target.higherEntry(key).getValue();
}
}
return element;
}
private static Map<String, Object> getDefaultElement(Map<String, Object> firstElement) {
Map<String, Object> element = Maps.newHashMap(firstElement);
for (String key : element.keySet()) {
element.put(key, Fill.NONE.getValue());
element.put(key, null);
}
return element;
}
}

10
src/main/java/com/mesalab/common/utils/sqlparser/ExampleDataHelper.java
View File

@@ -3,7 +3,6 @@ package com.mesalab.common.utils.sqlparser;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.BooleanUtil;
import cn.hutool.core.util.CharsetUtil;
import com.alibaba.fastjson2.JSONPath;
import com.google.common.collect.Lists;
@@ -39,7 +38,7 @@ public class ExampleDataHelper {
public static final String QUERY_TYPE_GROUP_BY = "GroupBy";
public static final String QUERY_TYPE_AGG_STATISTICS = "AggStatistics";
public static final String QUERY_TYPE_OTHER = "Other";
private static final String[] JSONPATH_SPECIAL_CHARS = {"/", ".", "[", "]", "'", "\""};
private static final String[] JSONPATH_SPECIAL_CHARS = {"/", ".", "[", "]", "'", "\"", "(", ")"};
private ExampleDataHelper() {
}
@@ -214,11 +213,10 @@ public class ExampleDataHelper {
}
private static List getExampleValue(Map<String, String> meta, Object initValue, Boolean isDesc, boolean repeatable, int size, Map<String, List<Object>> schemaDataDict) {
boolean isArray = BooleanUtil.toBoolean(meta.get(MetaConst.IS_ARRAY));
List result = inExampleData(meta, schemaDataDict)
? getExampleDataValues(meta, repeatable, size, schemaDataDict)
: getOtherExampleValues(meta, isDesc, size);
return isArray ? convertToArrayList(result) : result;
: getExampleValuesByType(meta, isDesc, size);
return meta.get("type").startsWith(DataTypeMapping.ARRAY) ? convertToArrayList(result) : result;
}
private static List<List<Object>> convertToArrayList(List<Object> result) {
@@ -227,7 +225,7 @@ public class ExampleDataHelper {
.collect(Collectors.toList());
}
private static List getOtherExampleValues(Map<String, String> meta, Boolean isDesc, int size) {
private static List getExampleValuesByType(Map<String, String> meta, Boolean isDesc, int size) {
String dataType = meta.get(MetaConst.META_DATA_TYPE) == null ? meta.get(MetaConst.META_TYPE) : meta.get(MetaConst.META_DATA_TYPE);
if (DataTypeMapping.LONG.equals(dataType) || DataTypeMapping.INT.equals(dataType)) {
return getExampleIntegerValues(null, isDesc, size);

4
src/main/java/com/mesalab/common/utils/sqlparser/SelectItemHelper.java
View File

@@ -128,7 +128,6 @@ public class SelectItemHelper {
AliasColumn aliasColumn = new AliasColumn();
aliasColumn.setName(name);
aliasColumn.setFieldName(column.getColumnName());
aliasColumn.setArray(true);
this.aliasExpr = aliasColumn;
return null;
} else if ((expr1 instanceof ParenthesedExpressionList) && ((ParenthesedExpressionList) expr1).get(0) instanceof Column) {
@@ -136,7 +135,6 @@ public class SelectItemHelper {
AliasColumn aliasColumn = new AliasColumn();
aliasColumn.setName(name);
aliasColumn.setFieldName(column.getColumnName());
aliasColumn.setArray(true);
this.aliasExpr = aliasColumn;
return null;
}/* else if (expr1 instanceof RowConstructor
@@ -144,7 +142,6 @@ public class SelectItemHelper {
&& ((RowConstructor) expr1).getExprList().getExpressions().get(0) instanceof Column) {
AliasColumn aliasColumn = new AliasColumn();
aliasColumn.setName(name);
aliasColumn.setArray(true);
this.aliasExpr = aliasColumn;
return null;
}*/
@@ -168,7 +165,6 @@ public class SelectItemHelper {
public static class AliasObject {
private String name;
private int index;
private boolean isArray;
}
@Data

15
src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java
View File

@@ -47,6 +47,7 @@ public class ClickHouseDialect extends AbstractDataSourceDialect {
private static final Log log = LogFactory.get();
private static final Pattern pNullable = Pattern.compile("^Nullable\\((.*?)\\)", Pattern.CASE_INSENSITIVE);
private static final Pattern pDateTime = Pattern.compile("\\b(DateTime|DateTime64|Date)\\((.*?)\\)", Pattern.CASE_INSENSITIVE);
private static final Pattern pArray = Pattern.compile("^Array\\((.*?)\\)", Pattern.CASE_INSENSITIVE);
private static final Pattern pIn = Pattern.compile("(?<!\\bnot|global)\\s+in\\s*\\(\\s*SELECT\\b", Pattern.CASE_INSENSITIVE);
private static final Pattern pNotIn = Pattern.compile("(?<!\\bnot|global)\\s+not\\s+in\\s*\\(\\s*SELECT\\b", Pattern.CASE_INSENSITIVE);
private static final Pattern pUniq = Pattern.compile("count\\s*\\(\\s*distinct\\s*\\((.*?)\\)", Pattern.CASE_INSENSITIVE);
@@ -207,8 +208,7 @@ public class ClickHouseDialect extends AbstractDataSourceDialect {
return metaList;
}
metaList = JsonPath.read(meta, "$");
metaList.forEach(x ->
{
metaList.forEach(x -> {
String type = x.get("type");
Matcher matcher = pNullable.matcher(type);
if (matcher.find()) {
@@ -218,7 +218,15 @@ public class ClickHouseDialect extends AbstractDataSourceDialect {
if (matcher.find()) {
type = matcher.group(1);
}
x.put("type", metaMap.get(type) == null ? DataTypeMapping.STRING : metaMap.get(type));
matcher = pArray.matcher(type);
if (matcher.find()) {
String innerType = matcher.group(1);
String innerTypeMapped = metaMap.get(innerType) == null ? DataTypeMapping.STRING : metaMap.get(innerType);
x.put("type", DataTypeMapping.ARRAY + "(" + innerTypeMapped + ")");
} else {
x.put("type", metaMap.get(type) == null ? DataTypeMapping.STRING : metaMap.get(type));
}
});
return metaList;
}
@@ -823,7 +831,6 @@ public class ClickHouseDialect extends AbstractDataSourceDialect {
SelectItemHelper.AliasColumn aliasColumn = (SelectItemHelper.AliasColumn) aliasObject;
String fieldName = aliasColumn.getFieldName();
meta.put(MetaConst.META_FIELD_NAME, fieldName);
meta.put(MetaConst.IS_ARRAY, String.valueOf(aliasColumn.isArray()));
Map schemaInfo = databaseService.getSchemaInfo(MetadataType.FIELDS.getValue(), sqlQueryContext.getDbSelectStatement().getTableNames().get(0), false);
Object dateType = JSONPath.extract(JSON.toJSONString(schemaInfo), "$.fields[?(@.name == \"" + fieldName + "\")].doc.constraints.type");
if (dateType instanceof List && ((List<?>) dateType).size() > 0) {

9
src/main/java/com/mesalab/qgw/model/basic/udf/DATETIME_FLOOR_WITH_FILL.java
View File

@@ -96,6 +96,7 @@ public class DATETIME_FLOOR_WITH_FILL implements UDF {
if (isDryRun) {
return results;
}
List<Map<String, String>> metaList = (List<Map<String, String>>) baseResult.getMeta();
List<Map<String, Object>> targetResult = Lists.newArrayList();
List<String> groupLabels = Lists.newArrayList();
String timeGranLabel = null;
@@ -132,7 +133,7 @@ public class DATETIME_FLOOR_WITH_FILL implements UDF {
WhereTimeRange whereTimeRange = getWhereTimeRange(selectStatement);
Date benchmarkDate = getBenchmarkDate(resultsFirst, timeGranLabel);
targetResult = fillResultBasedOnTimeSeries(results, groupLabels, timeGranLabel);
targetResult = fillResultBasedOnTimeSeries(results, metaList, groupLabels, timeGranLabel);
List<Date> dateRangeList = null;
@@ -172,7 +173,7 @@ public class DATETIME_FLOOR_WITH_FILL implements UDF {
groupLabels, timeGranLabel);
if (StringUtil.isNotEmpty(dateRangeList)) {
for (String key : targetTimeSeriesMap.keySet()) {
targetTimeSeriesMap.put(key, ConvertUtil.completeTimeseries((TreeMap<String, Map<String, Object>>) targetTimeSeriesMap.get(key), dateRangeList, fill));
targetTimeSeriesMap.put(key, ConvertUtil.completeTimeseries((TreeMap<String, Map<String, Object>>) targetTimeSeriesMap.get(key), dateRangeList, metaList, fill));
}
}
targetResult = ConvertUtil.convertTimeSeriesToList(targetTimeSeriesMap, groupLabels, timeGranLabel);
@@ -264,7 +265,7 @@ public class DATETIME_FLOOR_WITH_FILL implements UDF {
}
}
private List<Map<String, Object>> fillResultBasedOnTimeSeries(List<Object> results, List<String> groupLabels, String timeGranLabel) {
private List<Map<String, Object>> fillResultBasedOnTimeSeries(List<Object> results, List<Map<String, String>> metaList, List<String> groupLabels, String timeGranLabel) {
Map<String, Map<String, Map<String, Object>>> targetTimeSeriesMap = ConvertUtil.convertResultListToTimeSeries(results,
groupLabels, timeGranLabel);
Set<Date> dateSet = Sets.newHashSet();
@@ -279,7 +280,7 @@ public class DATETIME_FLOOR_WITH_FILL implements UDF {
Collections.sort(dateRangeList);
if (StringUtil.isNotEmpty(dateRangeList)) {
for (String key : targetTimeSeriesMap.keySet()) {
targetTimeSeriesMap.put(key, ConvertUtil.completeTimeseries((TreeMap<String, Map<String, Object>>) targetTimeSeriesMap.get(key), dateRangeList, fill));
targetTimeSeriesMap.put(key, ConvertUtil.completeTimeseries((TreeMap<String, Map<String, Object>>) targetTimeSeriesMap.get(key), dateRangeList, metaList, fill));
}
}
return ConvertUtil.convertTimeSeriesToList(targetTimeSeriesMap, groupLabels, timeGranLabel);

9
src/main/java/com/mesalab/qgw/model/basic/udf/TIME_FLOOR_WITH_FILL.java
View File

@@ -97,6 +97,7 @@ public class TIME_FLOOR_WITH_FILL implements UDF {
if (isDryRun) {
return results;
}
List<Map<String, String>> metaList = (List<Map<String, String>>) baseResult.getMeta();
List<Map<String, Object>> targetResult = Lists.newArrayList();
List<String> groupLabels = Lists.newArrayList();
String timeGranLabel = null;
@@ -133,7 +134,7 @@ public class TIME_FLOOR_WITH_FILL implements UDF {
WhereTimeRange whereTimeRange = getWhereTimeRange(selectStatement);
Date benchmarkDate = getBenchmarkDate(resultsFirst, timeGranLabel);
targetResult = fillResultBasedOnTimeSeries(results, groupLabels, timeGranLabel);
targetResult = fillResultBasedOnTimeSeries(results, metaList, groupLabels, timeGranLabel);
List<Date> dateRangeList = null;
@@ -173,7 +174,7 @@ public class TIME_FLOOR_WITH_FILL implements UDF {
groupLabels, timeGranLabel);
if (StringUtil.isNotEmpty(dateRangeList)) {
for (String key : targetTimeSeriesMap.keySet()) {
targetTimeSeriesMap.put(key, ConvertUtil.completeTimeseries((TreeMap<String, Map<String, Object>>) targetTimeSeriesMap.get(key), dateRangeList, fill));
targetTimeSeriesMap.put(key, ConvertUtil.completeTimeseries((TreeMap<String, Map<String, Object>>) targetTimeSeriesMap.get(key), dateRangeList, metaList, fill));
}
}
targetResult = ConvertUtil.convertTimeSeriesToList(targetTimeSeriesMap, groupLabels, timeGranLabel);
@@ -287,7 +288,7 @@ public class TIME_FLOOR_WITH_FILL implements UDF {
return targetResult;
}
private List<Map<String, Object>> fillResultBasedOnTimeSeries(List<Object> results, List<String> groupLabels, String timeGranLabel) {
private List<Map<String, Object>> fillResultBasedOnTimeSeries(List<Object> results, List<Map<String, String>> metaList, List<String> groupLabels, String timeGranLabel) {
Map<String, Map<String, Map<String, Object>>> targetTimeSeriesMap = ConvertUtil.convertResultListToTimeSeries(results,
groupLabels, timeGranLabel);
Set<Date> dateSet = Sets.newHashSet();
@@ -302,7 +303,7 @@ public class TIME_FLOOR_WITH_FILL implements UDF {
Collections.sort(dateRangeList);
if (StringUtil.isNotEmpty(dateRangeList)) {
for (String key : targetTimeSeriesMap.keySet()) {
targetTimeSeriesMap.put(key, ConvertUtil.completeTimeseries((TreeMap<String, Map<String, Object>>) targetTimeSeriesMap.get(key), dateRangeList, fill));
targetTimeSeriesMap.put(key, ConvertUtil.completeTimeseries((TreeMap<String, Map<String, Object>>) targetTimeSeriesMap.get(key), dateRangeList, metaList, fill));
}
}
return ConvertUtil.convertTimeSeriesToList(targetTimeSeriesMap, groupLabels, timeGranLabel);