gfwleak/galaxy-tsg-olap-storm-log-stream-completion
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复Json解析为Map时函数处理逻辑。

Browse Source
This commit is contained in:
qidaijie
2021-07-17 16:54:59 +08:00
parent 2ccc0d77a3
commit f80beb8324
5 changed files with 58 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@@ -4,7 +4,7 @@
<groupId>com.zdjizhi</groupId> <groupId>com.zdjizhi</groupId>
<artifactId>log-stream-completion-schema</artifactId> <artifactId>log-stream-completion-schema</artifactId>
<version>v3.21.07.13-map</version> <version>v3.21.07.17-typeMap</version>
<packaging>jar</packaging> <packaging>jar</packaging>
<name>log-stream-completion-schema</name> <name>log-stream-completion-schema</name>

2
properties/service_flow_config.properties
View File

@@ -20,7 +20,7 @@ ip.library=D:\\K18-Phase2\\tsgSpace\\dat\\
#网关的schema位置 #网关的schema位置
#schema.http=http://192.168.40.203:9999/metadata/schema/v1/fields/connection_record_log #schema.http=http://192.168.40.203:9999/metadata/schema/v1/fields/connection_record_log
schema.http=http://192.168.44.12:9999/metadata/schema/v1/fields/connection_record_log schema.http=http://192.168.44.67:9999/metadata/schema/v1/fields/security_event_log
#网关APP_ID 获取接口 #网关APP_ID 获取接口
app.id.http=http://192.168.44.12:9999/open-api/appDicList app.id.http=http://192.168.44.12:9999/open-api/appDicList

47
src/main/java/com/zdjizhi/utils/general/TransFormMap.java
View File

@@ -10,6 +10,7 @@ import com.zdjizhi.utils.json.JsonParseUtil;
import com.zdjizhi.utils.json.JsonTypeUtils; import com.zdjizhi.utils.json.JsonTypeUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map; import java.util.Map;
@@ -40,7 +41,7 @@ public class TransFormMap {
Map<String, Object> jsonMap = (Map<String, Object>) JsonMapper.fromJsonString(message, Map.class); Map<String, Object> jsonMap = (Map<String, Object>) JsonMapper.fromJsonString(message, Map.class);
for (String[] strings : jobList) { for (String[] strings : jobList) {
//用到的参数的值 //用到的参数的值
Object name = JsonParseUtil.getValue(jsonMap, strings[0]); Object logValue = JsonParseUtil.getValue(jsonMap, strings[0]);
//需要补全的字段的key //需要补全的字段的key
String appendToKeyName = strings[1]; String appendToKeyName = strings[1];
//需要补全的字段的值 //需要补全的字段的值
@@ -49,7 +50,7 @@ public class TransFormMap {
String function = strings[2]; String function = strings[2];
//额外的参数的值 //额外的参数的值
String param = strings[3]; String param = strings[3];
functionSet(function, jsonMap, appendToKeyName, appendTo, name, param); functionSet(function, jsonMap, appendToKeyName, appendTo, logValue, param);
} }
return JsonMapper.toJsonString(jsonMap); return JsonMapper.toJsonString(jsonMap);
} catch (RuntimeException e) { } catch (RuntimeException e) {
@@ -66,10 +67,10 @@ public class TransFormMap {
* @param jsonMap 原始日志解析map * @param jsonMap 原始日志解析map
* @param appendToKeyName 需要补全的字段的key * @param appendToKeyName 需要补全的字段的key
* @param appendTo 需要补全的字段的值 * @param appendTo 需要补全的字段的值
* @param name 用到的参数的值 * @param logValue 用到的参数的值
* @param param 额外的参数的值 * @param param 额外的参数的值
*/ */
private static void functionSet(String function, Map<String, Object> jsonMap, String appendToKeyName, Object appendTo, Object name, String param) { private static void functionSet(String function, Map<String, Object> jsonMap, String appendToKeyName, Object appendTo, Object logValue, String param) {
switch (function) { switch (function) {
case "current_timestamp": case "current_timestamp":
if (!(appendTo instanceof Long)) { if (!(appendTo instanceof Long)) {
@@ -80,28 +81,28 @@ public class TransFormMap {
JsonParseUtil.setValue(jsonMap, appendToKeyName, SnowflakeId.generateId()); JsonParseUtil.setValue(jsonMap, appendToKeyName, SnowflakeId.generateId());
break; break;
case "geo_ip_detail": case "geo_ip_detail":
if (name != null && appendTo == null) { if (logValue != null && appendTo == null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpDetail(name.toString())); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpDetail(logValue.toString()));
} }
break; break;
case "geo_asn": case "geo_asn":
if (name != null && appendTo == null) { if (logValue != null && appendTo == null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoAsn(name.toString())); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoAsn(logValue.toString()));
} }
break; break;
case "geo_ip_country": case "geo_ip_country":
if (name != null && appendTo == null) { if (logValue != null && appendTo == null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpCountry(name.toString())); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpCountry(logValue.toString()));
} }
break; break;
case "set_value": case "set_value":
if (name != null && param != null) { if (param != null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, param); JsonParseUtil.setValue(jsonMap, appendToKeyName, param);
} }
break; break;
case "get_value": case "get_value":
if (name != null) { if (logValue != null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, name); JsonParseUtil.setValue(jsonMap, appendToKeyName, logValue);
} }
break; break;
case "if": case "if":
@@ -110,28 +111,28 @@ public class TransFormMap {
} }
break; break;
case "sub_domain": case "sub_domain":
if (appendTo == null && name != null) { if (appendTo == null && logValue != null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getTopDomain(name.toString())); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getTopDomain(logValue.toString()));
} }
break; break;
case "radius_match": case "radius_match":
if (name != null) { if (logValue != null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.radiusMatch(name.toString())); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.radiusMatch(logValue.toString()));
} }
break; break;
case "app_match": case "app_match":
if (name != null && appendTo == null) { if (logValue != null && appendTo == null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.appMatch(name.toString())); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.appMatch(logValue.toString()));
} }
break; break;
case "decode_of_base64": case "decode_of_base64":
if (name != null) { if (logValue != null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.decodeBase64(name.toString(), TransFunction.isJsonValue(jsonMap, param))); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.decodeBase64(logValue.toString(), TransFunction.isJsonValue(jsonMap, param)));
} }
break; break;
case "flattenSpec": case "flattenSpec":
if (name != null && param != null) { if (logValue != null && param != null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.flattenSpec(name.toString(), param)); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.flattenSpec(logValue.toString(), param));
} }
break; break;
default: default:

62
src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java
View File

@@ -10,6 +10,7 @@ import com.zdjizhi.utils.json.JsonParseUtil;
import com.zdjizhi.utils.json.JsonTypeUtils; import com.zdjizhi.utils.json.JsonTypeUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map; import java.util.Map;
@@ -40,18 +41,19 @@ public class TransFormTypeMap {
Map<String, Object> jsonMap = (Map<String, Object>) JsonMapper.fromJsonString(message, Map.class); Map<String, Object> jsonMap = (Map<String, Object>) JsonMapper.fromJsonString(message, Map.class);
for (String[] strings : jobList) { for (String[] strings : jobList) {
//用到的参数的值 //用到的参数的值
Object name = JsonParseUtil.getValue(jsonMap, strings[0]); Object logValue = JsonParseUtil.getValue(jsonMap, strings[0]);
//需要补全的字段的key //需要补全的字段的key
String appendToKeyName = strings[1]; String appendToKeyName = strings[1];
//需要补全的字段的值 //需要补全的字段的值
Object appendTo = JsonParseUtil.getValue(jsonMap, appendToKeyName); Object appendToKeyValue = JsonParseUtil.getValue(jsonMap, appendToKeyName);
//匹配操作函数的字段 //匹配操作函数的字段
String function = strings[2]; String function = strings[2];
//额外的参数的值 //额外的参数的值
String param = strings[3]; String param = strings[3];
functionSet(function, jsonMap, appendToKeyName, appendTo, name, param); functionSet(function, jsonMap, appendToKeyName, appendToKeyValue, logValue, param);
} }
return JsonMapper.toJsonString(JsonTypeUtils.typeTransform(jsonMap)); return JsonMapper.toJsonString(JsonTypeUtils.typeTransform(jsonMap));
// return JsonMapper.toJson(JsonTypeUtils.typeTransform(jsonMap));
} catch (RuntimeException e) { } catch (RuntimeException e) {
logger.error("解析补全日志信息过程异常,异常信息:" + e + "\n" + message); logger.error("解析补全日志信息过程异常,异常信息:" + e + "\n" + message);
return ""; return "";
@@ -62,17 +64,17 @@ public class TransFormTypeMap {
/** /**
* 根据schema描述对应字段进行操作的 函数集合 * 根据schema描述对应字段进行操作的 函数集合
* *
* @param function 匹配操作函数的字段 * @param function 匹配操作函数的字段
* @param jsonMap 原始日志解析map * @param jsonMap 原始日志解析map
* @param appendToKeyName 需要补全的字段的key * @param appendToKeyName 需要补全的字段的key
* @param appendTo 需要补全的字段的值 * @param appendToKeyValue 需要补全的字段的值
* @param name 用到的参数的值 * @param logValue 用到的参数的值
* @param param 额外的参数的值 * @param param 额外的参数的值
*/ */
private static void functionSet(String function, Map<String, Object> jsonMap, String appendToKeyName, Object appendTo, Object name, String param) { private static void functionSet(String function, Map<String, Object> jsonMap, String appendToKeyName, Object appendToKeyValue, Object logValue, String param) {
switch (function) { switch (function) {
case "current_timestamp": case "current_timestamp":
if (!(appendTo instanceof Long)) { if (!(appendToKeyValue instanceof Long)) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getCurrentTime()); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getCurrentTime());
} }
break; break;
@@ -80,28 +82,28 @@ public class TransFormTypeMap {
JsonParseUtil.setValue(jsonMap, appendToKeyName, SnowflakeId.generateId()); JsonParseUtil.setValue(jsonMap, appendToKeyName, SnowflakeId.generateId());
break; break;
case "geo_ip_detail": case "geo_ip_detail":
if (name != null && appendTo == null) { if (logValue != null && appendToKeyValue == null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpDetail(name.toString())); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpDetail(logValue.toString()));
} }
break; break;
case "geo_asn": case "geo_asn":
if (name != null && appendTo == null) { if (logValue != null && appendToKeyValue == null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoAsn(name.toString())); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoAsn(logValue.toString()));
} }
break; break;
case "geo_ip_country": case "geo_ip_country":
if (name != null && appendTo == null) { if (logValue != null && appendToKeyValue == null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpCountry(name.toString())); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpCountry(logValue.toString()));
} }
break; break;
case "set_value": case "set_value":
if (name != null && param != null) { if (param != null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, param); JsonParseUtil.setValue(jsonMap, appendToKeyName, param);
} }
break; break;
case "get_value": case "get_value":
if (name != null) { if (logValue != null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, name); JsonParseUtil.setValue(jsonMap, appendToKeyName, logValue);
} }
break; break;
case "if": case "if":
@@ -110,28 +112,28 @@ public class TransFormTypeMap {
} }
break; break;
case "sub_domain": case "sub_domain":
if (appendTo == null && name != null) { if (appendToKeyValue == null && logValue != null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getTopDomain(name.toString())); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getTopDomain(logValue.toString()));
} }
break; break;
case "radius_match": case "radius_match":
if (name != null) { if (logValue != null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.radiusMatch(name.toString())); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.radiusMatch(logValue.toString()));
} }
break; break;
case "app_match": case "app_match":
if (name != null && appendTo == null) { if (logValue != null && appendToKeyValue == null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.appMatch(name.toString())); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.appMatch(logValue.toString()));
} }
break; break;
case "decode_of_base64": case "decode_of_base64":
if (name != null) { if (logValue != null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.decodeBase64(name.toString(), TransFunction.isJsonValue(jsonMap, param))); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.decodeBase64(logValue.toString(), TransFunction.isJsonValue(jsonMap, param)));
} }
break; break;
case "flattenSpec": case "flattenSpec":
if (name != null && param != null) { if (logValue != null && param != null) {
JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.flattenSpec(name.toString(), param)); JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.flattenSpec(logValue.toString(), param));
} }
break; break;
default: default:

17
src/main/java/com/zdjizhi/utils/json/TypeUtils.java
View File

@@ -154,10 +154,6 @@ public class TypeUtils {
return null; return null;
} }
if(value instanceof BigDecimal){
return longValue((BigDecimal) value);
}
// 此判断数值超范围不抛出异常，会截取成对应类型数值 // 此判断数值超范围不抛出异常，会截取成对应类型数值
if (value instanceof Number) { if (value instanceof Number) {
return ((Number) value).longValue(); return ((Number) value).longValue();
@@ -185,17 +181,4 @@ public class TypeUtils {
throw new FlowWriteException("can not cast to long, value : " + value); throw new FlowWriteException("can not cast to long, value : " + value);
} }
private static long longValue(BigDecimal decimal) {
if (decimal == null) {
return 0;
}
int scale = decimal.scale();
if (scale >= -100 && scale <= 100) {
return decimal.longValue();
}
return decimal.longValueExact();
}
} }