From f80beb832470dc1bccb4d91221b6056993ec3b0d Mon Sep 17 00:00:00 2001 From: qidaijie Date: Sat, 17 Jul 2021 16:54:59 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8DJson=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E4=B8=BAMap=E6=97=B6=E5=87=BD=E6=95=B0=E5=A4=84=E7=90=86?= =?UTF-8?q?=E9=80=BB=E8=BE=91=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- properties/service_flow_config.properties | 2 +- .../zdjizhi/utils/general/TransFormMap.java | 47 +++++++------- .../utils/general/TransFormTypeMap.java | 62 ++++++++++--------- .../com/zdjizhi/utils/json/TypeUtils.java | 17 ----- 5 files changed, 58 insertions(+), 72 deletions(-) diff --git a/pom.xml b/pom.xml index 15bcbca..8d7e874 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.zdjizhi log-stream-completion-schema - v3.21.07.13-map + v3.21.07.17-typeMap jar log-stream-completion-schema diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index 06e5a54..05a3b9a 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -20,7 +20,7 @@ ip.library=D:\\K18-Phase2\\tsgSpace\\dat\\ #网关的schema位置 #schema.http=http://192.168.40.203:9999/metadata/schema/v1/fields/connection_record_log -schema.http=http://192.168.44.12:9999/metadata/schema/v1/fields/connection_record_log +schema.http=http://192.168.44.67:9999/metadata/schema/v1/fields/security_event_log #网关APP_ID 获取接口 app.id.http=http://192.168.44.12:9999/open-api/appDicList diff --git a/src/main/java/com/zdjizhi/utils/general/TransFormMap.java b/src/main/java/com/zdjizhi/utils/general/TransFormMap.java index c936eed..a182421 100644 --- a/src/main/java/com/zdjizhi/utils/general/TransFormMap.java +++ b/src/main/java/com/zdjizhi/utils/general/TransFormMap.java @@ -10,6 +10,7 @@ import com.zdjizhi.utils.json.JsonParseUtil; import com.zdjizhi.utils.json.JsonTypeUtils; import java.util.ArrayList; +import java.util.Arrays; import java.util.Map; @@ -40,7 +41,7 @@ public class TransFormMap { Map jsonMap = (Map) JsonMapper.fromJsonString(message, Map.class); for (String[] strings : jobList) { //用到的参数的值 - Object name = JsonParseUtil.getValue(jsonMap, strings[0]); + Object logValue = JsonParseUtil.getValue(jsonMap, strings[0]); //需要补全的字段的key String appendToKeyName = strings[1]; //需要补全的字段的值 @@ -49,7 +50,7 @@ public class TransFormMap { String function = strings[2]; //额外的参数的值 String param = strings[3]; - functionSet(function, jsonMap, appendToKeyName, appendTo, name, param); + functionSet(function, jsonMap, appendToKeyName, appendTo, logValue, param); } return JsonMapper.toJsonString(jsonMap); } catch (RuntimeException e) { @@ -66,10 +67,10 @@ public class TransFormMap { * @param jsonMap 原始日志解析map * @param appendToKeyName 需要补全的字段的key * @param appendTo 需要补全的字段的值 - * @param name 用到的参数的值 + * @param logValue 用到的参数的值 * @param param 额外的参数的值 */ - private static void functionSet(String function, Map jsonMap, String appendToKeyName, Object appendTo, Object name, String param) { + private static void functionSet(String function, Map jsonMap, String appendToKeyName, Object appendTo, Object logValue, String param) { switch (function) { case "current_timestamp": if (!(appendTo instanceof Long)) { @@ -80,28 +81,28 @@ public class TransFormMap { JsonParseUtil.setValue(jsonMap, appendToKeyName, SnowflakeId.generateId()); break; case "geo_ip_detail": - if (name != null && appendTo == null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpDetail(name.toString())); + if (logValue != null && appendTo == null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpDetail(logValue.toString())); } break; case "geo_asn": - if (name != null && appendTo == null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoAsn(name.toString())); + if (logValue != null && appendTo == null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoAsn(logValue.toString())); } break; case "geo_ip_country": - if (name != null && appendTo == null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpCountry(name.toString())); + if (logValue != null && appendTo == null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpCountry(logValue.toString())); } break; case "set_value": - if (name != null && param != null) { + if (param != null) { JsonParseUtil.setValue(jsonMap, appendToKeyName, param); } break; case "get_value": - if (name != null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, name); + if (logValue != null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, logValue); } break; case "if": @@ -110,28 +111,28 @@ public class TransFormMap { } break; case "sub_domain": - if (appendTo == null && name != null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getTopDomain(name.toString())); + if (appendTo == null && logValue != null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getTopDomain(logValue.toString())); } break; case "radius_match": - if (name != null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.radiusMatch(name.toString())); + if (logValue != null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.radiusMatch(logValue.toString())); } break; case "app_match": - if (name != null && appendTo == null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.appMatch(name.toString())); + if (logValue != null && appendTo == null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.appMatch(logValue.toString())); } break; case "decode_of_base64": - if (name != null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.decodeBase64(name.toString(), TransFunction.isJsonValue(jsonMap, param))); + if (logValue != null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.decodeBase64(logValue.toString(), TransFunction.isJsonValue(jsonMap, param))); } break; case "flattenSpec": - if (name != null && param != null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.flattenSpec(name.toString(), param)); + if (logValue != null && param != null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.flattenSpec(logValue.toString(), param)); } break; default: diff --git a/src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java b/src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java index d536945..4995461 100644 --- a/src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java +++ b/src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java @@ -10,6 +10,7 @@ import com.zdjizhi.utils.json.JsonParseUtil; import com.zdjizhi.utils.json.JsonTypeUtils; import java.util.ArrayList; +import java.util.Arrays; import java.util.Map; @@ -40,18 +41,19 @@ public class TransFormTypeMap { Map jsonMap = (Map) JsonMapper.fromJsonString(message, Map.class); for (String[] strings : jobList) { //用到的参数的值 - Object name = JsonParseUtil.getValue(jsonMap, strings[0]); + Object logValue = JsonParseUtil.getValue(jsonMap, strings[0]); //需要补全的字段的key String appendToKeyName = strings[1]; //需要补全的字段的值 - Object appendTo = JsonParseUtil.getValue(jsonMap, appendToKeyName); + Object appendToKeyValue = JsonParseUtil.getValue(jsonMap, appendToKeyName); //匹配操作函数的字段 String function = strings[2]; //额外的参数的值 String param = strings[3]; - functionSet(function, jsonMap, appendToKeyName, appendTo, name, param); + functionSet(function, jsonMap, appendToKeyName, appendToKeyValue, logValue, param); } return JsonMapper.toJsonString(JsonTypeUtils.typeTransform(jsonMap)); +// return JsonMapper.toJson(JsonTypeUtils.typeTransform(jsonMap)); } catch (RuntimeException e) { logger.error("解析补全日志信息过程异常,异常信息:" + e + "\n" + message); return ""; @@ -62,17 +64,17 @@ public class TransFormTypeMap { /** * 根据schema描述对应字段进行操作的 函数集合 * - * @param function 匹配操作函数的字段 - * @param jsonMap 原始日志解析map - * @param appendToKeyName 需要补全的字段的key - * @param appendTo 需要补全的字段的值 - * @param name 用到的参数的值 - * @param param 额外的参数的值 + * @param function 匹配操作函数的字段 + * @param jsonMap 原始日志解析map + * @param appendToKeyName 需要补全的字段的key + * @param appendToKeyValue 需要补全的字段的值 + * @param logValue 用到的参数的值 + * @param param 额外的参数的值 */ - private static void functionSet(String function, Map jsonMap, String appendToKeyName, Object appendTo, Object name, String param) { + private static void functionSet(String function, Map jsonMap, String appendToKeyName, Object appendToKeyValue, Object logValue, String param) { switch (function) { case "current_timestamp": - if (!(appendTo instanceof Long)) { + if (!(appendToKeyValue instanceof Long)) { JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getCurrentTime()); } break; @@ -80,28 +82,28 @@ public class TransFormTypeMap { JsonParseUtil.setValue(jsonMap, appendToKeyName, SnowflakeId.generateId()); break; case "geo_ip_detail": - if (name != null && appendTo == null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpDetail(name.toString())); + if (logValue != null && appendToKeyValue == null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpDetail(logValue.toString())); } break; case "geo_asn": - if (name != null && appendTo == null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoAsn(name.toString())); + if (logValue != null && appendToKeyValue == null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoAsn(logValue.toString())); } break; case "geo_ip_country": - if (name != null && appendTo == null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpCountry(name.toString())); + if (logValue != null && appendToKeyValue == null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpCountry(logValue.toString())); } break; case "set_value": - if (name != null && param != null) { + if (param != null) { JsonParseUtil.setValue(jsonMap, appendToKeyName, param); } break; case "get_value": - if (name != null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, name); + if (logValue != null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, logValue); } break; case "if": @@ -110,28 +112,28 @@ public class TransFormTypeMap { } break; case "sub_domain": - if (appendTo == null && name != null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getTopDomain(name.toString())); + if (appendToKeyValue == null && logValue != null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getTopDomain(logValue.toString())); } break; case "radius_match": - if (name != null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.radiusMatch(name.toString())); + if (logValue != null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.radiusMatch(logValue.toString())); } break; case "app_match": - if (name != null && appendTo == null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.appMatch(name.toString())); + if (logValue != null && appendToKeyValue == null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.appMatch(logValue.toString())); } break; case "decode_of_base64": - if (name != null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.decodeBase64(name.toString(), TransFunction.isJsonValue(jsonMap, param))); + if (logValue != null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.decodeBase64(logValue.toString(), TransFunction.isJsonValue(jsonMap, param))); } break; case "flattenSpec": - if (name != null && param != null) { - JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.flattenSpec(name.toString(), param)); + if (logValue != null && param != null) { + JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.flattenSpec(logValue.toString(), param)); } break; default: diff --git a/src/main/java/com/zdjizhi/utils/json/TypeUtils.java b/src/main/java/com/zdjizhi/utils/json/TypeUtils.java index 3d1e8b4..f4d54af 100644 --- a/src/main/java/com/zdjizhi/utils/json/TypeUtils.java +++ b/src/main/java/com/zdjizhi/utils/json/TypeUtils.java @@ -154,10 +154,6 @@ public class TypeUtils { return null; } - if(value instanceof BigDecimal){ - return longValue((BigDecimal) value); - } - // 此判断数值超范围不抛出异常，会截取成对应类型数值 if (value instanceof Number) { return ((Number) value).longValue(); @@ -185,17 +181,4 @@ public class TypeUtils { throw new FlowWriteException("can not cast to long, value : " + value); } - private static long longValue(BigDecimal decimal) { - if (decimal == null) { - return 0; - } - - int scale = decimal.scale(); - if (scale >= -100 && scale <= 100) { - return decimal.longValue(); - } - - return decimal.longValueExact(); - } - }