From 150cf4c3673b8b3f8db9f521e58c90592016a309 Mon Sep 17 00:00:00 2001 From: qidaijie Date: Fri, 3 Dec 2021 10:24:43 +0300 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0jackson=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E7=B1=BB=E5=BA=93=E5=88=B0=E5=8E=BB=E9=99=A4html4=E7=89=88?= =?UTF-8?q?=E6=9C=AC=EF=BC=8C=E4=BF=AE=E6=94=B9=E9=83=A8=E5=88=86=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E9=A1=B9=E5=91=BD=E5=90=8D=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 18 +++++++++--------- properties/default_config.properties | 2 +- properties/service_flow_config.properties | 6 +++--- .../com/zdjizhi/utils/fast/TransFormFast.java | 14 ++++++-------- .../functions/TypeMapCompletedFunction.java | 2 ++ .../utils/general/TransFormTypeMap.java | 7 +------ 6 files changed, 22 insertions(+), 27 deletions(-) diff --git a/pom.xml b/pom.xml index feb29c6..f2eb9c1 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.zdjizhi log-completion-schema - 211109-Jackson + 211116-jackson log-completion-schema http://www.example.com @@ -21,13 +21,13 @@ maven-ali http://maven.aliyun.com/nexus/content/groups/public/ - - - - - - - + + true + + + true + fail + @@ -115,7 +115,7 @@ com.zdjizhi galaxy - 1.0.6 + 1.0.7 slf4j-log4j12 diff --git a/properties/default_config.properties b/properties/default_config.properties index 01bb5ca..99d8c79 100644 --- a/properties/default_config.properties +++ b/properties/default_config.properties @@ -51,4 +51,4 @@ mail.default.charset=UTF-8 log.transform.type=2 #两个输出之间的最大时间(单位milliseconds) -buffer.timeout=100000 \ No newline at end of file +buffer.timeout=5000 \ No newline at end of file diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties index 1a38ca4..23e52db 100644 --- a/properties/service_flow_config.properties +++ b/properties/service_flow_config.properties @@ -1,10 +1,10 @@ #--------------------------------地址配置------------------------------# #管理kafka地址 -source.kafka.servers=10.231.12.4:9094 +source.kafka.servers=10.224.11.14:9094 #管理输出kafka地址 -sink.kafka.servers=10.224.11.14:9094,10.224.11.15:9094,10.224.11.16:9094,10.224.11.17:9094,10.224.11.18:9094,10.224.11.19:9094,10.224.11.20:9094,10.224.11.21:9094,10.224.11.22:9094,10.224.11.23:9094 +sink.kafka.servers=10.224.11.14:9095,10.224.11.15:9095,10.224.11.16:9095,10.224.11.17:9095,10.224.11.18:9095,10.224.11.19:9095,10.224.11.20:9095,10.224.11.21:9095,10.224.11.22:9095,10.224.11.23:9095 #zookeeper 地址 用于配置log_id zookeeper.servers=10.224.11.11:2181,10.224.11.12:2181,10.224.11.13:2181 @@ -25,7 +25,7 @@ app.id.http=http://10.224.11.244:9999/open-api/appDicList #--------------------------------Kafka消费组信息------------------------------# #kafka 接收数据topic -source.kafka.topic=SESSION-RECORD +source.kafka.topic=test #补全数据 输出 topic sink.kafka.topic=test-result diff --git a/src/main/java/com/zdjizhi/utils/fast/TransFormFast.java b/src/main/java/com/zdjizhi/utils/fast/TransFormFast.java index c6ff46f..3beb5ce 100644 --- a/src/main/java/com/zdjizhi/utils/fast/TransFormFast.java +++ b/src/main/java/com/zdjizhi/utils/fast/TransFormFast.java @@ -58,17 +58,15 @@ public class TransFormFast { functionSet(function, jsonMap, appendToKeyName, appendToKeyValue, logValue, param); } - return JSONObject.toJSONString(jsonMap, - SerializerFeature.DisableCircularReferenceDetect - , SerializerFeature.WriteNullStringAsEmpty - , SerializerFeature.WriteNullNumberAsZero); + return FastJsonTypeUtils.typeTransform(jsonMap); } else { - return null; + return ""; } } catch (RuntimeException e) { - logger.error("解析补全日志信息过程异常,异常信息:" + e + "\n" + message); - return null; + logger.error("Logs TransForm Exception! Error message is:" + e); + e.printStackTrace(); + return ""; } } @@ -77,7 +75,7 @@ public class TransFormFast { * 根据schema描述对应字段进行操作的 函数集合 * * @param function 匹配操作函数的字段 - * @param jsonMap 原始日志解析map + * @param jsonMap 原始日志解析map * @param appendToKeyName 需要补全的字段的key * @param appendToKeyValue 需要补全的字段的值 * @param logValue 用到的参数的值 diff --git a/src/main/java/com/zdjizhi/utils/functions/TypeMapCompletedFunction.java b/src/main/java/com/zdjizhi/utils/functions/TypeMapCompletedFunction.java index 99c92e8..46c8709 100644 --- a/src/main/java/com/zdjizhi/utils/functions/TypeMapCompletedFunction.java +++ b/src/main/java/com/zdjizhi/utils/functions/TypeMapCompletedFunction.java @@ -1,5 +1,6 @@ package com.zdjizhi.utils.functions; +import com.zdjizhi.utils.fast.TransFormFast; import com.zdjizhi.utils.general.TransFormTypeMap; import org.apache.flink.api.common.functions.MapFunction; @@ -17,5 +18,6 @@ public class TypeMapCompletedFunction implements MapFunction { public String map(String logs) { return TransFormTypeMap.dealCommonMessage(logs); +// return TransFormFast.dealCommonMessage(logs); } } diff --git a/src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java b/src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java index 5f2100b..765e23e 100644 --- a/src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java +++ b/src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java @@ -57,13 +57,8 @@ public class TransFormTypeMap { String param = strings[3]; functionSet(function, jsonMap, appendToKeyName, appendToKeyValue, logValue, param); } -// return JsonMapper.toJsonString(jsonMap); + return JsonMapper.toJsonString(jsonMap); - //fastjson test - return JSONObject.toJSONString(jsonMap, - SerializerFeature.DisableCircularReferenceDetect - ,SerializerFeature.WriteNullStringAsEmpty - ,SerializerFeature.WriteNullNumberAsZero); } else { return null; }