gfwleak/galaxy-tsg-olap-log-completion-schema
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新jackson工具类库到去除html4版本，修改部分配置项命名。

Browse Source
This commit is contained in:
qidaijie
2021-12-03 10:24:43 +03:00
parent 1b75e5b1c0
commit 150cf4c367
6 changed files with 22 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
pom.xml
View File

@@ -6,7 +6,7 @@
<groupId>com.zdjizhi</groupId> <groupId>com.zdjizhi</groupId>
<artifactId>log-completion-schema</artifactId> <artifactId>log-completion-schema</artifactId>
<version>211109-Jackson</version> <version>211116-jackson</version>
<name>log-completion-schema</name> <name>log-completion-schema</name>
<url>http://www.example.com</url> <url>http://www.example.com</url>
@@ -21,13 +21,13 @@
<repository> <repository>
<id>maven-ali</id> <id>maven-ali</id>
<url>http://maven.aliyun.com/nexus/content/groups/public/</url> <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
<!--<releases>--> <releases>
<!--<enabled>true</enabled>--> <enabled>true</enabled>
<!--</releases>--> </releases>
<!--<snapshots>--> <snapshots>
<!--<enabled>true</enabled>--> <enabled>true</enabled>
<!--<checksumPolicy>fail</checksumPolicy>--> <checksumPolicy>fail</checksumPolicy>
<!--</snapshots>--> </snapshots>
</repository> </repository>
</repositories> </repositories>
@@ -115,7 +115,7 @@
<dependency> <dependency>
<groupId>com.zdjizhi</groupId> <groupId>com.zdjizhi</groupId>
<artifactId>galaxy</artifactId> <artifactId>galaxy</artifactId>
<version>1.0.6</version> <version>1.0.7</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<artifactId>slf4j-log4j12</artifactId> <artifactId>slf4j-log4j12</artifactId>

2
properties/default_config.properties
View File

@@ -51,4 +51,4 @@ mail.default.charset=UTF-8
log.transform.type=2 log.transform.type=2
#两个输出之间的最大时间(单位milliseconds) #两个输出之间的最大时间(单位milliseconds)
buffer.timeout=100000 buffer.timeout=5000

6
properties/service_flow_config.properties
View File

@@ -1,10 +1,10 @@
#--------------------------------地址配置------------------------------# #--------------------------------地址配置------------------------------#
#管理kafka地址 #管理kafka地址
source.kafka.servers=10.231.12.4:9094 source.kafka.servers=10.224.11.14:9094
#管理输出kafka地址 #管理输出kafka地址
sink.kafka.servers=10.224.11.14:9094,10.224.11.15:9094,10.224.11.16:9094,10.224.11.17:9094,10.224.11.18:9094,10.224.11.19:9094,10.224.11.20:9094,10.224.11.21:9094,10.224.11.22:9094,10.224.11.23:9094 sink.kafka.servers=10.224.11.14:9095,10.224.11.15:9095,10.224.11.16:9095,10.224.11.17:9095,10.224.11.18:9095,10.224.11.19:9095,10.224.11.20:9095,10.224.11.21:9095,10.224.11.22:9095,10.224.11.23:9095
#zookeeper 地址 用于配置log_id #zookeeper 地址 用于配置log_id
zookeeper.servers=10.224.11.11:2181,10.224.11.12:2181,10.224.11.13:2181 zookeeper.servers=10.224.11.11:2181,10.224.11.12:2181,10.224.11.13:2181
@@ -25,7 +25,7 @@ app.id.http=http://10.224.11.244:9999/open-api/appDicList
#--------------------------------Kafka消费组信息------------------------------# #--------------------------------Kafka消费组信息------------------------------#
#kafka 接收数据topic #kafka 接收数据topic
source.kafka.topic=SESSION-RECORD source.kafka.topic=test
#补全数据 输出 topic #补全数据 输出 topic
sink.kafka.topic=test-result sink.kafka.topic=test-result

12
src/main/java/com/zdjizhi/utils/fast/TransFormFast.java
View File

@@ -58,17 +58,15 @@ public class TransFormFast {
functionSet(function, jsonMap, appendToKeyName, appendToKeyValue, logValue, param); functionSet(function, jsonMap, appendToKeyName, appendToKeyValue, logValue, param);
} }
return JSONObject.toJSONString(jsonMap, return FastJsonTypeUtils.typeTransform(jsonMap);
SerializerFeature.DisableCircularReferenceDetect
, SerializerFeature.WriteNullStringAsEmpty
, SerializerFeature.WriteNullNumberAsZero);
} else { } else {
return null; return "";
} }
} catch (RuntimeException e) { } catch (RuntimeException e) {
logger.error("解析补全日志信息过程异常,异常信息:" + e + "\n" + message); logger.error("Logs TransForm Exception! Error message is:" + e);
return null; e.printStackTrace();
return "";
} }
} }

2
src/main/java/com/zdjizhi/utils/functions/TypeMapCompletedFunction.java
View File

@@ -1,5 +1,6 @@
package com.zdjizhi.utils.functions; package com.zdjizhi.utils.functions;
import com.zdjizhi.utils.fast.TransFormFast;
import com.zdjizhi.utils.general.TransFormTypeMap; import com.zdjizhi.utils.general.TransFormTypeMap;
import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.common.functions.MapFunction;
@@ -17,5 +18,6 @@ public class TypeMapCompletedFunction implements MapFunction<String, String> {
public String map(String logs) { public String map(String logs) {
return TransFormTypeMap.dealCommonMessage(logs); return TransFormTypeMap.dealCommonMessage(logs);
// return TransFormFast.dealCommonMessage(logs);
} }
} }

7
src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java
View File

@@ -57,13 +57,8 @@ public class TransFormTypeMap {
String param = strings[3]; String param = strings[3];
functionSet(function, jsonMap, appendToKeyName, appendToKeyValue, logValue, param); functionSet(function, jsonMap, appendToKeyName, appendToKeyValue, logValue, param);
} }
// return JsonMapper.toJsonString(jsonMap); return JsonMapper.toJsonString(jsonMap);
//fastjson test
return JSONObject.toJSONString(jsonMap,
SerializerFeature.DisableCircularReferenceDetect
,SerializerFeature.WriteNullStringAsEmpty
,SerializerFeature.WriteNullNumberAsZero);
} else { } else {
return null; return null;
} }