gfwleak/galaxy-tsg-olap-log-completion-schema
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新jackson工具类库到去除html4版本，修改部分配置项命名。

Browse Source
This commit is contained in:
qidaijie
2021-12-03 10:24:43 +03:00
parent 1b75e5b1c0
commit 150cf4c367
6 changed files with 22 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
pom.xml
View File

@@ -6,7 +6,7 @@
<groupId>com.zdjizhi</groupId>
<artifactId>log-completion-schema</artifactId>
<version>211109-Jackson</version>
<version>211116-jackson</version>
<name>log-completion-schema</name>
<url>http://www.example.com</url>
@@ -21,13 +21,13 @@
<repository>
<id>maven-ali</id>
<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
<!--<releases>-->
<!--<enabled>true</enabled>-->
<!--</releases>-->
<!--<snapshots>-->
<!--<enabled>true</enabled>-->
<!--<checksumPolicy>fail</checksumPolicy>-->
<!--</snapshots>-->
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
<checksumPolicy>fail</checksumPolicy>
</snapshots>
</repository>
</repositories>
@@ -115,7 +115,7 @@
<dependency>
<groupId>com.zdjizhi</groupId>
<artifactId>galaxy</artifactId>
<version>1.0.6</version>
<version>1.0.7</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>

2
properties/default_config.properties
View File

@@ -51,4 +51,4 @@ mail.default.charset=UTF-8
log.transform.type=2
#两个输出之间的最大时间(单位milliseconds)
buffer.timeout=100000
buffer.timeout=5000

6
properties/service_flow_config.properties
View File

@@ -1,10 +1,10 @@
#--------------------------------地址配置------------------------------#
#管理kafka地址
source.kafka.servers=10.231.12.4:9094
source.kafka.servers=10.224.11.14:9094
#管理输出kafka地址
sink.kafka.servers=10.224.11.14:9094,10.224.11.15:9094,10.224.11.16:9094,10.224.11.17:9094,10.224.11.18:9094,10.224.11.19:9094,10.224.11.20:9094,10.224.11.21:9094,10.224.11.22:9094,10.224.11.23:9094
sink.kafka.servers=10.224.11.14:9095,10.224.11.15:9095,10.224.11.16:9095,10.224.11.17:9095,10.224.11.18:9095,10.224.11.19:9095,10.224.11.20:9095,10.224.11.21:9095,10.224.11.22:9095,10.224.11.23:9095
#zookeeper 地址 用于配置log_id
zookeeper.servers=10.224.11.11:2181,10.224.11.12:2181,10.224.11.13:2181
@@ -25,7 +25,7 @@ app.id.http=http://10.224.11.244:9999/open-api/appDicList
#--------------------------------Kafka消费组信息------------------------------#
#kafka 接收数据topic
source.kafka.topic=SESSION-RECORD
source.kafka.topic=test
#补全数据 输出 topic
sink.kafka.topic=test-result

14
src/main/java/com/zdjizhi/utils/fast/TransFormFast.java
View File

@@ -58,17 +58,15 @@ public class TransFormFast {
functionSet(function, jsonMap, appendToKeyName, appendToKeyValue, logValue, param);
}
return JSONObject.toJSONString(jsonMap,
SerializerFeature.DisableCircularReferenceDetect
, SerializerFeature.WriteNullStringAsEmpty
, SerializerFeature.WriteNullNumberAsZero);
return FastJsonTypeUtils.typeTransform(jsonMap);
} else {
return null;
return "";
}
} catch (RuntimeException e) {
logger.error("解析补全日志信息过程异常,异常信息:" + e + "\n" + message);
return null;
logger.error("Logs TransForm Exception! Error message is:" + e);
e.printStackTrace();
return "";
}
}
@@ -77,7 +75,7 @@ public class TransFormFast {
* 根据schema描述对应字段进行操作的 函数集合
*
* @param function 匹配操作函数的字段
* @param jsonMap 原始日志解析map
* @param jsonMap 原始日志解析map
* @param appendToKeyName 需要补全的字段的key
* @param appendToKeyValue 需要补全的字段的值
* @param logValue 用到的参数的值

2
src/main/java/com/zdjizhi/utils/functions/TypeMapCompletedFunction.java
View File

@@ -1,5 +1,6 @@
package com.zdjizhi.utils.functions;
import com.zdjizhi.utils.fast.TransFormFast;
import com.zdjizhi.utils.general.TransFormTypeMap;
import org.apache.flink.api.common.functions.MapFunction;
@@ -17,5 +18,6 @@ public class TypeMapCompletedFunction implements MapFunction<String, String> {
public String map(String logs) {
return TransFormTypeMap.dealCommonMessage(logs);
// return TransFormFast.dealCommonMessage(logs);
}
}

7
src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java
View File

@@ -57,13 +57,8 @@ public class TransFormTypeMap {
String param = strings[3];
functionSet(function, jsonMap, appendToKeyName, appendToKeyValue, logValue, param);
}
// return JsonMapper.toJsonString(jsonMap);
return JsonMapper.toJsonString(jsonMap);
//fastjson test
return JSONObject.toJSONString(jsonMap,
SerializerFeature.DisableCircularReferenceDetect
,SerializerFeature.WriteNullStringAsEmpty
,SerializerFeature.WriteNullNumberAsZero);
} else {
return null;
}