gfwleak/galaxy-tsg-olap-app-protocol-stat-traffic-merge
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[GAL-526] - 兼容字段命名重组前的数据统计

Browse Source
This commit is contained in:
qidaijie
2024-03-26 15:27:48 +08:00
parent 6047a09e5f
commit 249a65948a
7 changed files with 107 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/main/java/com/zdjizhi/utils/functions/process/ParsingData.java
View File

@@ -15,6 +15,18 @@ import org.apache.flink.util.Collector;
public class ParsingData extends ProcessFunction<String, Tuple3<Tags, Fields, Long>> {
private static final Log logger = LogFactory.get();
/**
* 适配TSG 24.02日志重组前数据结构，待过期后删除此处代码
*/
@Deprecated
private static final String LEGACY_PROTOCOL_KEY_NAME = "protocol_label";
/**
* 适配TSG 24.02日志重组前数据结构，待过期后删除此处代码
*/
@Deprecated
private static final String LEGACY_APP_KEY_NAME = "app_full_path";
private static final String dataTypeExpr = "[?(@.name = 'traffic_application_protocol_stat')]";
@Override
@@ -24,6 +36,10 @@ public class ParsingData extends ProcessFunction<String, Tuple3<Tags, Fields, Lo
Object isProtocolData = JSONPath.eval(value, dataTypeExpr);
if (isProtocolData != null) {
JSONObject originalLog = JSON.parseObject(value);
//适配TSG 24.02日志重组前数据结构，待过期后删除此处代码
supportingLegacyField(originalLog);
Fields fields = JSONObject.parseObject(originalLog.getString("fields"), Fields.class);
Tags tags = JSONObject.parseObject(originalLog.getString("tags"), Tags.class);
Long timestamp_ms = originalLog.getLong("timestamp_ms");
@@ -40,6 +56,27 @@ public class ParsingData extends ProcessFunction<String, Tuple3<Tags, Fields, Lo
}
}
/**
* 适配TSG 24.02日志重组前数据结构，在不改变原有逻辑情况下将
* protocol_label 修改为 decoded_path
* app_full_path 修改为 app
* <p>
* 待过期后删除此处代码
*
* @param originalLog 原始Metrics日志
*/
@Deprecated
private static void supportingLegacyField(JSONObject originalLog) {
JSONObject tags = originalLog.getJSONObject("tags");
if (tags.containsKey(LEGACY_PROTOCOL_KEY_NAME)) {
tags.put("decoded_path", tags.remove(LEGACY_PROTOCOL_KEY_NAME));
tags.put("app", tags.remove(LEGACY_APP_KEY_NAME));
originalLog.put("tags", originalLog.remove("tags"));
}
}
/**
* 避免计算重复的协议，去除Decoded Path（最后一个元素） 与 Application（第一个元素）重复的基础协议。
*

4
src/main/java/com/zdjizhi/utils/kafka/KafkaProducer.java
View File

@@ -15,7 +15,7 @@ import java.util.Properties;
public class KafkaProducer {
public static FlinkKafkaProducer<String> getKafkaProducer(Properties properties, String topic, boolean logFailuresOnly) {
setDefaultConfig(properties, "ack", "1");
setDefaultConfig(properties, "acks", "1");
setDefaultConfig(properties, "retries", 0);
setDefaultConfig(properties, "linger.ms", 10);
setDefaultConfig(properties, "request.timeout.ms", 30000);
@@ -29,7 +29,7 @@ public class KafkaProducer {
new SimpleStringSchema(),
properties, Optional.empty());
kafkaProducer.setLogFailuresOnly(logFailuresOnly);
kafkaProducer.setLogFailuresOnly(false);
return kafkaProducer;
}