getJobList() {
+ return jobList;
+ }
+
+ /**
+ * 在配置变动时，清空缓存重新获取
+ */
+ private static void clearCache() {
+ jobList.clear();
+ schemaFieldsTypeMap.clear();
+ dropList.clear();
+ defaultFieldsMap.clear();
+ }
+
+}
\ No newline at end of file
diff --git a/src/main/java/com/zdjizhi/utils/ordinary/MD5Utils.java b/src/main/java/com/zdjizhi/tools/ordinary/MD5Utils.java
similarity index 97%
rename from src/main/java/com/zdjizhi/utils/ordinary/MD5Utils.java
rename to src/main/java/com/zdjizhi/tools/ordinary/MD5Utils.java
index aa55951..f918286 100644
--- a/src/main/java/com/zdjizhi/utils/ordinary/MD5Utils.java
+++ b/src/main/java/com/zdjizhi/tools/ordinary/MD5Utils.java
@@ -1,4 +1,4 @@
-package com.zdjizhi.utils.ordinary;
+package com.zdjizhi.tools.ordinary;
import org.apache.log4j.Logger;
diff --git a/src/main/java/com/zdjizhi/tools/transform/TransForm.java b/src/main/java/com/zdjizhi/tools/transform/TransForm.java
new file mode 100644
index 0000000..c76ca36
--- /dev/null
+++ b/src/main/java/com/zdjizhi/tools/transform/TransForm.java
@@ -0,0 +1,115 @@
+package com.zdjizhi.tools.transform;
+
+import com.alibaba.fastjson2.JSONObject;
+import com.zdjizhi.tools.json.MetaUtil;
+import com.zdjizhi.tools.transform.impl.TransformFunctionImpl;
+
+
+/**
+ * 描述:转换或补全工具类
+ *
+ * @author qidaijie
+ */
+public class TransForm {
+ private static TransformFunctionImpl transformFunction = new TransformFunctionImpl();
+
+ public static void transformLog(JSONObject jsonObject) {
+ for (String[] strings : MetaUtil.getJobList()) {
+ //该日志字段的值
+ Object logValue = jsonObject.get(strings[0]);
+ //结果值映射到的日志字段key
+ String appendToKey = strings[1];
+ //匹配操作函数的字段
+ String function = strings[2];
+ //额外的参数的值
+ String param = strings[3];
+
+ //结果值映射到的日志字段原始value
+ Object appendToValue = jsonObject.get(appendToKey);
+
+ functionSet(function, jsonObject, appendToKey, appendToValue, logValue, param);
+ }
+ }
+
+
+ /**
+ * 根据schema描述对应字段进行操作的 函数集合
+ *
+ * @param function 匹配操作函数的字段
+ * @param jsonObject 原始日志解析map
+ * @param appendToKey 需要补全的字段的key
+ * @param appendToValue 需要补全的字段的值
+ * @param logValue 用到的参数的值
+ * @param param 额外的参数的值
+ */
+ private static void functionSet(String function, JSONObject jsonObject, String appendToKey, Object appendToValue, Object logValue, String param) {
+
+ switch (function) {
+ case "current_timestamp":
+ if (!(appendToValue instanceof Long)) {
+ jsonObject.put(appendToKey, transformFunction.getCurrentTime());
+ }
+ break;
+ case "snowflake_id":
+ jsonObject.put(appendToKey, transformFunction.getSnowflakeId());
+ break;
+ case "geo_ip_detail":
+ if (logValue != null && appendToValue == null) {
+ jsonObject.put(appendToKey, transformFunction.getGeoIpDetail(logValue.toString()));
+ }
+ break;
+ case "geo_asn":
+ if (logValue != null && appendToValue == null) {
+ jsonObject.put(appendToKey, transformFunction.getGeoAsn(logValue.toString()));
+ }
+ break;
+ case "geo_ip_country":
+ if (logValue != null && appendToValue == null) {
+ jsonObject.put(appendToKey, transformFunction.getGeoIpCountry(logValue.toString()));
+ }
+ break;
+ case "flattenSpec":
+ if (logValue != null && param != null) {
+ jsonObject.put(appendToKey, transformFunction.flattenSpec(logValue.toString(), param));
+ }
+ break;
+ case "if":
+ if (param != null) {
+ jsonObject.put(appendToKey, transformFunction.condition(jsonObject, param));
+ }
+ break;
+ case "decode_of_base64":
+ if (logValue != null) {
+ jsonObject.put( appendToKey, transformFunction.decodeBase64(jsonObject, logValue.toString(), param));
+ }
+ break;
+ case "sub_domain":
+ if (appendToValue == null && logValue != null) {
+ jsonObject.put( appendToKey, transformFunction.getTopDomain(logValue.toString()));
+ }
+ break;
+ case "radius_match":
+ if (logValue != null) {
+ jsonObject.put( appendToKey, transformFunction.radiusMatch(jsonObject, logValue.toString()));
+ }
+ break;
+ case "gtpc_match":
+ if (logValue != null) {
+ transformFunction.gtpcMatch(jsonObject, logValue.toString(), appendToKey, param);
+ }
+ break;
+ case "set_value":
+ if (param != null) {
+ transformFunction.setValue(jsonObject, appendToKey, param);
+ }
+ break;
+ case "get_value":
+ if (logValue != null) {
+ transformFunction.getValue(jsonObject, appendToKey, logValue);
+ }
+ break;
+ default:
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/src/main/java/com/zdjizhi/tools/transform/TransformFunction.java b/src/main/java/com/zdjizhi/tools/transform/TransformFunction.java
new file mode 100644
index 0000000..53127d7
--- /dev/null
+++ b/src/main/java/com/zdjizhi/tools/transform/TransformFunction.java
@@ -0,0 +1,40 @@
+package com.zdjizhi.tools.transform;
+
+import com.alibaba.fastjson2.JSONObject;
+import com.geedgenetworks.utils.IpLookupV2;
+
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi.tools.general
+ * @Description:
+ * @date 2023/5/2010:11
+ */
+public interface TransformFunction {
+
+ long getCurrentTime();
+
+ long getSnowflakeId();
+
+ String getGeoIpDetail(String ip);
+
+ String getGeoIpCountry(String ip);
+
+ String getGeoAsn(String ip);
+
+ String radiusMatch(JSONObject jsonObject, String ip);
+
+ void gtpcMatch(JSONObject jsonObject, String logValue, String appendToKey, String param);
+
+ String getTopDomain(String domain);
+
+ String decodeBase64(JSONObject jsonObject,String message, String param);
+
+ Object flattenSpec(String message, String expr);
+
+ Object condition(JSONObject jsonObject, String ifParam);
+
+ void setValue(JSONObject jsonObject, String appendToKey, String param);
+
+ void getValue(JSONObject jsonObject, String appendToKey, Object logValue);
+}
diff --git a/src/main/java/com/zdjizhi/tools/transform/impl/TransformFunctionImpl.java b/src/main/java/com/zdjizhi/tools/transform/impl/TransformFunctionImpl.java
new file mode 100644
index 0000000..a176125
--- /dev/null
+++ b/src/main/java/com/zdjizhi/tools/transform/impl/TransformFunctionImpl.java
@@ -0,0 +1,307 @@
+package com.zdjizhi.tools.transform.impl;
+
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
+import com.alibaba.fastjson2.JSONObject;
+import com.geedgenetworks.utils.FormatUtils;
+import com.geedgenetworks.utils.StringUtil;
+import com.zdjizhi.common.FlowWriteConfig;
+import com.zdjizhi.tools.general.IpLookupUtils;
+import com.zdjizhi.tools.general.SnowflakeId;
+import com.zdjizhi.tools.transform.TransformFunction;
+import com.zdjizhi.tools.connections.hbase.HBaseUtils;
+import com.zdjizhi.tools.json.JsonPathUtil;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Base64;
+import java.util.HashMap;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi.tools.transform.impl
+ * @Description:
+ * @date 2023/5/20 10:12
+ */
+public class TransformFunctionImpl implements TransformFunction {
+ private static final Log logger = LogFactory.get();
+ private static final int IF_PARAM_LENGTH = 3;
+ /**
+ * if函数连接分隔符
+ */
+ private static final String IF_CONDITION_SPLITTER = "=";
+
+ private static final String SEPARATOR = "!=";
+
+ /**
+ * 标识字段为日志字段还是schema指定字段
+ */
+ private static final String IS_JSON_KEY_TAG = "$.";
+
+ /**
+ * 生成当前时间戳的操作
+ */
+ @Override
+ public long getCurrentTime() {
+ return System.currentTimeMillis() / 1000;
+ }
+
+ @Override
+ public long getSnowflakeId() {
+ return SnowflakeId.generateId();
+ }
+
+ /**
+ * 根据clientIp获取location信息
+ *
+ * @param ip client IP
+ * @return ip地址详细信息
+ */
+ @Override
+ public String getGeoIpDetail(String ip) {
+ String detail = "";
+ try {
+ detail = IpLookupUtils.getIpLookup().cityLookupDetail(ip);
+ } catch (NullPointerException npe) {
+ logger.error("The IP Location MMDB file is not loaded or IP is null! " + npe);
+ } catch (RuntimeException e) {
+ logger.error("Get clientIP location error! " + e.getMessage());
+ }
+ return detail;
+ }
+
+ /**
+ * 根据ip获取country信息
+ *
+ * @param ip server IP
+ * @return 国家
+ */
+ @Override
+ public String getGeoIpCountry(String ip) {
+ String country = "";
+ try {
+ country = IpLookupUtils.getIpLookup().countryLookup(ip);
+ } catch (NullPointerException npe) {
+ logger.error("The IP Location MMDB file is not loaded or IP is null! " + npe);
+ } catch (RuntimeException e) {
+ logger.error("Get ServerIP location error! " + e.getMessage());
+ }
+ return country;
+ }
+
+ /**
+ * 根据ip获取asn信息
+ *
+ * @param ip client/server IP
+ * @return ASN
+ */
+ @Override
+ public String getGeoAsn(String ip) {
+ String asn = "";
+ try {
+ asn = IpLookupUtils.getIpLookup().asnLookup(ip);
+ } catch (NullPointerException npe) {
+ logger.error("The ASN MMDB file is not loaded or IP is null! " + npe);
+ } catch (RuntimeException e) {
+ logger.error("Get IP ASN error! " + e.getMessage());
+ }
+ return asn;
+ }
+
+ /**
+ * radius借助HBase补齐
+ *
+ * @param ip client IP
+ * @return account
+ */
+ @Override
+ public String radiusMatch(JSONObject jsonObject, String ip) {
+ if (FlowWriteConfig.DEFAULT_RELATIONSHIP_MODULE.equals(FlowWriteConfig.DATA_RELATIONSHIP_MODEL)) {
+ int vsysId = jsonObject.getIntValue("common_vsys_id", 1);
+ return HBaseUtils.getAccount(ip + vsysId);
+ } else {
+ return HBaseUtils.getAccount(ip);
+ }
+ }
+
+
+ /**
+ * 借助HBase补齐GTP-C信息,解析tunnels信息，优先使用gtp_uplink_teid，其次使用gtp_downlink_teid
+ *
+ * "common_tunnels":[{"tunnels_schema_type":"GTP","gtp_endpoint_a2b_teid":235261261,"gtp_endpoint_b2a_teid":665547833,"gtp_sgw_ip":"192.56.5.2","gtp_pgw_ip":"192.56.10.20","gtp_sgw_port":2152,"gtp_pgw_port":2152}]
+ *
+ * @param jsonObject 原始日志json
+ * @param logValue 上行TEID
+ * @param appendToKey 结果值映射到的日志字段key
+ * @param param 用于解析jsonarray，直接定位到GTP信息所在的位置
+ */
+ @Override
+ public void gtpcMatch(JSONObject jsonObject, String logValue, String appendToKey, String param) {
+ try {
+ String teid = null;
+ String[] exprs = param.split(FlowWriteConfig.FORMAT_SPLITTER);
+ for (String expr : exprs) {
+ Object result = JsonPathUtil.analysis(logValue, expr);
+ if (result != null) {
+ teid = result.toString();
+ break;
+ }
+ }
+
+ if (teid != null) {
+ if (FlowWriteConfig.DEFAULT_RELATIONSHIP_MODULE.equals(FlowWriteConfig.DATA_RELATIONSHIP_MODEL)) {
+ int vsysId = jsonObject.getIntValue("common_vsys_id", 1);
+ teid = teid + vsysId;
+ }
+ String[] appendToKeys = appendToKey.split(FlowWriteConfig.FORMAT_SPLITTER);
+ HashMap userData = HBaseUtils.getGtpData(teid);
+ if (userData != null) {
+ for (String key : appendToKeys) {
+ jsonObject.put(key, userData.get(key).toString());
+ }
+ } else {
+ logger.warn("Description The user whose TEID is " + teid + " was not matched!");
+ }
+ }
+ } catch (RuntimeException re) {
+ logger.error("An exception occurred in teid type conversion or parsing of user information!" + re.getMessage());
+ re.printStackTrace();
+ }
+ }
+
+ /**
+ * 解析顶级域名
+ *
+ * @param domain 初始域名
+ * @return 顶级域名
+ */
+ @Override
+ public String getTopDomain(String domain) {
+ String topDomain = "";
+ try {
+ topDomain = FormatUtils.getTopPrivateDomain(domain);
+ } catch (StringIndexOutOfBoundsException outException) {
+ logger.error("Parse top-level domain exceptions, exception domain names:" + domain);
+ }
+ return topDomain;
+ }
+
+ /**
+ * 根据编码解码base64
+ *
+ * @param jsonObject 原始日志json
+ * @param message base64
+ * @param param 用于获取编码的参数
+ * @return 解码字符串
+ */
+ @Override
+ public String decodeBase64(JSONObject jsonObject, String message, String param) {
+ String decodeResult = "";
+ Object charset = isJsonValue(jsonObject, param);
+ try {
+ if (StringUtil.isNotBlank(message)) {
+ byte[] base64decodedBytes = Base64.getDecoder().decode(message);
+ if (charset == null) {
+ decodeResult = new String(base64decodedBytes);
+ } else {
+ decodeResult = new String(base64decodedBytes, charset.toString());
+ }
+ }
+ } catch (RuntimeException e) {
+ logger.error("Resolve Base64 exception, exception information:" + e.getMessage());
+ } catch (UnsupportedEncodingException e) {
+ logger.error("The Character Encoding [" + charset.toString() + "] is not supported.exception information:" + e.getMessage());
+ }
+ return decodeResult;
+ }
+
+ /**
+ * 根据表达式解析json
+ *
+ * @param message json
+ * @param expr 解析表达式
+ * @return 解析结果
+ */
+ @Override
+ public Object flattenSpec(String message, String expr) {
+ return JsonPathUtil.analysis(message, expr);
+ }
+
+ /**
+ * IF函数实现，解析日志构建三目运算;包含判断是否为数字若为数字则转换为long类型返回结果。
+ *
+ * @param jsonObject 原始日志反序列化对象
+ * @param ifParam 字段名/普通字符串
+ * @return resultA or resultB or null
+ */
+ @Override
+ public Object condition(JSONObject jsonObject, String ifParam) {
+ Object result = null;
+ try {
+ String[] split = ifParam.split(FlowWriteConfig.FORMAT_SPLITTER);
+ if (split.length == IF_PARAM_LENGTH) {
+ String expression = split[0];
+ Object resultA = isJsonValue(jsonObject, split[1]);
+ Object resultB = isJsonValue(jsonObject, split[2]);
+ if (expression.contains(SEPARATOR)) {
+ String[] regexp = expression.split(SEPARATOR);
+ Object direction = isJsonValue(jsonObject, regexp[0]);
+ if (direction instanceof Number) {
+ result = Integer.parseInt(direction.toString()) != Integer.parseInt(regexp[1]) ? resultA : resultB;
+ } else if (direction instanceof String) {
+ result = direction.equals(regexp[1]) ? resultA : resultB;
+ }
+ } else {
+ String[] regexp = expression.split(IF_CONDITION_SPLITTER);
+ Object direction = isJsonValue(jsonObject, regexp[0]);
+ if (direction instanceof Number) {
+ result = Integer.parseInt(direction.toString()) == Integer.parseInt(regexp[1]) ? resultA : resultB;
+ } else if (direction instanceof String) {
+ result = direction.equals(regexp[1]) ? resultA : resultB;
+ }
+ }
+ }
+ } catch (RuntimeException e) {
+ logger.error("IF function execution exception, exception information:" + e.getMessage());
+ }
+ return result;
+ }
+
+ /**
+ * 给json中的某个key赋值（指定值）
+ *
+ * @param jsonObject 原始日志json
+ * @param appendToKey 要赋值的key
+ * @param param 参数（指定值）
+ */
+ @Override
+ public void setValue(JSONObject jsonObject, String appendToKey, String param) {
+ jsonObject.put(appendToKey, param);
+ }
+
+ /**
+ * 从json中获取A的值，赋值给B
+ *
+ * @param jsonObject 原始日志json
+ * @param appendToKey 要赋值的key
+ * @param logValue 获取的值
+ */
+ @Override
+ public void getValue(JSONObject jsonObject, String appendToKey, Object logValue) {
+ jsonObject.put(appendToKey, logValue);
+ }
+
+ /**
+ * 判断是否为日志字段,是则返回对应value，否则返回原始字符串
+ *
+ * @param jsonObject 原始日志反序列化对象
+ * @param param 字段名/普通字符串
+ * @return JSON.Value or String
+ */
+ private static Object isJsonValue(JSONObject jsonObject, String param) {
+ if (param.contains(IS_JSON_KEY_TAG)) {
+ return jsonObject.get(param.substring(2));
+ } else {
+ return param;
+ }
+ }
+}
diff --git a/src/main/java/com/zdjizhi/topology/LogFlowWriteTopology.java b/src/main/java/com/zdjizhi/topology/LogFlowWriteTopology.java
index bbb190e..e473df6 100644
--- a/src/main/java/com/zdjizhi/topology/LogFlowWriteTopology.java
+++ b/src/main/java/com/zdjizhi/topology/LogFlowWriteTopology.java
@@ -2,66 +2,47 @@ package com.zdjizhi.topology;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
+
+import com.alibaba.fastjson2.JSONObject;
import com.zdjizhi.common.FlowWriteConfig;
-import com.zdjizhi.utils.functions.DealFileProcessFunction;
-import com.zdjizhi.utils.functions.FilterNullFunction;
-import com.zdjizhi.utils.functions.MapCompletedFunction;
-import com.zdjizhi.utils.functions.TypeMapCompletedFunction;
-import com.zdjizhi.utils.kafka.KafkaConsumer;
-import com.zdjizhi.utils.kafka.KafkaProducer;
-import org.apache.flink.streaming.api.datastream.DataStream;
+import com.zdjizhi.operator.count.SendCountProcess;
+import com.zdjizhi.operator.map.MapCompleted;
+import com.zdjizhi.operator.map.TypeMapCompleted;
+import com.zdjizhi.operator.process.DealFileProcessFunction;
+import com.zdjizhi.tools.connections.kafka.KafkaConsumer;
+import com.zdjizhi.tools.connections.kafka.KafkaProducer;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import java.util.Map;
-/**
- * @author 王成成
- * @Package com.zdjizhi.topology
- * @Description:
- * @date 2022.06.01
- */
+
public class LogFlowWriteTopology {
private static final Log logger = LogFactory.get();
public static void main(String[] args) {
+
final StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
+
//两个输出之间的最大时间 (单位milliseconds)
environment.setBufferTimeout(FlowWriteConfig.BUFFER_TIMEOUT);
- if (FlowWriteConfig.LOG_NEED_COMPLETE == 1) {
+ SingleOutputStreamOperator completedStream;
+ if (FlowWriteConfig.LOG_TRANSFORM_TYPE == 0) {//不对日志字段类型做校验。
+ completedStream = environment.addSource(KafkaConsumer.flinkConsumer()).name(FlowWriteConfig.SOURCE_KAFKA_TOPIC).setParallelism(FlowWriteConfig.SOURCE_PARALLELISM)
+ .process(new MapCompleted()).name("MapCompletedFunction").setParallelism(FlowWriteConfig.TRANSFORM_PARALLELISM);
- SingleOutputStreamOperator> streamSource = environment.addSource(KafkaConsumer.myDeserializationConsumer())
- .setParallelism(FlowWriteConfig.SOURCE_PARALLELISM).name(FlowWriteConfig.SOURCE_KAFKA_TOPIC);
+ } else {//对日志字段类型做弱校验，可根据schema进行强转。
+ completedStream = environment.addSource(KafkaConsumer.flinkConsumer()).name(FlowWriteConfig.SOURCE_KAFKA_TOPIC).setParallelism(FlowWriteConfig.SOURCE_PARALLELISM)
+ .process(new TypeMapCompleted()).name("TypeMapCompletedFunction").setParallelism(FlowWriteConfig.TRANSFORM_PARALLELISM);
- DataStream> cleaningLog;
- switch (FlowWriteConfig.LOG_TRANSFORM_TYPE) {
- case 0:
- //对原始日志进行处理补全转换等，不对日志字段类型做校验。
- cleaningLog = streamSource.map(new MapCompletedFunction()).name("MapCompletedFunction")
- .setParallelism(FlowWriteConfig.TRANSFORM_PARALLELISM);
- break;
- case 1:
- //对原始日志进行处理补全转换等，对日志字段类型做若校验，可根据schema进行强转。
- cleaningLog = streamSource.map(new TypeMapCompletedFunction()).name("TypeMapCompletedFunction")
- .setParallelism(FlowWriteConfig.TRANSFORM_PARALLELISM);
- break;
- default:
- //对原始日志进行处理补全转换等，不对日志字段类型做校验。
- cleaningLog = streamSource.map(new MapCompletedFunction()).name("MapCompletedFunction")
- .setParallelism(FlowWriteConfig.TRANSFORM_PARALLELISM);
- }
- //处理带有非结构化日志的数据
- SingleOutputStreamOperator process = cleaningLog.process(new DealFileProcessFunction());
- SingleOutputStreamOperator resultFileMetaData = process.getSideOutput(DealFileProcessFunction.metaToKafa).filter(new FilterNullFunction()).name("FilterAbnormalTrafficFileMetaData").setParallelism(FlowWriteConfig.TRANSFORM_PARALLELISM);
- SingleOutputStreamOperator result = process.filter(new FilterNullFunction()).name("FilterAbnormalData").setParallelism(FlowWriteConfig.TRANSFORM_PARALLELISM);
- //文件元数据发送至TRAFFIC-FILE-METADATA
- resultFileMetaData.addSink(KafkaProducer.getTrafficFileMetaKafkaProducer()).name("toTrafficFileMeta")
- .setParallelism(FlowWriteConfig.FILE_DATA_SINK_PARALLELISM);
- //补全后的数据发送给百分点的kafka
- result.addSink(KafkaProducer.getPercentKafkaProducer()).name("toPercentKafka")
- .setParallelism(FlowWriteConfig.PERCENT_SINK_PARALLELISM);
}
+ //处理带有非结构化文件字段的数据
+ SingleOutputStreamOperator dealFileProcessFunction = completedStream.process(new DealFileProcessFunction()).name("DealFileProcessFunction").uid("DealFile-ProcessFunction").setParallelism(FlowWriteConfig.DEAL_FILE_PARALLELISM);
+ //补全后的数据发送至百分点的kafka
+ dealFileProcessFunction.addSink(KafkaProducer.getPercentKafkaProducer()).name("ToPercentKafka").uid("To-Percent-Kafka").setParallelism(FlowWriteConfig.SINK_PERCENT_PARALLELISM);
+ //文件元数据发送至TRAFFIC-FILE-METADATA
+ dealFileProcessFunction.getSideOutput(DealFileProcessFunction.metaToKafa).addSink(KafkaProducer.getTrafficFileMetaKafkaProducer()).name("toTrafficFileMeta").uid("to-Traffic-FileMeta").setParallelism(FlowWriteConfig.SINK_FILE_DATA_PARALLELISM);
+ dealFileProcessFunction.getSideOutput(DealFileProcessFunction.dealFileMetircTag).process(new SendCountProcess()).name("SendCountProcess").uid("Send-Count-Process").setParallelism(1);
try {
environment.execute(args[0]);
} catch (Exception e) {
@@ -71,4 +52,5 @@ public class LogFlowWriteTopology {
}
+
}
diff --git a/src/main/java/com/zdjizhi/utils/app/AppUtils.java b/src/main/java/com/zdjizhi/utils/app/AppUtils.java
deleted file mode 100644
index 1425ce9..0000000
--- a/src/main/java/com/zdjizhi/utils/app/AppUtils.java
+++ /dev/null
@@ -1,124 +0,0 @@
-package com.zdjizhi.utils.app;
-
-import cn.hutool.log.Log;
-import cn.hutool.log.LogFactory;
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
-import com.zdjizhi.common.FlowWriteConfig;
-import com.zdjizhi.utils.StringUtil;
-import com.zdjizhi.utils.http.HttpClientUtil;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- * AppId 工具类
- *
- * @author qidaijie
- */
-
-@Deprecated
-public class AppUtils {
- private static final Log logger = LogFactory.get();
- private static Map appIdMap = new ConcurrentHashMap<>(128);
- private static AppUtils appUtils;
-
- private static void getAppInstance() {
- appUtils = new AppUtils();
- }
-
-
- /**
- * 构造函数-新
- */
- private AppUtils() {
- //定时更新
- updateAppIdCache();
- }
-
- /**
- * 更新变量
- */
- private static void change() {
- if (appUtils == null) {
- getAppInstance();
- }
- timestampsFilter();
- }
-
-
- /**
- * 获取变更内容
- */
- private static void timestampsFilter() {
- try {
- Long begin = System.currentTimeMillis();
- String schema = HttpClientUtil.requestByGetMethod(FlowWriteConfig.APP_ID_HTTP);
- if (StringUtil.isNotBlank(schema)) {
- String data = JSONObject.parseObject(schema).getString("data");
- JSONArray objects = JSONArray.parseArray(data);
- for (Object object : objects) {
- JSONArray jsonArray = JSONArray.parseArray(object.toString());
- int key = jsonArray.getInteger(0);
- String value = jsonArray.getString(1);
- if (appIdMap.containsKey(key)) {
- if (!value.equals(appIdMap.get(key))) {
- appIdMap.put(key, value);
- }
- } else {
- appIdMap.put(key, value);
- }
- }
- logger.warn("Updating the correspondence takes time:" + (begin - System.currentTimeMillis()));
- logger.warn("Pull the length of the interface data:[" + objects.size() + "]");
- }
- } catch (RuntimeException e) {
- logger.error("Update cache app-id failed, exception：" + e);
- }
- }
-
-
- /**
- * 验证定时器,每隔一段时间验证一次-验证获取新的Cookie
- */
- private void updateAppIdCache() {
- ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1);
- executorService.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- try {
- if (FlowWriteConfig.APP_TICK_TUPLE_FREQ_SECS != 0) {
- change();
- }
- } catch (RuntimeException e) {
- logger.error("AppUtils update AppCache is error===>{" + e + "}<===");
- }
- }
- }, 1, FlowWriteConfig.APP_TICK_TUPLE_FREQ_SECS, TimeUnit.SECONDS);
- }
-
-
- /**
- * 获取 appName
- *
- * @param appId app_id
- * @return account
- */
- public static String getAppName(int appId) {
-
- if (appUtils == null) {
- getAppInstance();
- }
-
- if (appIdMap.containsKey(appId)) {
- return appIdMap.get(appId);
- } else {
- logger.warn("AppMap get appName is null, ID is :" + appId);
- return "";
- }
- }
-
-}
diff --git a/src/main/java/com/zdjizhi/utils/functions/FilterNullFunction.java b/src/main/java/com/zdjizhi/utils/functions/FilterNullFunction.java
deleted file mode 100644
index de507ad..0000000
--- a/src/main/java/com/zdjizhi/utils/functions/FilterNullFunction.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package com.zdjizhi.utils.functions;
-
-import com.zdjizhi.utils.StringUtil;
-import org.apache.flink.api.common.functions.FilterFunction;
-
-/**
- * @author qidaijie
- * @Package com.zdjizhi.utils.functions
- * @Description:
- * @date 2021/5/2715:01
- */
-public class FilterNullFunction implements FilterFunction {
- @Override
- public boolean filter(String message) {
- return StringUtil.isNotBlank(message);
- }
-}
diff --git a/src/main/java/com/zdjizhi/utils/functions/MapCompletedFunction.java b/src/main/java/com/zdjizhi/utils/functions/MapCompletedFunction.java
deleted file mode 100644
index ec811f5..0000000
--- a/src/main/java/com/zdjizhi/utils/functions/MapCompletedFunction.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package com.zdjizhi.utils.functions;
-
-
-import com.zdjizhi.utils.general.TransFormMap;
-import org.apache.flink.api.common.functions.MapFunction;
-
-import java.util.Map;
-
-
-/**
- * @author qidaijie
- * @Package com.zdjizhi.utils.functions
- * @Description:
- * @date 2021/5/2715:01
- */
-public class MapCompletedFunction implements MapFunction, Map> {
-
- @Override
- @SuppressWarnings("unchecked")
- public Map map(Map logs) {
- return TransFormMap.dealCommonMessage(logs);
- }
-}
diff --git a/src/main/java/com/zdjizhi/utils/functions/TypeMapCompletedFunction.java b/src/main/java/com/zdjizhi/utils/functions/TypeMapCompletedFunction.java
deleted file mode 100644
index 61a6a46..0000000
--- a/src/main/java/com/zdjizhi/utils/functions/TypeMapCompletedFunction.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package com.zdjizhi.utils.functions;
-
-import com.zdjizhi.utils.general.TransFormTypeMap;
-import org.apache.flink.api.common.functions.MapFunction;
-
-import java.util.Map;
-
-
-/**
- * @author qidaijie
- * @Package com.zdjizhi.utils.functions
- * @Description:
- * @date 2021/5/2715:01
- */
-public class TypeMapCompletedFunction implements MapFunction, Map> {
-
- @Override
- @SuppressWarnings("unchecked")
- public Map map(Map logs) {
-
- return TransFormTypeMap.dealCommonMessage(logs);
- }
-}
diff --git a/src/main/java/com/zdjizhi/utils/general/CityHash.java b/src/main/java/com/zdjizhi/utils/general/CityHash.java
deleted file mode 100644
index 5de4785..0000000
--- a/src/main/java/com/zdjizhi/utils/general/CityHash.java
+++ /dev/null
@@ -1,180 +0,0 @@
-package com.zdjizhi.utils.general;
-
-
-
-
-/**
- * CityHash64算法对logid进行散列计算
- * 版本规划暂不实现-TSG22.01
- *
- * @author qidaijie
- */
-@Deprecated
-public class CityHash {
-
- private static final long k0 = 0xc3a5c85c97cb3127L;
- private static final long k1 = 0xb492b66fbe98f273L;
- private static final long k2 = 0x9ae16a3b2f90404fL;
- private static final long k3 = 0xc949d7c7509e6557L;
- private static final long k5 = 0x9ddfea08eb382d69L;
-
- private CityHash() {}
-
- public static long CityHash64(byte[] s, int index, int len) {
- if (len <= 16 ) {
- return HashLen0to16(s, index, len);
- } else if (len > 16 && len <= 32) {
- return HashLen17to32(s, index, len);
- } else if (len > 32 && len <= 64) {
- return HashLen33to64(s, index, len);
- } else {
- long x = Fetch64(s, index);
- long y = Fetch64(s, index + len - 16) ^ k1;
- long z = Fetch64(s, index + len - 56) ^ k0;
- long[] v = WeakHashLen32WithSeeds(s, len - 64, len, y);
- long[] w = WeakHashLen32WithSeeds(s, len - 32, len * k1, k0);
- z += ShiftMix(v[1]) * k1;
- x = Rotate(z + x, 39) * k1;
- y = Rotate(y, 33) * k1;
-
- len = (len - 1) & ~63;
- do {
- x = Rotate(x + y + v[0] + Fetch64(s, index + 16), 37) * k1;
- y = Rotate(y + v[1] + Fetch64(s, index + 48), 42) * k1;
- x ^= w[1];
- y ^= v[0];
- z = Rotate(z ^ w[0], 33);
- v = WeakHashLen32WithSeeds(s, index, v[1] * k1, x + w[0]);
- w = WeakHashLen32WithSeeds(s, index + 32, z + w[1], y);
- long t = z;
- z = x;
- x = t;
- index += 64;
- len -= 64;
- } while (len != 0);
- return HashLen16(HashLen16(v[0], w[0]) + ShiftMix(y) * k1 + z,
- HashLen16(v[1], w[1]) + x);
- }
- }
-
- private static long HashLen0to16(byte[] s, int index, int len) {
- if (len > 8) {
- long a = Fetch64(s, index);
- long b = Fetch64(s, index + len - 8);
- return HashLen16(a, RotateByAtLeastOne(b + len, len)) ^ b;
- }
- if (len >= 4) {
- long a = Fetch32(s, index);
- return HashLen16(len + (a << 3), Fetch32(s, index + len - 4));
- }
- if (len > 0) {
- byte a = s[index];
- byte b = s[index + len >>> 1];
- byte c = s[index + len - 1];
- int y = (a) + (b << 8);
- int z = len + (c << 2);
- return ShiftMix(y * k2 ^ z * k3) * k2;
- }
- return k2;
- }
-
- private static long HashLen17to32(byte[] s, int index, int len) {
- long a = Fetch64(s, index) * k1;
- long b = Fetch64(s, index + 8);
- long c = Fetch64(s, index + len - 8) * k2;
- long d = Fetch64(s, index + len - 16) * k0;
- return HashLen16(Rotate(a - b, 43) + Rotate(c, 30) + d,
- a + Rotate(b ^ k3, 20) - c + len);
- }
-
- private static long HashLen33to64(byte[] s, int index, int len) {
- long z = Fetch64(s, index + 24);
- long a = Fetch64(s, index) + (len + Fetch64(s, index + len - 16)) * k0;
- long b = Rotate(a + z, 52);
- long c = Rotate(a, 37);
- a += Fetch64(s, index + 8);
- c += Rotate(a, 7);
- a += Fetch64(s, index + 16);
- long vf = a + z;
- long vs = b + Rotate(a, 31) + c;
- a = Fetch64(s, index + 16) + Fetch64(s, index + len - 32);
- z = Fetch64(s, index + len - 8);
- b = Rotate(a + z, 52);
- c = Rotate(a, 37);
- a += Fetch64(s, index + len - 24);
- c += Rotate(a, 7);
- a += Fetch64(s, index + len - 16);
- long wf = a + z;
- long ws = b + Rotate(a, 31) + c;
- long r = ShiftMix((vf + ws) * k2 + (wf + vs) * k0);
- return ShiftMix(r * k0 + vs) * k2;
- }
-
- private static long Fetch64(byte[] p, int index) {
- return toLongLE(p,index);
- }
-
- private static long Fetch32(byte[] p, int index) {
- return toIntLE(p,index);
- }
- private static long[] WeakHashLen32WithSeeds(
- long w, long x, long y, long z, long a, long b) {
- a += w;
- b = Rotate(b + a + z, 21);
- long c = a;
- a += x;
- a += y;
- b += Rotate(a, 44);
- return new long[]{a + z, b + c};
- }
-
- private static long[] WeakHashLen32WithSeeds(byte[] s, int index, long a, long b) {
- return WeakHashLen32WithSeeds(Fetch64(s, index),
- Fetch64(s, index + 8),
- Fetch64(s, index + 16),
- Fetch64(s, index + 24),
- a,
- b);
- }
-
- private static long toLongLE(byte[] b, int i) {
- return 0xffffffffffffffffL & (((long) b[i + 7] << 56) + ((long) (b[i + 6] & 255) << 48) + ((long) (b[i + 5] & 255) << 40) + ((long) (b[i + 4] & 255) << 32) + ((long) (b[i + 3] & 255) << 24) + ((b[i + 2] & 255) << 16) + ((b[i + 1] & 255) << 8) + ((b[i + 0] & 255) << 0));
- }
-
- private static long toIntLE(byte[] b, int i) {
- return 0xffffffffL & (((b[i + 3] & 255) << 24) + ((b[i + 2] & 255) << 16) + ((b[i + 1] & 255) << 8) + ((b[i + 0] & 255) << 0));
- }
- private static long RotateByAtLeastOne(long val, int shift) {
- return (val >>> shift) | (val << (64 - shift));
- }
-
- private static long ShiftMix(long val) {
- return val ^ (val >>> 47);
- }
-
- private static long Uint128Low64(long[] x) {
- return x[0];
- }
-
- private static long Rotate(long val, int shift) {
- return shift == 0 ? val : (val >>> shift) | (val << (64 - shift));
- }
-
- private static long Uint128High64(long[] x) {
- return x[1];
- }
-
- private static long Hash128to64(long[] x) {
- long a = (Uint128Low64(x) ^ Uint128High64(x)) * k5;
- a ^= (a >>> 47);
- long b = (Uint128High64(x) ^ a) * k5;
- b ^= (b >>> 47);
- b *= k5;
- return b;
- }
-
- private static long HashLen16(long u, long v) {
- return Hash128to64(new long[]{u,v});
- }
-
-}
\ No newline at end of file
diff --git a/src/main/java/com/zdjizhi/utils/general/TransFormMap.java b/src/main/java/com/zdjizhi/utils/general/TransFormMap.java
deleted file mode 100644
index 9aa54c8..0000000
--- a/src/main/java/com/zdjizhi/utils/general/TransFormMap.java
+++ /dev/null
@@ -1,130 +0,0 @@
-package com.zdjizhi.utils.general;
-
-
-import cn.hutool.log.Log;
-import cn.hutool.log.LogFactory;
-import com.zdjizhi.utils.JsonMapper;
-import com.zdjizhi.utils.json.JsonParseUtil;
-
-import java.util.Map;
-
-
-/**
- * 描述:转换或补全工具类
- *
- * @author qidaijie
- */
-public class TransFormMap {
- private static final Log logger = LogFactory.get();
-
- /**
- * 解析日志，并补全
- *
- * @param jsonMap kafka Topic消费原始日志并解析
- * @return 补全后的日志
- */
- @SuppressWarnings("unchecked")
- public static Map dealCommonMessage(Map jsonMap) {
- try {
- JsonParseUtil.dropJsonField(jsonMap);
- for (String[] strings : JsonParseUtil.getJobList()) {
- //用到的参数的值
- Object logValue = JsonParseUtil.getValue(jsonMap, strings[0]);
- //需要补全的字段的key
- String appendToKeyName = strings[1];
- //需要补全的字段的值
- Object appendTo = JsonParseUtil.getValue(jsonMap, appendToKeyName);
- //匹配操作函数的字段
- String function = strings[2];
- //额外的参数的值
- String param = strings[3];
- functionSet(function, jsonMap, appendToKeyName, appendTo, logValue, param);
- }
- return jsonMap;
- } catch (RuntimeException e) {
- logger.error("TransForm logs failed,The exception is :" + e);
- return null;
- }
- }
-
-
- /**
- * 根据schema描述对应字段进行操作的 函数集合
- *
- * @param function 匹配操作函数的字段
- * @param jsonMap 原始日志解析map
- * @param appendToKeyName 需要补全的字段的key
- * @param appendTo 需要补全的字段的值
- * @param logValue 用到的参数的值
- * @param param 额外的参数的值
- */
- private static void functionSet(String function, Map jsonMap, String appendToKeyName, Object appendTo, Object logValue, String param) {
- switch (function) {
- case "current_timestamp":
- if (!(appendTo instanceof Long)) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getCurrentTime());
- }
- break;
- case "snowflake_id":
- JsonParseUtil.setValue(jsonMap, appendToKeyName, SnowflakeId.generateId());
- break;
- case "geo_ip_detail":
- if (logValue != null && appendTo == null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpDetail(logValue.toString()));
- }
- break;
- case "geo_asn":
- if (logValue != null && appendTo == null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoAsn(logValue.toString()));
- }
- break;
- case "geo_ip_country":
- if (logValue != null && appendTo == null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpCountry(logValue.toString()));
- }
- break;
- case "set_value":
- if (param != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, param);
- }
- break;
- case "get_value":
- if (logValue != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, logValue);
- }
- break;
- case "if":
- if (param != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.condition(jsonMap, param));
- }
- break;
- case "sub_domain":
- if (appendTo == null && logValue != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getTopDomain(logValue.toString()));
- }
- break;
- case "radius_match":
- if (logValue != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.radiusMatch(logValue.toString()));
- }
- break;
- case "decode_of_base64":
- if (logValue != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.decodeBase64(logValue.toString(), TransFunction.isJsonValue(jsonMap, param)));
- }
- break;
- case "flattenSpec":
- if (logValue != null && param != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.flattenSpec(logValue.toString(), param));
- }
- break;
- case "app_match":
- if (logValue != null && appendTo == null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.appMatch(logValue.toString()));
- }
- break;
- default:
- }
- }
-
-}
\ No newline at end of file
diff --git a/src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java b/src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java
deleted file mode 100644
index 95f6a89..0000000
--- a/src/main/java/com/zdjizhi/utils/general/TransFormTypeMap.java
+++ /dev/null
@@ -1,132 +0,0 @@
-package com.zdjizhi.utils.general;
-
-
-import cn.hutool.log.Log;
-import cn.hutool.log.LogFactory;
-import com.zdjizhi.utils.JsonMapper;
-import com.zdjizhi.utils.json.JsonParseUtil;
-
-import java.util.Map;
-
-
-/**
- * 描述:转换或补全工具类
- *
- * @author qidaijie
- */
-public class TransFormTypeMap {
- private static final Log logger = LogFactory.get();
-
- /**
- * 解析日志，并补全
- *
- * @param message kafka Topic原始日志
- * @return 补全后的日志
- */
- @SuppressWarnings("unchecked")
- public static Map dealCommonMessage(Map message) {
- try {
- Map jsonMap = JsonParseUtil.typeTransform(message);
- for (String[] strings : JsonParseUtil.getJobList()) {
- //用到的参数的值
- Object logValue = JsonParseUtil.getValue(jsonMap, strings[0]);
- //需要补全的字段的key
- String appendToKeyName = strings[1];
- //需要补全的字段的值
- Object appendToKeyValue = JsonParseUtil.getValue(jsonMap, appendToKeyName);
- //匹配操作函数的字段
- String function = strings[2];
- //额外的参数的值
- String param = strings[3];
- functionSet(function, jsonMap, appendToKeyName, appendToKeyValue, logValue, param);
- }
- return jsonMap;
- } catch (RuntimeException e) {
- logger.error("TransForm logs failed,The exception is :" + e);
- return null;
- }
- }
-
-
- /**
- * 根据schema描述对应字段进行操作的 函数集合
- *
- * @param function 匹配操作函数的字段
- * @param jsonMap 原始日志解析map
- * @param appendToKeyName 需要补全的字段的key
- * @param appendToKeyValue 需要补全的字段的值
- * @param logValue 用到的参数的值
- * @param param 额外的参数的值
- */
- private static void functionSet(String function, Map jsonMap, String appendToKeyName, Object appendToKeyValue, Object logValue, String param) {
- switch (function) {
- case "current_timestamp":
- if (!(appendToKeyValue instanceof Long)) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getCurrentTime());
- }
- break;
- case "snowflake_id":
- JsonParseUtil.setValue(jsonMap, appendToKeyName, SnowflakeId.generateId());
- //版本规划暂不实现TSG-22.01
-// JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getDecimalHash(SnowflakeId.generateId()));
- break;
- case "geo_ip_detail":
- if (logValue != null && appendToKeyValue == null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpDetail(logValue.toString()));
- }
- break;
- case "geo_asn":
- if (logValue != null && appendToKeyValue == null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoAsn(logValue.toString()));
- }
- break;
- case "geo_ip_country":
- if (logValue != null && appendToKeyValue == null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getGeoIpCountry(logValue.toString()));
- }
- break;
- case "set_value":
- if (param != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, param);
- }
- break;
- case "get_value":
- if (logValue != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, logValue);
- }
- break;
- case "if":
- if (param != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.condition(jsonMap, param));
- }
- break;
- case "sub_domain":
- if (appendToKeyValue == null && logValue != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.getTopDomain(logValue.toString()));
- }
- break;
- case "radius_match":
- if (logValue != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.radiusMatch(logValue.toString()));
- }
- break;
- case "decode_of_base64":
- if (logValue != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.decodeBase64(logValue.toString(), TransFunction.isJsonValue(jsonMap, param)));
- }
- break;
- case "flattenSpec":
- if (logValue != null && param != null) {
- JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.flattenSpec(logValue.toString(), param));
- }
- break;
- case "app_match":
- if (logValue != null && appendToKeyValue == null) {
-// JsonParseUtil.setValue(jsonMap, appendToKeyName, TransFunction.appMatch(logValue.toString()));
- }
- break;
- default:
- }
- }
-
-}
\ No newline at end of file
diff --git a/src/main/java/com/zdjizhi/utils/general/TransFunction.java b/src/main/java/com/zdjizhi/utils/general/TransFunction.java
deleted file mode 100644
index bc9e893..0000000
--- a/src/main/java/com/zdjizhi/utils/general/TransFunction.java
+++ /dev/null
@@ -1,297 +0,0 @@
-package com.zdjizhi.utils.general;
-
-import cn.hutool.core.codec.Base64;
-import cn.hutool.core.util.StrUtil;
-import cn.hutool.log.Log;
-import cn.hutool.log.LogFactory;
-import com.jayway.jsonpath.InvalidPathException;
-import com.jayway.jsonpath.JsonPath;
-import com.zdjizhi.common.FlowWriteConfig;
-import com.zdjizhi.utils.FormatUtils;
-import com.zdjizhi.utils.IpLookupV2;
-import com.zdjizhi.utils.StringUtil;
-import com.zdjizhi.utils.app.AppUtils;
-import com.zdjizhi.utils.hbase.HBaseUtils;
-import com.zdjizhi.utils.json.JsonParseUtil;
-import com.zdjizhi.utils.json.TypeUtils;
-
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * @author qidaijie
- */
-class TransFunction {
- private static final Log logger = LogFactory.get();
-
- /**
- * 校验数字正则
- */
- private static final Pattern PATTERN = Pattern.compile("[0-9]*");
-
- /**
- * IP定位库工具类
- */
- private static IpLookupV2 ipLookup = new IpLookupV2.Builder(false)
- .loadDataFileV4(FlowWriteConfig.TOOLS_LIBRARY + "ip_v4_built_in.mmdb")
- .loadDataFileV6(FlowWriteConfig.TOOLS_LIBRARY + "ip_v6_built_in.mmdb")
- .loadDataFilePrivateV4(FlowWriteConfig.TOOLS_LIBRARY + "ip_v4_user_defined.mmdb")
- .loadDataFilePrivateV6(FlowWriteConfig.TOOLS_LIBRARY + "ip_v6_user_defined.mmdb")
- .loadAsnDataFile(FlowWriteConfig.TOOLS_LIBRARY + "asn_v4.mmdb")
- .loadAsnDataFileV6(FlowWriteConfig.TOOLS_LIBRARY + "asn_v6.mmdb")
- .build();
-
- /**
- * 生成当前时间戳的操作
- */
- static long getCurrentTime() {
-
- return System.currentTimeMillis() / 1000;
- }
-
- /**
- * CityHash64算法
- * 版本规划暂不实现-TSG22.01
- *
- * @param data 原始数据
- * @return 散列结果
- */
- @Deprecated
- static BigInteger getDecimalHash(long data) {
- byte[] dataBytes = String.valueOf(data).getBytes();
- long hashValue = CityHash.CityHash64(dataBytes, 0, dataBytes.length);
- String decimalValue = Long.toUnsignedString(hashValue, 10);
- BigInteger result = new BigInteger(decimalValue);
- return result;
- }
-
- /**
- * 根据clientIp获取location信息
- *
- * @param ip client IP
- * @return ip地址详细信息
- */
- static String getGeoIpDetail(String ip) {
- return ipLookup.cityLookupDetail(ip);
- }
-
- /**
- * 根据ip获取asn信息
- *
- * @param ip client/server IP
- * @return ASN
- */
- static String getGeoAsn(String ip) {
- return ipLookup.asnLookup(ip);
- }
-
- /**
- * 根据ip获取country信息
- *
- * @param ip server IP
- * @return 国家
- */
- static String getGeoIpCountry(String ip) {
-
- return ipLookup.countryLookup(ip);
- }
-
-
- /**
- * radius借助HBase补齐
- *
- * @param ip client IP
- * @return account
- */
- static String radiusMatch(String ip) {
- return HBaseUtils.getAccount(ip.trim());
- }
-
- /**
- * appId与缓存中对应关系补全appName
- *
- * @param appIds app id 列表
- * @return appName
- */
- @Deprecated
- static String appMatch(String appIds) {
- try {
- String appId = StrUtil.split(appIds, FlowWriteConfig.FORMAT_SPLITTER, true, true).get(0);
- return AppUtils.getAppName(Integer.parseInt(appId));
- } catch (NumberFormatException | ClassCastException exception) {
- logger.error("APP ID列表分割转换异常，异常APP ID列表:" + appIds);
- return "";
- }
- }
-
- /**
- * 解析顶级域名
- *
- * @param domain 初始域名
- * @return 顶级域名
- */
- static String getTopDomain(String domain) {
- try {
- return FormatUtils.getTopPrivateDomain(domain);
- } catch (StringIndexOutOfBoundsException outException) {
- logger.error("解析顶级域名异常,异常域名:" + domain);
- return "";
- }
- }
-
- /**
- * 根据编码解码base64
- *
- * @param message base64
- * @param charset 编码
- * @return 解码字符串
- */
- static String decodeBase64(String message, Object charset) {
- String result = "";
- try {
- if (StringUtil.isNotBlank(message)) {
- if (charset == null) {
- result = Base64.decodeStr(message, FlowWriteConfig.MAIL_DEFAULT_CHARSET);
- } else {
- result = Base64.decodeStr(message, charset.toString());
- }
- }
- } catch (RuntimeException rune) {
- logger.error("解析 Base64 异常,异常信息:" + rune);
- }
- return result;
- }
-
- /**
- * 根据表达式解析json
- *
- * @param message json
- * @param expr 解析表达式
- * @return 解析结果
- */
- static String flattenSpec(String message, String expr) {
- String flattenResult = "";
- try {
- if (StringUtil.isNotBlank(expr)) {
- ArrayList read = JsonPath.parse(message).read(expr);
- if (read.size() >= 1) {
- flattenResult = read.get(0);
- }
- }
- } catch (ClassCastException | InvalidPathException | ArrayIndexOutOfBoundsException e) {
- logger.error("设备标签解析异常，[ " + expr + " ]解析表达式错误" + e);
- }
- return flattenResult;
- }
-
-
- /**
- * 判断是否为日志字段,是则返回对应value，否则返回原始字符串
- *
- * @param object 内存实体类
- * @param param 字段名/普通字符串
- * @return JSON.Value or String
- */
- static Object isJsonValue(Object object, String param) {
- if (param.contains(FlowWriteConfig.IS_JSON_KEY_TAG)) {
- return JsonParseUtil.getValue(object, param.substring(2));
- } else {
- return param;
- }
- }
-
- /**
- * 判断是否为日志字段,是则返回对应value，否则返回原始字符串
- *
- * @param jsonMap 内存实体类
- * @param param 字段名/普通字符串
- * @return JSON.Value or String
- */
- static Object isJsonValue(Map jsonMap, String param) {
- if (param.contains(FlowWriteConfig.IS_JSON_KEY_TAG)) {
- return JsonParseUtil.getValue(jsonMap, param.substring(2));
- } else {
- return param;
- }
- }
-
- /**
- * IF函数实现，解析日志构建三目运算;包含判断是否为数字若为数字则转换为long类型返回结果。
- *
- * @param object 内存实体类
- * @param ifParam 字段名/普通字符串
- * @return resultA or resultB or null
- */
- static Object condition(Object object, String ifParam) {
- Object result = null;
- try {
- String[] split = ifParam.split(FlowWriteConfig.FORMAT_SPLITTER);
- if (split.length == FlowWriteConfig.IF_PARAM_LENGTH) {
- String[] norms = split[0].split(FlowWriteConfig.IF_CONDITION_SPLITTER);
- Object direction = isJsonValue(object, norms[0]);
- Object resultA = isJsonValue(object, split[1]);
- Object resultB = isJsonValue(object, split[2]);
- if (direction instanceof Number) {
- result = TypeUtils.castToIfFunction((Integer.parseInt(direction.toString()) == Integer.parseInt(norms[1])) ? resultA : resultB);
- } else if (direction instanceof String) {
- result = TypeUtils.castToIfFunction(direction.equals(norms[1]) ? resultA : resultB);
- }
- }
- } catch (RuntimeException e) {
- logger.error("IF 函数执行异常,异常信息:" + e);
- }
- return result;
- }
-
- /**
- * IF函数实现，解析日志构建三目运算;包含判断是否为数字若为数字则转换为long类型返回结果。
- *
- * @param jsonMap 内存实体类
- * @param ifParam 字段名/普通字符串
- * @return resultA or resultB or null
- */
- static Object condition(Map jsonMap, String ifParam) {
- Object result = null;
- try {
- String[] split = ifParam.split(FlowWriteConfig.FORMAT_SPLITTER);
- if (split.length == FlowWriteConfig.IF_PARAM_LENGTH) {
- String[] norms = split[0].split(FlowWriteConfig.IF_CONDITION_SPLITTER);
- Object direction = isJsonValue(jsonMap, norms[0]);
- Object resultA = isJsonValue(jsonMap, split[1]);
- Object resultB = isJsonValue(jsonMap, split[2]);
- if (direction instanceof Number) {
- result = (Integer.parseInt(direction.toString()) == Integer.parseInt(norms[1])) ? resultA : resultB;
- } else if (direction instanceof String) {
- result = direction.equals(norms[1]) ? resultA : resultB;
- }
- }
- } catch (RuntimeException e) {
- logger.error("IF 函数执行异常,异常信息:" + e);
- }
- return result;
- }
-
-
- /**
- * 设置固定值函数 若为数字则转为long返回
- *
- * @param param 默认值
- * @return 返回数字或字符串
- */
- static Object setValue(String param) {
- try {
- Matcher isNum = PATTERN.matcher(param);
- if (isNum.matches()) {
- return Long.parseLong(param);
- } else {
- return param;
- }
- } catch (RuntimeException e) {
- logger.error("SetValue 函数异常，异常信息：" + e);
- }
- return null;
- }
-}
diff --git a/src/main/java/com/zdjizhi/utils/hbase/HBaseUtils.java b/src/main/java/com/zdjizhi/utils/hbase/HBaseUtils.java
deleted file mode 100644
index 6aa904f..0000000
--- a/src/main/java/com/zdjizhi/utils/hbase/HBaseUtils.java
+++ /dev/null
@@ -1,208 +0,0 @@
-package com.zdjizhi.utils.hbase;
-
-import cn.hutool.log.Log;
-import cn.hutool.log.LogFactory;
-import com.zdjizhi.common.FlowWriteConfig;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- * HBase 工具类
- *
- * @author qidaijie
- */
-
-public class HBaseUtils {
- private static final Log logger = LogFactory.get();
- private static Map subIdMap = new ConcurrentHashMap<>(16);
- private static Connection connection;
- private static Long time;
-
- private static HBaseUtils hBaseUtils;
-
- private static void getInstance() {
- hBaseUtils = new HBaseUtils();
- }
-
-
- /**
- * 构造函数-新
- */
- private HBaseUtils() {
- //获取连接
- getConnection();
- //拉取所有
- getAll();
- //定时更新
- updateCache();
- }
-
- private static void getConnection() {
- try {
- // 管理Hbase的配置信息
- Configuration configuration = HBaseConfiguration.create();
- // 设置zookeeper节点
- configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS);
- configuration.set("hbase.client.retries.number", "3");
- configuration.set("hbase.bulkload.retries.number", "3");
- configuration.set("zookeeper.recovery.retry", "3");
- connection = ConnectionFactory.createConnection(configuration);
- time = System.currentTimeMillis();
- logger.warn("HBaseUtils get HBase connection,now to getAll().");
- } catch (IOException ioe) {
- logger.error("HBaseUtils getHbaseConn() IOException===>{" + ioe + "}<===");
- } catch (RuntimeException e) {
- logger.error("HBaseUtils getHbaseConn() Exception===>{" + e + "}<===");
- }
- }
-
- /**
- * 更新变量
- */
- private static void change() {
- if (hBaseUtils == null) {
- getInstance();
- }
- long nowTime = System.currentTimeMillis();
- timestampsFilter(time - 1000, nowTime + 500);
- }
-
-
- /**
- * 获取变更内容
- *
- * @param startTime 开始时间
- * @param endTime 结束时间
- */
- private static void timestampsFilter(Long startTime, Long endTime) {
- Long begin = System.currentTimeMillis();
- Table table = null;
- ResultScanner scanner = null;
- Scan scan2 = new Scan();
- try {
- table = connection.getTable(TableName.valueOf(FlowWriteConfig.HBASE_TABLE_NAME));
- scan2.setTimeRange(startTime, endTime);
- scanner = table.getScanner(scan2);
- for (Result result : scanner) {
- int acctStatusType = getAcctStatusType(result);
- String framedIp = Bytes.toString(result.getValue(Bytes.toBytes("radius"), Bytes.toBytes("framed_ip"))).trim();
- String account = Bytes.toString(result.getValue(Bytes.toBytes("radius"), Bytes.toBytes("account"))).trim();
- if (acctStatusType == 1) {
- if (subIdMap.containsKey(framedIp)) {
- boolean same = account.equals(subIdMap.get(framedIp));
- if (!same) {
- subIdMap.put(framedIp, account);
- }
- } else {
- subIdMap.put(framedIp, account);
- }
- } else if (acctStatusType == 2) {
- subIdMap.remove(framedIp);
- }
- }
- Long end = System.currentTimeMillis();
- logger.warn("HBaseUtils Now subIdMap.keySet().size() is: " + subIdMap.keySet().size());
- logger.warn("HBaseUtils Update cache timeConsuming is: " + (end - begin) + ",BeginTime: " + startTime + ",EndTime: " + endTime);
- time = endTime;
- } catch (IOException ioe) {
- logger.error("HBaseUtils timestampsFilter is IOException===>{" + ioe + "}<===");
- } catch (RuntimeException e) {
- logger.error("HBaseUtils timestampsFilter is Exception===>{" + e + "}<===");
- } finally {
- if (scanner != null) {
- scanner.close();
- }
- if (table != null) {
- try {
- table.close();
- } catch (IOException e) {
- logger.error("HBase Table Close ERROR! Exception message is:" + e);
- }
- }
- }
- }
-
- /**
- * 获取所有的 key value
- */
- private static void getAll() {
- long begin = System.currentTimeMillis();
- try {
- Table table = connection.getTable(TableName.valueOf(FlowWriteConfig.HBASE_TABLE_NAME));
- Scan scan2 = new Scan();
- ResultScanner scanner = table.getScanner(scan2);
- for (Result result : scanner) {
- int acctStatusType = getAcctStatusType(result);
- String framedIp = Bytes.toString(result.getValue(Bytes.toBytes("radius"), Bytes.toBytes("framed_ip")));
- String account = Bytes.toString(result.getValue(Bytes.toBytes("radius"), Bytes.toBytes("account")));
- if (acctStatusType == 1) {
- subIdMap.put(framedIp, account);
- }
- }
- logger.warn("HBaseUtils Get fullAmount List size->subIdMap.size(): " + subIdMap.size());
- logger.warn("HBaseUtils Get fullAmount List size->subIdMap.size() timeConsuming is: " + (System.currentTimeMillis() - begin));
- scanner.close();
- } catch (IOException ioe) {
- logger.error("HBaseUtils getAll() is IOException===>{" + ioe + "}<===");
- } catch (RuntimeException e) {
- logger.error("HBaseUtils getAll() is Exception===>{" + e + "}<===");
- }
- }
-
- /**
- * 验证定时器,每隔一段时间验证一次-验证获取新的Cookie
- */
- private void updateCache() {
-// ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1,
-// new BasicThreadFactory.Builder().namingPattern("hbase-change-pool-%d").daemon(true).build());
- ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1);
- executorService.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- try {
- if (FlowWriteConfig.HBASE_TICK_TUPLE_FREQ_SECS != 0) {
- change();
- }
- } catch (RuntimeException e) {
- logger.error("HBaseUtils update hbaseCache is error===>{" + e + "}<===");
- }
- }
- }, 1, FlowWriteConfig.HBASE_TICK_TUPLE_FREQ_SECS, TimeUnit.SECONDS);
- }
-
-
- /**
- * 获取 account
- *
- * @param clientIp client_ip
- * @return account
- */
- public static String getAccount(String clientIp) {
-
- if (hBaseUtils == null) {
- getInstance();
- }
- return subIdMap.get(clientIp);
-
- }
-
- private static int getAcctStatusType(Result result) {
- boolean hasType = result.containsColumn(Bytes.toBytes("radius"), Bytes.toBytes("acct_status_type"));
- if (hasType) {
- return Bytes.toInt(result.getValue(Bytes.toBytes("radius"), Bytes.toBytes("acct_status_type")));
- } else {
- return 1;
- }
- }
-
-}
diff --git a/src/main/java/com/zdjizhi/utils/http/HttpClientUtil.java b/src/main/java/com/zdjizhi/utils/http/HttpClientUtil.java
deleted file mode 100644
index 1adb1d1..0000000
--- a/src/main/java/com/zdjizhi/utils/http/HttpClientUtil.java
+++ /dev/null
@@ -1,77 +0,0 @@
-package com.zdjizhi.utils.http;
-
-import cn.hutool.log.Log;
-import cn.hutool.log.LogFactory;
-import org.apache.commons.io.IOUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-/**
- * 获取网关schema的工具类
- *
- * @author qidaijie
- */
-public class HttpClientUtil {
- private static final Log logger = LogFactory.get();
-
- /**
- * 请求网关获取schema
- *
- * @param http 网关url
- * @return schema
- */
- public static String requestByGetMethod(String http) {
- CloseableHttpClient httpClient = HttpClients.createDefault();
- StringBuilder entityStringBuilder;
-
- HttpGet get = new HttpGet(http);
- BufferedReader bufferedReader = null;
- CloseableHttpResponse httpResponse = null;
- try {
- httpResponse = httpClient.execute(get);
- HttpEntity entity = httpResponse.getEntity();
- entityStringBuilder = new StringBuilder();
- if (null != entity) {
- bufferedReader = new BufferedReader(new InputStreamReader(httpResponse.getEntity().getContent(), "UTF-8"), 8 * 1024);
- int intC;
- while ((intC = bufferedReader.read()) != -1) {
- char c = (char) intC;
- if (c == '\n') {
- break;
- }
- entityStringBuilder.append(c);
- }
-
- return entityStringBuilder.toString();
- }
- } catch (IOException e) {
- logger.error("Get Schema from Query engine ERROR! Exception message is:" + e);
- } finally {
- if (httpClient != null) {
- try {
- httpClient.close();
- } catch (IOException e) {
- logger.error("Close HTTP Client ERROR! Exception messgae is:" + e);
- }
- }
- if (httpResponse != null) {
- try {
- httpResponse.close();
- } catch (IOException e) {
- logger.error("Close httpResponse ERROR! Exception messgae is:" + e);
- }
- }
- if (bufferedReader != null) {
- IOUtils.closeQuietly(bufferedReader);
- }
- }
- return "";
- }
-}
diff --git a/src/main/java/com/zdjizhi/utils/json/JsonParseUtil.java b/src/main/java/com/zdjizhi/utils/json/JsonParseUtil.java
deleted file mode 100644
index f477848..0000000
--- a/src/main/java/com/zdjizhi/utils/json/JsonParseUtil.java
+++ /dev/null
@@ -1,372 +0,0 @@
-package com.zdjizhi.utils.json;
-
-import cn.hutool.log.Log;
-import cn.hutool.log.LogFactory;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
-import com.alibaba.nacos.api.NacosFactory;
-import com.alibaba.nacos.api.PropertyKeyConst;
-import com.alibaba.nacos.api.config.ConfigService;
-import com.alibaba.nacos.api.config.listener.Listener;
-import com.alibaba.nacos.api.exception.NacosException;
-import com.jayway.jsonpath.JsonPath;
-import com.zdjizhi.common.FlowWriteConfig;
-import com.zdjizhi.utils.StringUtil;
-import net.sf.cglib.beans.BeanMap;
-
-import java.util.*;
-import java.util.concurrent.Executor;
-
-
-/**
- * 使用FastJson解析json的工具类
- *
- * @author qidaijie
- */
-public class JsonParseUtil {
- private static final Log logger = LogFactory.get();
- private static Properties propNacos = new Properties();
-
- /**
- * 获取需要删除字段的列表
- */
- private static ArrayList dropList = new ArrayList<>();
-
- /**
- * 在内存中加载反射类用的map
- */
- private static HashMap jsonFieldsMap;
-
- /**
- * 获取任务列表
- * list的每个元素是一个四元字符串数组 (有format标识的字段，补全的字段，用到的功能函数，用到的参数)，例如：
- * (mail_subject mail_subject decode_of_base64 mail_subject_charset)
- */
- private static ArrayList jobList;
-
- static {
- propNacos.setProperty(PropertyKeyConst.SERVER_ADDR, FlowWriteConfig.NACOS_SERVER);
- propNacos.setProperty(PropertyKeyConst.NAMESPACE, FlowWriteConfig.NACOS_SCHEMA_NAMESPACE);
- propNacos.setProperty(PropertyKeyConst.USERNAME, FlowWriteConfig.NACOS_USERNAME);
- propNacos.setProperty(PropertyKeyConst.PASSWORD, FlowWriteConfig.NACOS_PIN);
- try {
- ConfigService configService = NacosFactory.createConfigService(propNacos);
- String dataId = FlowWriteConfig.NACOS_DATA_ID;
- String group = FlowWriteConfig.NACOS_GROUP;
- String schema = configService.getConfig(dataId, group, 5000);
- if (StringUtil.isNotBlank(schema)) {
- jsonFieldsMap = getMapFromHttp(schema);
- jobList = getJobListFromHttp(schema);
- }
- configService.addListener(dataId, group, new Listener() {
- @Override
- public Executor getExecutor() {
- return null;
- }
-
- @Override
- public void receiveConfigInfo(String configMsg) {
- if (StringUtil.isNotBlank(configMsg)) {
- clearCache();
- jsonFieldsMap = getMapFromHttp(configMsg);
- jobList = getJobListFromHttp(configMsg);
- }
- }
- });
- } catch (NacosException e) {
- logger.error("Get Schema config from Nacos error,The exception message is :" + e.getMessage());
- }
- }
-
- /**
- * 模式匹配，给定一个类型字符串返回一个类类型
- *
- * @param type 类型
- * @return 类类型
- */
-
- private static Class getClassName(String type) {
- Class clazz;
-
- switch (type) {
- case "int":
- clazz = Integer.class;
- break;
- case "string":
- clazz = String.class;
- break;
- case "long":
- clazz = long.class;
- break;
- case "array":
- clazz = List.class;
- break;
- case "double":
- clazz = double.class;
- break;
- case "float":
- clazz = float.class;
- break;
- case "char":
- clazz = char.class;
- break;
- case "byte":
- clazz = byte.class;
- break;
- case "boolean":
- clazz = boolean.class;
- break;
- case "short":
- clazz = short.class;
- break;
- default:
- clazz = String.class;
- }
- return clazz;
- }
-
- /**
- * 获取属性值的方法
- *
- * @param obj 对象
- * @param property key
- * @return 属性的值
- */
- public static Object getValue(Object obj, String property) {
- try {
- BeanMap beanMap = BeanMap.create(obj);
- return beanMap.get(property);
- } catch (RuntimeException e) {
- logger.error("获取json-value异常，异常key：" + property + "异常信息为：" + e);
- return null;
- }
- }
-
- /**
- * 获取属性值的方法
- *
- * @param jsonMap 原始日志
- * @param property key
- * @return 属性的值
- */
- public static Object getValue(Map jsonMap, String property) {
- try {
- return jsonMap.getOrDefault(property, null);
- } catch (RuntimeException e) {
- logger.error("获取json-value异常，异常key：" + property + "异常信息为：" + e);
- return null;
- }
- }
-
- /**
- * 更新属性值的方法
- *
- * @param jsonMap 原始日志json map
- * @param property 更新的key
- * @param value 更新的值
- */
- public static void setValue(Map jsonMap, String property, Object value) {
- try {
- jsonMap.put(property, value);
- } catch (RuntimeException e) {
- logger.error("赋予实体类错误类型数据", e);
- }
- }
-
- /**
- * 更新属性值的方法
- *
- * @param obj 对象
- * @param property 更新的key
- * @param value 更新的值
- */
- public static void setValue(Object obj, String property, Object value) {
- try {
- BeanMap beanMap = BeanMap.create(obj);
- beanMap.put(property, value);
- } catch (ClassCastException e) {
- logger.error("赋予实体类错误类型数据", e);
- }
- }
-
- /**
- * 类型转换
- *
- * @param jsonMap 原始日志map
- */
- public static Map typeTransform(Map jsonMap) throws RuntimeException {
- JsonParseUtil.dropJsonField(jsonMap);
- HashMap tmpMap = new HashMap<>(192);
- for (String key : jsonMap.keySet()) {
- if (jsonFieldsMap.containsKey(key)) {
- String simpleName = jsonFieldsMap.get(key).getSimpleName();
- switch (simpleName) {
- case "String":
- tmpMap.put(key, JsonTypeUtil.checkString(jsonMap.get(key)));
- break;
- case "Integer":
- tmpMap.put(key, JsonTypeUtil.getIntValue(jsonMap.get(key)));
- break;
- case "long":
- tmpMap.put(key, JsonTypeUtil.checkLongValue(jsonMap.get(key)));
- break;
- case "List":
- tmpMap.put(key, JsonTypeUtil.checkArray(jsonMap.get(key)));
- break;
- case "Map":
- tmpMap.put(key, JsonTypeUtil.checkObject(jsonMap.get(key)));
- break;
- case "double":
- tmpMap.put(key, JsonTypeUtil.checkDouble(jsonMap.get(key)));
- break;
- default:
- tmpMap.put(key, JsonTypeUtil.checkString(jsonMap.get(key)));
- }
- }
- }
- return tmpMap;
- }
-
- public static ArrayList getJobList() {
- return jobList;
- }
-
-
- /**
- * 通过获取String类型的网关schema链接来获取map，用于生成一个Object类型的对象
- *
- * // * @param http 网关schema地址
- *
- * @return 用于反射生成schema类型的对象的一个map集合
- */
- private static HashMap getMapFromHttp(String schema) {
- HashMap map = new HashMap<>(16);
-
- //获取fields，并转化为数组，数组的每个元素都是一个name doc type
- JSONObject schemaJson = JSON.parseObject(schema);
- JSONArray fields = (JSONArray) schemaJson.get("fields");
-
- for (Object field : fields) {
- String filedStr = field.toString();
- if (checkKeepField(filedStr)) {
- String name = JsonPath.read(filedStr, "$.name").toString();
- String type = JsonPath.read(filedStr, "$.type").toString();
- if (type.contains("{")) {
- type = JsonPath.read(filedStr, "$.type.type").toString();
- }
- //组合用来生成实体类的map
- map.put(name, getClassName(type));
- } else {
- dropList.add(filedStr);
- }
- }
- return map;
- }
-
- /**
- * 判断字段是否需要保留
- *
- * @param message 单个field-json
- * @return true or false
- */
- private static boolean checkKeepField(String message) {
- boolean isKeepField = true;
- boolean isHiveDoc = JSON.parseObject(message).containsKey("doc");
- if (isHiveDoc) {
- boolean isHiveVi = JsonPath.read(message, "$.doc").toString().contains("visibility");
- if (isHiveVi) {
- String visibility = JsonPath.read(message, "$.doc.visibility").toString();
- if (FlowWriteConfig.VISIBILITY.equals(visibility)) {
- isKeepField = false;
- }
- }
- }
- return isKeepField;
- }
-
- /**
- * 删除schema内指定的无效字段（jackson）
- *
- * @param jsonMap
- */
- public static void dropJsonField(Map jsonMap) {
- for (String field : dropList) {
- jsonMap.remove(field);
- }
- }
-
- /**
- * 解析schema，解析之后返回一个任务列表 (useList toList funcList paramlist)
- *
- * @param schema 日志schema
- * @return 任务列表
- */
- private static ArrayList getJobListFromHttp(String schema) {
- ArrayList list = new ArrayList<>();
-
- //获取fields，并转化为数组，数组的每个元素都是一个name doc type
- JSONObject schemaJson = JSON.parseObject(schema);
- JSONArray fields = (JSONArray) schemaJson.get("fields");
-
- for (Object field : fields) {
-
- if (JSON.parseObject(field.toString()).containsKey("doc")) {
- Object doc = JSON.parseObject(field.toString()).get("doc");
-
- if (JSON.parseObject(doc.toString()).containsKey("format")) {
- String name = JSON.parseObject(field.toString()).get("name").toString();
- Object format = JSON.parseObject(doc.toString()).get("format");
- JSONObject formatObject = JSON.parseObject(format.toString());
-
- String functions = formatObject.get("functions").toString();
- String appendTo = null;
- String params = null;
-
- if (formatObject.containsKey("appendTo")) {
- appendTo = formatObject.get("appendTo").toString();
- }
-
- if (formatObject.containsKey("param")) {
- params = formatObject.get("param").toString();
- }
-
-
- if (StringUtil.isNotBlank(appendTo) && StringUtil.isBlank(params)) {
- String[] functionArray = functions.split(FlowWriteConfig.FORMAT_SPLITTER);
- String[] appendToArray = appendTo.split(FlowWriteConfig.FORMAT_SPLITTER);
-
- for (int i = 0; i < functionArray.length; i++) {
- list.add(new String[]{name, appendToArray[i], functionArray[i], null});
- }
-
- } else if (StringUtil.isNotBlank(appendTo) && StringUtil.isNotBlank(params)) {
- String[] functionArray = functions.split(FlowWriteConfig.FORMAT_SPLITTER);
- String[] appendToArray = appendTo.split(FlowWriteConfig.FORMAT_SPLITTER);
- String[] paramArray = params.split(FlowWriteConfig.FORMAT_SPLITTER);
-
- for (int i = 0; i < functionArray.length; i++) {
- list.add(new String[]{name, appendToArray[i], functionArray[i], paramArray[i]});
-
- }
- } else {
- list.add(new String[]{name, name, functions, params});
- }
-
- }
- }
-
- }
- return list;
- }
-
- /**
- * 在配置变动时，清空缓存重新获取
- */
- private static void clearCache() {
- jobList.clear();
- jsonFieldsMap.clear();
- dropList.clear();
- }
-
-}
\ No newline at end of file
diff --git a/src/main/java/com/zdjizhi/utils/json/JsonTypeUtil.java b/src/main/java/com/zdjizhi/utils/json/JsonTypeUtil.java
deleted file mode 100644
index 79087a6..0000000
--- a/src/main/java/com/zdjizhi/utils/json/JsonTypeUtil.java
+++ /dev/null
@@ -1,129 +0,0 @@
-package com.zdjizhi.utils.json;
-
-import com.zdjizhi.common.FlowWriteConfig;
-import com.zdjizhi.utils.JsonMapper;
-import com.zdjizhi.utils.exception.FlowWriteException;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-
-/**
- * @author qidaijie
- * @Package PACKAGE_NAME
- * @Description:
- * @date 2021/7/1217:34
- */
-public class JsonTypeUtil {
- /**
- * 类型转换
- *
- * @param jsonMap 原始日志map
- */
-
- /**
- * String 类型检验转换方法
- *
- * @param value json value
- * @return String value
- */
- static String checkString(Object value) {
- if (value == null) {
- return null;
- }
-
- if (value instanceof Map) {
- return JsonMapper.toJsonString(value);
- }
-
- if (value instanceof List) {
- return JsonMapper.toJsonString(value);
- }
-
- return value.toString();
- }
-
- /**
- * array 类型检验转换方法
- *
- * @param value json value
- * @return List value
- */
- static Map checkObject(Object value) {
- if (value == null) {
- return null;
- }
-
- if (value instanceof Map) {
- return (Map) value;
- }
-
- throw new FlowWriteException("can not cast to map, value : " + value);
- }
-
- /**
- * array 类型检验转换方法
- *
- * @param value json value
- * @return List value
- */
- static List checkArray(Object value) {
- if (value == null) {
- return null;
- }
-
- if (value instanceof List) {
- return (List) value;
- }
-
- throw new FlowWriteException("can not cast to List, value : " + value);
- }
-
- /**
- * long 类型检验转换方法,若为空返回基础值
- *
- * @param value json value
- * @return Long value
- */
- static long checkLongValue(Object value) {
- Long longVal = TypeUtils.castToLong(value);
-
- if (longVal == null) {
- return 0L;
- }
-
- return longVal;
- }
-
- /**
- * Double 类型校验转换方法
- *
- * @param value json value
- * @return Double value
- */
- static Double checkDouble(Object value) {
- if (value == null) {
- return null;
- }
-
- return TypeUtils.castToDouble(value);
- }
-
-
- /**
- * int 类型检验转换方法,若为空返回基础值
- *
- * @param value json value
- * @return int value
- */
- static int getIntValue(Object value) {
-
- Integer intVal = TypeUtils.castToInt(value);
- if (intVal == null) {
- return 0;
- }
- return intVal;
- }
-
-}
diff --git a/src/main/java/com/zdjizhi/utils/json/TypeUtils.java b/src/main/java/com/zdjizhi/utils/json/TypeUtils.java
deleted file mode 100644
index b13627f..0000000
--- a/src/main/java/com/zdjizhi/utils/json/TypeUtils.java
+++ /dev/null
@@ -1,171 +0,0 @@
-package com.zdjizhi.utils.json;
-
-import cn.hutool.log.Log;
-import cn.hutool.log.LogFactory;
-import com.zdjizhi.common.FlowWriteConfig;
-import com.zdjizhi.utils.StringUtil;
-import com.zdjizhi.utils.exception.FlowWriteException;
-
-/**
- * @author qidaijie
- * @Package PACKAGE_NAME
- * @Description:
- * @date 2021/7/1218:20
- */
-public class TypeUtils {
- private static final Log logger = LogFactory.get();
-
- /**
- * Integer 类型判断方法
- *
- * @param value json value
- * @return Integer value or null
- */
- public static Object castToIfFunction(Object value) {
- if (value == null) {
- return null;
- }
-
- if (value instanceof String) {
- return value.toString();
- }
-
- if (value instanceof Integer) {
- return ((Number) value).intValue();
- }
-
- if (value instanceof Long) {
- return ((Number) value).longValue();
- }
-
-// if (value instanceof Map) {
-// return (Map) value;
-// }
-//
-// if (value instanceof List) {
-// return Collections.singletonList(value.toString());
-// }
-
- if (value instanceof Boolean) {
- return (Boolean) value ? 1 : 0;
- }
-
- throw new FlowWriteException("can not cast to int, value : " + value);
- }
-
- /**
- * Integer 类型判断方法
- *
- * @param value json value
- * @return Integer value or null
- */
- static Integer castToInt(Object value) {
- if (value == null) {
- return null;
- }
-
- if (value instanceof Integer) {
- return (Integer) value;
- }
-
- if (value instanceof Number) {
- return ((Number) value).intValue();
- }
-
- if (value instanceof String) {
- String strVal = (String) value;
- if (StringUtil.isBlank(strVal)) {
- return null;
- }
-
- //将 10,20 类数据转换为10
- if (strVal.contains(FlowWriteConfig.FORMAT_SPLITTER)) {
- strVal = strVal.split(FlowWriteConfig.FORMAT_SPLITTER)[0];
- }
-
- try {
- return Integer.parseInt(strVal);
- } catch (NumberFormatException ex) {
- logger.error("String change Integer Error,The error Str is:" + strVal);
- }
- }
-
- if (value instanceof Boolean) {
- return (Boolean) value ? 1 : 0;
- }
-
- throw new FlowWriteException("can not cast to int, value : " + value);
- }
-
- /**
- * Double类型判断方法
- *
- * @param value json value
- * @return double value or null
- */
- static Double castToDouble(Object value) {
-
- if (value instanceof Number) {
- return ((Number) value).doubleValue();
- }
-
- if (value instanceof String) {
- String strVal = (String) value;
-
- if (StringUtil.isBlank(strVal)) {
- return null;
- }
-
- //将 10,20 类数据转换为10
- if (strVal.contains(FlowWriteConfig.FORMAT_SPLITTER)) {
- strVal = strVal.split(FlowWriteConfig.FORMAT_SPLITTER)[0];
- }
-
- try {
- return Double.parseDouble(strVal);
- } catch (NumberFormatException ex) {
- logger.error("String change Double Error,The error Str is:" + strVal);
- }
- }
-
- throw new FlowWriteException("can not cast to double, value : " + value);
- }
-
- /**
- * Long类型判断方法
- *
- * @param value json value
- * @return (Long)value or null
- */
- static Long castToLong(Object value) {
- if (value == null) {
- return null;
- }
-
- if (value instanceof Number) {
- return ((Number) value).longValue();
- }
-
- if (value instanceof String) {
- String strVal = (String) value;
-
- if (StringUtil.isBlank(strVal)) {
- return null;
- }
-
- //将 10,20 类数据转换为10
- if (strVal.contains(FlowWriteConfig.FORMAT_SPLITTER)) {
- strVal = strVal.split(FlowWriteConfig.FORMAT_SPLITTER)[0];
- }
-
- try {
- return Long.parseLong(strVal);
- } catch (NumberFormatException ex) {
- logger.error("String change Long Error,The error Str is:" + strVal);
- }
- }
-
- throw new FlowWriteException("can not cast to long, value : " + value);
- }
-
-}
diff --git a/src/main/java/com/zdjizhi/utils/kafka/KafkaProducer.java b/src/main/java/com/zdjizhi/utils/kafka/KafkaProducer.java
deleted file mode 100644
index a431c09..0000000
--- a/src/main/java/com/zdjizhi/utils/kafka/KafkaProducer.java
+++ /dev/null
@@ -1,82 +0,0 @@
-package com.zdjizhi.utils.kafka;
-
-import com.zdjizhi.common.FlowWriteConfig;
-import org.apache.flink.api.common.serialization.SimpleStringSchema;
-import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
-
-import java.util.Optional;
-import java.util.Properties;
-
-/**
- * @author qidaijie
- * @Package com.zdjizhi.utils.kafka
- * @Description:
- * @date 2021/6/814:04
- */
-public class KafkaProducer {
-
-
- private static Properties createPercentProducerConfig() {
- Properties properties = new Properties();
- properties.put("bootstrap.servers", FlowWriteConfig.PERCENT_SINK_KAFKA_SERVERS);
- properties.put("acks", FlowWriteConfig.PRODUCER_ACK);
- properties.put("retries", FlowWriteConfig.RETRIES);
- properties.put("linger.ms", FlowWriteConfig.LINGER_MS);
- properties.put("request.timeout.ms", FlowWriteConfig.REQUEST_TIMEOUT_MS);
- properties.put("batch.size", FlowWriteConfig.BATCH_SIZE);
- properties.put("buffer.memory", FlowWriteConfig.BUFFER_MEMORY);
- properties.put("max.request.size", FlowWriteConfig.MAX_REQUEST_SIZE);
- properties.put("compression.type", FlowWriteConfig.PRODUCER_KAFKA_COMPRESSION_TYPE);
-
- CertUtils.chooseCert(FlowWriteConfig.PERCENT_SINK_KAFKA_SERVERS, properties);
-
- return properties;
- }
-
-
-
- private static Properties createTrafficFileMetaProducerConfig() {
- Properties properties = new Properties();
- properties.put("bootstrap.servers", FlowWriteConfig.FILE_DATA_SINK_KAFKA_SERVERS);
- properties.put("acks", FlowWriteConfig.PRODUCER_ACK);
- properties.put("retries", FlowWriteConfig.RETRIES);
- properties.put("linger.ms", FlowWriteConfig.LINGER_MS);
- properties.put("request.timeout.ms", FlowWriteConfig.REQUEST_TIMEOUT_MS);
- properties.put("batch.size", FlowWriteConfig.BATCH_SIZE);
- properties.put("buffer.memory", FlowWriteConfig.BUFFER_MEMORY);
- properties.put("max.request.size", FlowWriteConfig.MAX_REQUEST_SIZE);
- properties.put("compression.type", FlowWriteConfig.PRODUCER_KAFKA_COMPRESSION_TYPE);
-
- CertUtils.chooseCert(FlowWriteConfig.FILE_DATA_SINK_KAFKA_SERVERS, properties);
-
- return properties;
- }
-
-
-
- public static FlinkKafkaProducer getPercentKafkaProducer() {
- FlinkKafkaProducer kafkaProducer = new FlinkKafkaProducer(
- FlowWriteConfig.PERCENT_KAFKA_TOPIC,
- new SimpleStringSchema(),
- createPercentProducerConfig(), Optional.empty());
-
- kafkaProducer.setLogFailuresOnly(false);
-
-// kafkaProducer.setWriteTimestampToKafka(true);
-
- return kafkaProducer;
- }
- public static FlinkKafkaProducer getTrafficFileMetaKafkaProducer() {
- FlinkKafkaProducer kafkaProducer = new FlinkKafkaProducer(
- FlowWriteConfig.FILE_DATA_SINK_KAFKA_TOPIC,
- new SimpleStringSchema(),
- createTrafficFileMetaProducerConfig(), Optional.empty());
-
- kafkaProducer.setLogFailuresOnly(false);
-
-// kafkaProducer.setWriteTimestampToKafka(true);
-
- return kafkaProducer;
- }
-
-}
diff --git a/src/main/java/com/zdjizhi/utils/kafka/TimestampDeserializationSchema.java b/src/main/java/com/zdjizhi/utils/kafka/TimestampDeserializationSchema.java
deleted file mode 100644
index 920ffab..0000000
--- a/src/main/java/com/zdjizhi/utils/kafka/TimestampDeserializationSchema.java
+++ /dev/null
@@ -1,48 +0,0 @@
-package com.zdjizhi.utils.kafka;
-
-import cn.hutool.log.Log;
-import cn.hutool.log.LogFactory;
-import com.zdjizhi.common.FlowWriteConfig;
-import com.zdjizhi.utils.JsonMapper;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.streaming.connectors.kafka.KafkaDeserializationSchema;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-
-import java.util.Map;
-
-/**
- * @author qidaijie
- * @Package com.zdjizhi.utils.kafka
- * @Description:
- * @date 2022/3/89:42
- */
-public class TimestampDeserializationSchema implements KafkaDeserializationSchema {
- private static final Log logger = LogFactory.get();
-
- @Override
- public TypeInformation getProducedType() {
- return TypeInformation.of(Map.class);
- }
-
- @Override
- public boolean isEndOfStream(Object nextElement) {
- return false;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Map deserialize(ConsumerRecord record) throws Exception {
- if (record != null) {
- try {
- long timestamp = record.timestamp() / 1000;
- String value = new String((byte[]) record.value(), FlowWriteConfig.ENCODING);
- Map json = (Map) JsonMapper.fromJsonString(value, Map.class);
- json.put("common_ingestion_time", timestamp);
- return json;
- } catch (RuntimeException e) {
- logger.error("KafkaConsumer Deserialize failed,The exception is : " + e.getMessage());
- }
- }
- return null;
- }
-}
diff --git a/src/main/java/com/zdjizhi/utils/system/FlowWriteConfigurations.java b/src/main/java/com/zdjizhi/utils/system/FlowWriteConfigurations.java
deleted file mode 100644
index 8d91027..0000000
--- a/src/main/java/com/zdjizhi/utils/system/FlowWriteConfigurations.java
+++ /dev/null
@@ -1,84 +0,0 @@
-package com.zdjizhi.utils.system;
-
-import com.alibaba.nacos.api.NacosFactory;
-import com.alibaba.nacos.api.PropertyKeyConst;
-import com.alibaba.nacos.api.config.ConfigService;
-import com.alibaba.nacos.api.exception.NacosException;
-import com.zdjizhi.common.FlowWriteConfig;
-import com.zdjizhi.utils.StringUtil;
-
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-
-
-/**
- * @author Administrator
- */
-
-public final class FlowWriteConfigurations {
-
- private static Properties propKafka = new Properties();
- private static Properties propService = new Properties();
- private static Properties propfiletype = new Properties();
- private static Map fileTypeMap;
-
- public static boolean judgeFileType(String filetype){
- return fileTypeMap.containsKey(filetype);
- }
-
- public static String getStringProperty(Integer type, String key) {
- if (type == 0) {
- return propService.getProperty(key);
- } else if (type == 1) {
- return propKafka.getProperty(key);
- } else {
- return null;
- }
- }
-
- public static Integer getIntProperty(Integer type, String key) {
- if (type == 0) {
- return Integer.parseInt(propService.getProperty(key));
- } else if (type == 1) {
- return Integer.parseInt(propKafka.getProperty(key));
- } else {
- return null;
- }
- }
-
- public static Long getLongProperty(Integer type, String key) {
- if (type == 0) {
- return Long.parseLong(propService.getProperty(key));
- } else if (type == 1) {
- return Long.parseLong(propKafka.getProperty(key));
- } else {
- return null;
- }
- }
-
- public static Boolean getBooleanProperty(Integer type, String key) {
- if (type == 0) {
- return StringUtil.equals(propService.getProperty(key).toLowerCase().trim().toUpperCase(Locale.ENGLISH), "true");
- } else if (type == 1) {
- return StringUtil.equals(propKafka.getProperty(key).toLowerCase().trim().toUpperCase(Locale.ENGLISH), "true");
- } else {
- return null;
- }
- }
-
- static {
- try {
- propService.load(FlowWriteConfigurations.class.getClassLoader().getResourceAsStream("service_flow_config.properties"));
- propKafka.load(FlowWriteConfigurations.class.getClassLoader().getResourceAsStream("default_config.properties"));
- propfiletype.load(FlowWriteConfigurations.class.getClassLoader().getResourceAsStream("file_type.properties"));
- fileTypeMap = new HashMap((Map) propfiletype);
- } catch (IOException | RuntimeException e) {
- propKafka = null;
- propService = null;
- }
- }
-}
diff --git a/src/main/log4j.properties b/src/main/log4j.properties
index 9d91936..20e542e 100644
--- a/src/main/log4j.properties
+++ b/src/main/log4j.properties
@@ -1,8 +1,8 @@
#Log4j
-log4j.rootLogger=info,console,file
+log4j.rootLogger=warn,console,file
# 控制台日志设置
log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.Threshold=info
+log4j.appender.console.Threshold=warn
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=[%d{yyyy-MM-dd HH\:mm\:ss}] [%-5p] [Thread\:%t] %l %x - <%m>%n
@@ -18,8 +18,8 @@ log4j.appender.file.layout=org.apache.log4j.PatternLayout
#log4j.appender.file.layout.ConversionPattern=%d{HH:mm:ss} %X{ip} [%t] %5p %c{1} %m%n
log4j.appender.file.layout.ConversionPattern=[%d{yyyy-MM-dd HH\:mm\:ss}] [%-5p] %X{ip} [Thread\:%t] %l %x - %m%n
#MyBatis 配置，com.nis.web.dao是mybatis接口所在包
-log4j.logger.com.nis.web.dao=debug
+log4j.logger.com.nis.web.dao=info
#bonecp数据源配置
-log4j.category.com.jolbox=debug,console
+log4j.category.com.jolbox=info,console
diff --git a/src/main/logback.xml b/src/main/logback.xml
deleted file mode 100644
index a508b6b..0000000
--- a/src/main/logback.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
- ${LOG_PATTERN}
-
-
-
-
-
-
- ${LOG_FILE_PATH}
-
- 30
-
-
- 20MB
-
-
-
-
- ${LOG_PATTERN}
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/src/test/java/com/zdjizhi/function/Base64Test.java b/src/test/java/com/zdjizhi/function/Base64Test.java
new file mode 100644
index 0000000..7282647
--- /dev/null
+++ b/src/test/java/com/zdjizhi/function/Base64Test.java
@@ -0,0 +1,75 @@
+package com.zdjizhi.function;
+
+import cn.hutool.core.codec.Base64;
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
+import org.junit.Test;
+
+import java.io.UnsupportedEncodingException;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi.function
+ * @Description:
+ * @date 2022/11/39:36
+ */
+public class Base64Test {
+ private static final Log logger = LogFactory.get();
+
+ /**
+ * 根据编码解码base64(hutool)
+
+ * @return 解码字符串
+ */
+ @Test
+ public void decodeBase64Hutool() {
+ try {
+ System.out.println(Base64.decodeStr("bWFpbF90ZXN0X2VuZ2xpc2gudHh0"));
+ System.out.println(Base64.decodeStr("aGVsbG8="));
+ } catch (RuntimeException e) {
+ logger.error("Resolve Base64 exception, exception information:" + e.getMessage());
+ e.printStackTrace();
+ }
+ }
+
+
+ /**
+ * 根据编码解码base64
+
+ * @return 解码字符串
+ */
+ @Test
+ public void encodeBase64() {
+ try {
+ System.out.println(java.util.Base64.getUrlEncoder().encodeToString("runoob?java8".getBytes("ISO-8859-1")));
+ System.out.println(java.util.Base64.getUrlEncoder().encodeToString("runoob?java8".getBytes("utf-8")));
+
+ } catch (RuntimeException e) {
+ logger.error("Resolve Base64 exception, exception information:" + e.getMessage());
+ e.printStackTrace();
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * 根据编码解码base64
+
+ * @return 解码字符串
+ */
+ @Test
+ public void decodeBase64() {
+ try {
+ byte[] base64decodedBytes = java.util.Base64.getDecoder().decode("bWFpbF90ZXN0X2VuZ2xpc2gudHh0");
+
+ System.out.println("原始字符串: " + new String(base64decodedBytes, "utf-8"));
+ System.out.println("原始字符串: " + new String(base64decodedBytes));
+
+ } catch (RuntimeException e) {
+ logger.error("Resolve Base64 exception, exception information:" + e.getMessage());
+ e.printStackTrace();
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/src/test/java/com/zdjizhi/function/EncryptorTest.java b/src/test/java/com/zdjizhi/function/EncryptorTest.java
new file mode 100644
index 0000000..dadb292
--- /dev/null
+++ b/src/test/java/com/zdjizhi/function/EncryptorTest.java
@@ -0,0 +1,53 @@
+package com.zdjizhi.function;
+
+import org.jasypt.encryption.pbe.StandardPBEStringEncryptor;
+import org.junit.Test;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi
+ * @Description:
+ * @date 2022/3/1610:55
+ */
+public class EncryptorTest {
+
+
+ @Test
+ public void passwordTest(){
+ StandardPBEStringEncryptor encryptor = new StandardPBEStringEncryptor();
+ // 配置加密解密的密码/salt值
+ encryptor.setPassword("galaxy");
+ // 对"raw_password"进行加密：S5kR+Y7CI8k7MaecZpde25yK8NKUnd6p
+ String kafkaUser = encryptor.encrypt("admin");
+ String kafkaPin = encryptor.encrypt("galaxy2019");
+ String nacosPin = encryptor.encrypt("nacos");
+ String nacosUser = encryptor.encrypt("nacos");
+
+ System.out.println("Kafka:\n"+"The username is: "+kafkaUser);
+ System.out.println("The pin is: "+kafkaPin);
+ System.out.println("Nacos:\n"+"The username is: "+nacosUser);
+ System.out.println("The pin is: "+nacosPin);
+ // 再进行解密：raw_password
+ System.out.println("Kafka:\n"+"The username is: "+encryptor.decrypt(kafkaUser));
+ System.out.println("The pin is: "+encryptor.decrypt(kafkaPin));
+
+ System.out.println("Nacos:\n"+"The username is: "+encryptor.decrypt(nacosUser));
+ System.out.println("The pin is: "+encryptor.decrypt(nacosPin));
+ System.out.println("------------------------------------------------------");
+
+ System.out.println("The vknRT6U4I739rLIha9CvojM+4uFyXZLEYpO2HZayLnRak1HPW0K2yZ3vnQBA2foo decryption result is: "+encryptor.decrypt("vknRT6U4I739rLIha9CvojM+4uFyXZLEYpO2HZayLnRak1HPW0K2yZ3vnQBA2foo"));
+ System.out.println("The SU05WiYTLLrF+cVZ410gJdrfEgxgnYG9RGsI+3dZZq54XFLOXEzOvRuMAvLlBf4k decryption result is: "+encryptor.decrypt("SU05WiYTLLrF+cVZ410gJdrfEgxgnYG9RGsI+3dZZq54XFLOXEzOvRuMAvLlBf4k"));
+ System.out.println("The qUA355VopKSx6kwwwXZwqWWEYSu76Slz decryption result is: "+encryptor.decrypt("qUA355VopKSx6kwwwXZwqWWEYSu76Slz"));
+ System.out.println("The sJ9iiTeE/7moI2hKn8asMg== decryption result is: "+encryptor.decrypt("sJ9iiTeE/7moI2hKn8asMg=="));
+ System.out.println("The 63aTpwv2vH0vPikW+3Jjig== decryption result is: "+encryptor.decrypt("63aTpwv2vH0vPikW+3Jjig=="));
+ System.out.println("The Ei1P4R1e5KTdJR+ZVnBmug== decryption result is: "+encryptor.decrypt("Ei1P4R1e5KTdJR+ZVnBmug=="));
+ System.out.println("The iW8ekP1SZC6v/7cfJKAqXXrjApJox+cH decryption result is: "+encryptor.decrypt("iW8ekP1SZC6v/7cfJKAqXXrjApJox+cH"));
+ System.out.println("The TV7Jm4dQCE/LJznp4iTm4ICkBscquv9G decryption result is: "+encryptor.decrypt("TV7Jm4dQCE/LJznp4iTm4ICkBscquv9G"));
+ System.out.println("The LDEb2OekU7iZWiFw6pUYBSozVKP27r1y decryption result is: "+encryptor.decrypt("LDEb2OekU7iZWiFw6pUYBSozVKP27r1y"));
+
+ StandardPBEStringEncryptor encryptorCM = new StandardPBEStringEncryptor();
+ encryptorCM.setPassword("bifang-api");
+ System.out.println("The Zp65tFN3CsAXwpNfgfjZ0PrquSrokM1c decryption result is: "+encryptorCM.decrypt("Zp65tFN3CsAXwpNfgfjZ0PrquSrokM1c"));
+ }
+
+}
diff --git a/src/test/java/com/zdjizhi/function/GtpcTest.java b/src/test/java/com/zdjizhi/function/GtpcTest.java
new file mode 100644
index 0000000..44f11e5
--- /dev/null
+++ b/src/test/java/com/zdjizhi/function/GtpcTest.java
@@ -0,0 +1,47 @@
+package com.zdjizhi.function;
+
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
+import com.zdjizhi.common.FlowWriteConfig;
+import com.zdjizhi.tools.connections.hbase.HBaseUtils;
+import com.zdjizhi.tools.json.JsonPathUtil;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+public class GtpcTest {
+ private static final Log logger = LogFactory.get();
+
+ @Test
+ public void gtpcMatch() {
+ String param = "$.[?(@.tunnels_schema_type=='GTP')].gtp_endpoint_a2b_teid,$.[?(@.tunnels_schema_type=='GTP')].gtp_endpoint_b2a_teid";
+ String logValue = "[{\"tunnels_schema_type\":\"GTP\",\"gtp_endpoint_a2b_teid\":4129335432,\"gtp_endpoint_b2a_teid\":4129335434,\"gtp_sgw_ip\":\"120.36.3.97\",\"gtp_pgw_ip\":\"43.224.53.100\",\"gtp_sgw_port\":2152,\"gtp_pgw_port\":51454},{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"80:69:33:ea:a5:57\",\"destination_mac\":\"14:09:dc:df:a3:40\"}]";
+ String appendToKey = "common_imsi,common_imei,common_phone_number";
+
+ try {
+ String teid = null;
+ String[] exprs = param.split(FlowWriteConfig.FORMAT_SPLITTER);
+ for (String expr : exprs) {
+ String value = JsonPathUtil.analysis(logValue, expr).toString();
+ if (value != null) {
+ teid = value;
+ break;
+ }
+ }
+ System.out.println(teid);
+ if (teid != null) {
+ String[] appendToKeys = appendToKey.split(FlowWriteConfig.FORMAT_SPLITTER);
+ HashMap userData = HBaseUtils.getGtpData(teid);
+ if (userData != null) {
+ for (String key : appendToKeys) {
+ System.out.println(userData.get(key).toString());
+ }
+ } else {
+ logger.warn("Description The user whose TEID is " + teid + " was not matched!");
+ }
+ }
+ } catch (RuntimeException re) {
+ logger.error("An exception occurred in teid type conversion or parsing of user information!" + re);
+ }
+ }
+}
diff --git a/src/test/java/com/zdjizhi/function/HBaseTest.java b/src/test/java/com/zdjizhi/function/HBaseTest.java
new file mode 100644
index 0000000..4b93e6f
--- /dev/null
+++ b/src/test/java/com/zdjizhi/function/HBaseTest.java
@@ -0,0 +1,239 @@
+package com.zdjizhi.function;
+
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
+import com.zdjizhi.common.FlowWriteConfig;
+import com.geedgenetworks.utils.StringUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi
+ * @Description:
+ * @date 2021/12/310:42
+ */
+public class HBaseTest {
+ private static final Log logger = LogFactory.get();
+ private static Map radiusMap = new ConcurrentHashMap<>(16);
+
+ private static Map> gtpcMap = new ConcurrentHashMap<>(16);
+
+ @Test
+ public void getColumn() {
+ // 管理Hbase的配置信息
+ Configuration configuration = HBaseConfiguration.create();
+ // 设置zookeeper节点
+ configuration.set("hbase.zookeeper.quorum", "192.168.44.12:2181");
+ configuration.set("hbase.client.retries.number", "1");
+ configuration.set("hbase.client.pause", "50");
+ configuration.set("hbase.rpc.timeout", "3000");
+ configuration.set("zookeeper.recovery.retry", "1");
+ configuration.set("zookeeper.recovery.retry.intervalmill", "200");
+ try {
+ System.out.println(System.currentTimeMillis());
+ Connection connection = ConnectionFactory.createConnection(configuration);
+ Table table = connection.getTable(TableName.valueOf("tsg_galaxy:relation_framedip_account"));
+ Scan scan2 = new Scan();
+ ResultScanner scanner = table.getScanner(scan2);
+ for (Result result : scanner) {
+ int acctStatusType;
+ boolean hasType = result.containsColumn(Bytes.toBytes("radius"), Bytes.toBytes("acct_status_type"));
+ if (hasType) {
+ acctStatusType = Bytes.toInt(result.getValue(Bytes.toBytes("radius"), Bytes.toBytes("acct_status_type")));
+ } else {
+ acctStatusType = 3;
+ }
+ String framedIp = Bytes.toString(result.getValue(Bytes.toBytes("radius"), Bytes.toBytes("framed_ip")));
+ String account = Bytes.toString(result.getValue(Bytes.toBytes("radius"), Bytes.toBytes("account")));
+ System.out.println("status" + acctStatusType + "key:" + framedIp + "value:" + account);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }finally {
+ System.out.println(System.currentTimeMillis());
+ }
+ }
+
+
+ @Test
+ public void getGtpcData() {
+ // 管理Hbase的配置信息
+ Configuration configuration = HBaseConfiguration.create();
+ // 设置zookeeper节点
+ configuration.set("hbase.zookeeper.quorum", "192.168.44.12:2181");
+ configuration.set("hbase.client.retries.number", "1");
+ configuration.set("hbase.client.pause", "50");
+ configuration.set("hbase.rpc.timeout", "3000");
+ configuration.set("zookeeper.recovery.retry", "1");
+ configuration.set("zookeeper.recovery.retry.intervalmill", "200");
+ long begin = System.currentTimeMillis();
+ ResultScanner scanner = null;
+ try {
+ Connection connection = ConnectionFactory.createConnection(configuration);
+ Table table = connection.getTable(TableName.valueOf(FlowWriteConfig.HBASE_GTPC_TABLE_NAME));
+ Scan scan2 = new Scan();
+ scanner = table.getScanner(scan2);
+ for (Result result : scanner) {
+ String upLinkTeid = getTeid(result, "uplink_teid");
+ String downLinkTeid = getTeid(result, "downlink_teid");
+ String phoneNumber = getString(result, FlowWriteConfig.GTPC_FAMILY_NAME, "phone_number").trim();
+ String imsi = getString(result, FlowWriteConfig.GTPC_FAMILY_NAME, "imsi").trim();
+ String imei = getString(result, FlowWriteConfig.GTPC_FAMILY_NAME, "imei").trim();
+ Long lastUpdateTime = getLong(result, FlowWriteConfig.GTPC_FAMILY_NAME, "last_update_time");
+
+ HashMap buildUserData = buildUserData(phoneNumber, imsi, imei, lastUpdateTime);
+
+ if (FlowWriteConfig.DEFAULT_RELATIONSHIP_MODULE.equals(FlowWriteConfig.DATA_RELATIONSHIP_MODEL)) {
+ String vsysId = getVsysId(result).trim();
+ updateCache(gtpcMap, upLinkTeid+vsysId, buildUserData, lastUpdateTime);
+ updateCache(gtpcMap, downLinkTeid+vsysId, buildUserData, lastUpdateTime);
+ } else {
+ updateCache(gtpcMap, upLinkTeid, buildUserData, lastUpdateTime);
+ updateCache(gtpcMap, downLinkTeid, buildUserData, lastUpdateTime);
+ }
+ }
+ logger.warn("The obtain the number of GTP-C relationships : " + gtpcMap.size());
+ logger.warn("The time spent to obtain GTP-C relationships : " + (System.currentTimeMillis() - begin));
+ } catch (IOException | RuntimeException e) {
+ logger.error("The relationship between USER and TEID obtained from HBase is abnormal! message is :" + e);
+ e.printStackTrace();
+ } finally {
+ if (scanner != null) {
+ scanner.close();
+ }
+ }
+
+ for (String key : gtpcMap.keySet()){
+ System.out.println(key +"---"+gtpcMap.get(key));
+ }
+ }
+
+ /**
+ * 获取HBase内String类型的值
+ *
+ * @param result 结果集
+ * @param familyName 列族名称
+ * @param columnName 列名称
+ * @return 结果数据
+ */
+ private static String getString(Result result, String familyName, String columnName) {
+ byte[] familyBytes = Bytes.toBytes(familyName);
+ byte[] columnBytes = Bytes.toBytes(columnName);
+ boolean contains = result.containsColumn(familyBytes, columnBytes);
+ if (contains) {
+ String data = Bytes.toString(result.getValue(familyBytes, columnBytes)).trim();
+ if (StringUtil.isNotBlank(data)) {
+ return data;
+ }
+ }
+
+ return "";
+ }
+
+ /**
+ * 获取HBase内String类型的值
+ *
+ * @param result 结果集
+ * @param columnName 列名称
+ * @return 结果数据
+ */
+ private static Long getLong(Result result, String familyName, String columnName) {
+ byte[] familyBytes = Bytes.toBytes(familyName);
+ byte[] columnBytes = Bytes.toBytes(columnName);
+ boolean contains = result.containsColumn(familyBytes, columnBytes);
+ if (contains) {
+ return Bytes.toLong(result.getValue(familyBytes, columnBytes));
+ }
+ return 0L;
+ }
+
+ /**
+ * 获取HBase内String类型的值
+ *
+ * @param result 结果集
+ * @param columnName 列名称
+ * @return 结果数据
+ */
+ private static String getTeid(Result result, String columnName) {
+ byte[] familyBytes = Bytes.toBytes(FlowWriteConfig.GTPC_FAMILY_NAME);
+ byte[] columnBytes = Bytes.toBytes(columnName);
+ boolean contains = result.containsColumn(familyBytes, columnBytes);
+ if (contains) {
+ String data = String.valueOf(Bytes.toLong(result.getValue(familyBytes, columnBytes))).trim();
+ if (StringUtil.isNotBlank(data)) {
+ return data;
+ }
+ }
+ return "0";
+ }
+
+ /**
+ * 构建用户信息
+ *
+ * @param phoneNumber 手机号
+ * @param imsi 用户标识
+ * @param imei 设备标识
+ * @return 用户信息
+ */
+ private static HashMap buildUserData(String phoneNumber, String imsi, String imei, Long lastUpdateTime) {
+ HashMap tmpMap = new HashMap<>(4);
+ tmpMap.put("common_phone_number", phoneNumber);
+ tmpMap.put("common_imsi", imsi);
+ tmpMap.put("common_imei", imei);
+ tmpMap.put("last_update_time", lastUpdateTime);
+ return tmpMap;
+ }
+
+
+ /**
+ * 获取HBase内String类型的值
+ *
+ * @param result 结果集
+ * @return 结果数据
+ */
+ static String getVsysId(Result result) {
+ byte[] familyBytes = Bytes.toBytes("common");
+ byte[] columnBytes = Bytes.toBytes("vsys_id");
+ boolean contains = result.containsColumn(familyBytes, columnBytes);
+ if (contains) {
+ String data = String.valueOf(Bytes.toInt(result.getValue(familyBytes, columnBytes))).trim();
+ if (StringUtil.isNotBlank(data)) {
+ return data;
+ }
+ }
+ return "1";
+ }
+
+ /**
+ * 判断缓存与新获取的数据时间戳大小，若大于缓存内记录的时间戳；则更新缓存
+ *
+ * @param gtpcMap 缓存集合
+ * @param key 上下行teid
+ * @param userData 获取HBase内的用户信息
+ * @param lastUpdateTime 该用户信息最后更新时间
+ */
+ private static void updateCache(Map> gtpcMap, String key, HashMap userData, Long lastUpdateTime) {
+ if (StringUtil.isNotBlank(key)){
+ if (gtpcMap.containsKey(key)) {
+ Long oldUpdateTime = Long.parseLong(gtpcMap.get(key).get("last_update_time").toString());
+ if (lastUpdateTime > oldUpdateTime) {
+ gtpcMap.put(key, userData);
+ }
+ } else {
+ gtpcMap.put(key, userData);
+ }
+ }
+ }
+
+
+}
diff --git a/src/test/java/com/zdjizhi/function/IpLookupTest.java b/src/test/java/com/zdjizhi/function/IpLookupTest.java
new file mode 100644
index 0000000..7ae311a
--- /dev/null
+++ b/src/test/java/com/zdjizhi/function/IpLookupTest.java
@@ -0,0 +1,165 @@
+package com.zdjizhi.function;
+
+import cn.hutool.core.io.IoUtil;
+import cn.hutool.core.io.file.FileReader;
+import cn.hutool.crypto.digest.DigestUtil;
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
+import com.geedgenetworks.utils.GalaxyDataBaseReader;
+import com.geedgenetworks.utils.IpLookupV2;
+import com.maxmind.db.CHMCache;
+import com.maxmind.db.Reader;
+import com.zdjizhi.common.FlowWriteConfig;
+import com.zdjizhi.tools.connections.http.HttpClientService;
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+
+import java.io.*;
+import java.net.InetAddress;
+import java.util.Map;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi
+ * @Description:
+ * @date 2021/11/611:38
+ */
+public class IpLookupTest {
+ private static final Log logger = LogFactory.get();
+ private static final String DAT_FILEPATH = "D:\\workerspace\\dat\\";
+
+
+ @Test
+ public void ipLookupDatabaseTest() {
+
+ IpLookupV2.Builder ipLookupBuilder = new IpLookupV2.Builder(false);
+
+ ipLookupBuilder.loadDataFileV4(DAT_FILEPATH + "ip_v4_built_in.mmdb");
+// ipLookupBuilder.loadDataFileV6(DAT_FILEPATH + "ip_v6_built_in.mmdb");
+// ipLookupBuilder.loadDataFilePrivateV4(DAT_FILEPATH + "ip_v4_user_defined.mmdb");
+// ipLookupBuilder.loadDataFilePrivateV6(DAT_FILEPATH + "ip_v6_user_defined.mmdb");
+// ipLookupBuilder.loadAsnDataFile(DAT_FILEPATH + "asn_v4.mmdb");
+// ipLookupBuilder.loadAsnDataFileV6(DAT_FILEPATH + "asn_v6.mmdb");
+ IpLookupV2 ipLookup = ipLookupBuilder.build();
+
+
+// String ip = "23.200.74.224";
+ String ip = "121.14.89.209";
+
+ try {
+ System.out.println(ipLookup.cityLookup(ip));
+ System.out.println(ipLookup.cityLookupDetail(ip));
+ System.out.println(ipLookup.cityLatLngLookup(ip));
+ System.out.println(ipLookup.provinceLookup(ip));
+ System.out.println(ipLookup.administrativeAreaLookupDetail(ip));
+ System.out.println(ipLookup.countryLookup(ip));
+ System.out.println(ipLookup.locationLookupDetail(ip));
+ System.out.println(ipLookup.administrativeAreaLookupDetail(ip));
+ System.out.println(ipLookup.infoLookupToJSONString(ip));
+
+ //ASN
+ System.out.println(ipLookup.asnLookup(ip));
+ System.out.println(ipLookup.asnLookupInfo(ip));
+ System.out.println(ipLookup.asnLookupDetail(ip));
+ System.out.println(ipLookup.asnLookupOrganization(ip));
+
+ } catch (NullPointerException npe) {
+ logger.error("The MMDB file is not loaded or IP is null! " + npe.getMessage());
+ } catch (RuntimeException e) {
+ logger.error("Get clientIP location error! " + e.getMessage());
+ }
+ }
+
+
+ @Test
+ public void ipLookupForBytesTest() {
+ InputStream inputStream = null;
+ try {
+ String url = "http://192.168.44.12:9098/hos/knowledge_base_hos_bucket/d2ab3313-1941-4847-84fa-5dbbd8a9007f-aXBfdjRfYnVpbHRfaW4=.mmdb";
+ HttpClientService httpClientService = new HttpClientService();
+ inputStream = httpClientService.httpGetInputStream(url, 3000);
+ Reader reader = new Reader(inputStream, new CHMCache());
+ InetAddress ipAddress = InetAddress.getByName("121.14.89.209");
+ Map map = reader.get(ipAddress, Map.class);
+ System.out.println(map.toString());
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ IoUtil.close(inputStream);
+ }
+ }
+
+
+ @Test
+ public void ipLookupInputStreamTest() {
+ InputStream asnInputStream = null;
+ InputStream ipv4InputStream = null;
+ String ip = "114.64.231.114";
+ try {
+ HttpClientService httpClientService = new HttpClientService();
+ String asnUrl = "http://192.168.44.12:9098/hos/knowledge_base_hos_bucket/1b96764c-59dd-4d6b-8edb-623705f708a5-YXNuX3Y0.mmdb";
+ String ipv4Url = "http://192.168.44.12:9098/hos/knowledge_base_hos_bucket/24cb6a74-f048-4672-988f-112858427a3b-aXBfdjRfYnVpbHRfaW4=.mmdb";
+ asnInputStream = httpClientService.httpGetInputStream(asnUrl, 3000);
+ ipv4InputStream = httpClientService.httpGetInputStream(ipv4Url, 3000);
+
+ IpLookupV2 ipLookup = new IpLookupV2.Builder(false)
+ .loadDataFileV4(ipv4InputStream)
+ .loadAsnDataFileV4(asnInputStream)
+ .build();
+
+ System.out.println(ipLookup.cityLookup(ip));
+ System.out.println(ipLookup.cityLookupDetail(ip));
+ System.out.println(ipLookup.cityLatLngLookup(ip));
+ System.out.println(ipLookup.provinceLookup(ip));
+ System.out.println(ipLookup.administrativeAreaLookupDetail(ip));
+ System.out.println(ipLookup.countryLookup(ip));
+ System.out.println(ipLookup.locationLookupDetail(ip));
+ System.out.println(ipLookup.asnLookup(ip));
+ System.out.println(ipLookup.administrativeAreaLookupDetail(ip));
+
+ } catch (NullPointerException npe) {
+ logger.error("The MMDB file is not loaded or IP is null! " + npe.getMessage());
+ } catch (RuntimeException e) {
+ logger.error("Get clientIP location error! " + e.getMessage());
+ } finally {
+ IoUtil.close(asnInputStream);
+ IoUtil.close(ipv4InputStream);
+ }
+ }
+
+
+ @Test
+ public void inputLookupTest() {
+// String ip = "121.14.89.209";
+//
+ byte[] localFile = new FileReader(DAT_FILEPATH + "built_in_ip_location.mmdb").readBytes();
+ String localFileSha256 = DigestUtil.sha256Hex(localFile);
+ System.out.println("本地文件SHA256："+localFileSha256);
+//
+// IpLookupV2 ipLookup = new IpLookupV2.Builder(false)
+// .loadDataFileV4(DAT_FILEPATH + "built_in_ip_location.mmdb").build();
+//
+// System.out.println(ipLookup.infoLookup(ip));
+
+ try {
+ HttpClientService httpClientService = new HttpClientService();
+ InputStream inputStream = httpClientService.httpGetInputStream("http://192.168.44.55:9098/hos/knowledge_base_hos_bucket/9b1ce6b4-024d-4343-80d5-6e6dc0ad0863-aXA0.mmdb", FlowWriteConfig.HTTP_SOCKET_TIMEOUT);
+ byte[] bytes = IOUtils.toByteArray(inputStream);
+ String downloadFileSha256 = DigestUtil.sha256Hex(bytes);
+ InputStream byteArrayInputStream = new ByteArrayInputStream(bytes);
+
+ System.out.println("HOS下载文件2 SHA256：" + downloadFileSha256);
+ System.out.println("HOS下载文件2 size：" + bytes.length);
+
+// IpLookupV2 ipLookup2 = new IpLookupV2.Builder(false)
+// .loadDataFileV4(byteArrayInputStream).build();
+ new GalaxyDataBaseReader.Builder(byteArrayInputStream).withCache(new CHMCache()).build();
+ InputStream inputStream1 = httpClientService.httpGetInputStream("http://192.168.44.55:9098/hos/knowledge_base_hos_bucket/9b1ce6b4-024d-4343-80d5-6e6dc0ad0863-aXA0.mmdb", FlowWriteConfig.HTTP_SOCKET_TIMEOUT);
+ new GalaxyDataBaseReader.Builder(inputStream1).withCache(new CHMCache()).build();
+// System.out.println(ipLookup2.infoLookup(ip));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+}
diff --git a/src/test/java/com/zdjizhi/function/TimestampTest.java b/src/test/java/com/zdjizhi/function/TimestampTest.java
new file mode 100644
index 0000000..7b2e513
--- /dev/null
+++ b/src/test/java/com/zdjizhi/function/TimestampTest.java
@@ -0,0 +1,89 @@
+package com.zdjizhi.function;
+
+import org.junit.Test;
+
+import java.time.*;
+import java.time.format.DateTimeFormatter;
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+
+public class TimestampTest {
+
+ @Test
+ public void timestampToDate() {
+ System.out.println(getLocalDateTime(1693905281L, 0, "Asia/Shanghai"));
+ System.out.println(getLocalDateTime(1693905281048L, 3, "Asia/Shanghai"));
+ System.out.println(getLocalDateTime(1693905281048L, 6, "Asia/Shanghai"));
+ System.out.println(getLocalDateTime(1693905281048L, 9, "UTC+03:00"));
+ System.out.println(getZoneDateTime(1693905281L, 3, "Asia/Shanghai"));
+ System.out.println(getZoneDateTime(1693905281048L, 6, "Asia/Shanghai"));
+ }
+
+
+ @Test
+ public void timestampConversion() {
+ long nanosTimestamp = 1630988475000000000L; // 纳秒级时间戳
+ System.out.println("纳秒级时间戳: " + timestampToSeconds(nanosTimestamp));
+
+ long microsTimestamp = 1630988475000000L; // 微秒级时间戳
+ System.out.println("微秒级时间戳: " + timestampToSeconds(microsTimestamp));
+
+ long millisTimestamp = 1693969952127L; // 微秒级时间戳
+ System.out.println("毫秒级时间戳: " + timestampToSeconds(millisTimestamp));
+
+ long errorTimestamp = 169396995L; // 微秒级时间戳
+ System.out.println("异常时间戳: " + timestampToSeconds(errorTimestamp));
+ }
+
+ private Long timestampToSeconds(long timestamp) {
+ int timestampLength = Long.toString(timestamp).length();
+ switch (timestampLength) {
+ case 13:
+ return TimeUnit.MILLISECONDS.toSeconds(timestamp);
+ case 16:
+ return TimeUnit.MICROSECONDS.toSeconds(timestamp);
+ case 19:
+ return TimeUnit.NANOSECONDS.toSeconds(timestamp);
+ default:
+// throw new RuntimeException("This timestamp:" + timestamp + " format is not nanosecond, microsecond, millisecond, or second");
+ return timestamp;
+ }
+ }
+
+ private String getLocalDateTime(Long timestamp, int precision, String timeZone) {
+ boolean isMillis = String.valueOf(timestamp).length() > 10;
+ String timePattern = "yyyy-MM-dd HH:mm:ss";
+ if (precision > 0) {
+ String s = String.join("", Collections.nCopies(precision, "S"));
+ timePattern = String.join(".", timePattern, s);
+ }
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern(timePattern);
+
+ Instant instant;
+ if (isMillis) {
+ instant = Instant.ofEpochMilli(timestamp);
+ } else {
+ instant = Instant.ofEpochSecond(timestamp);
+ }
+ return LocalDateTime.ofInstant(instant, ZoneId.of(timeZone)).format(formatter);
+ }
+
+
+ private String getZoneDateTime(Long timestamp, int precision, String timeZone) {
+ boolean isMillis = String.valueOf(timestamp).length() > 10;
+ String timePattern = "yyyy-MM-dd'T'HH:mm:ss.";
+ if (precision > 0) {
+ String s = String.join("", Collections.nCopies(precision, "S"));
+ timePattern = String.join("", timePattern, s, "XXX");
+ }
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern(timePattern);
+
+ Instant instant;
+ if (isMillis) {
+ instant = Instant.ofEpochMilli(timestamp);
+ } else {
+ instant = Instant.ofEpochSecond(timestamp);
+ }
+ return ZonedDateTime.ofInstant(instant, ZoneId.of(timeZone)).format(formatter);
+ }
+}
diff --git a/src/test/java/com/zdjizhi/function/TopDomainTest.java b/src/test/java/com/zdjizhi/function/TopDomainTest.java
new file mode 100644
index 0000000..4502310
--- /dev/null
+++ b/src/test/java/com/zdjizhi/function/TopDomainTest.java
@@ -0,0 +1,39 @@
+package com.zdjizhi.function;
+
+import com.google.common.net.InternetDomainName;
+import com.geedgenetworks.utils.FormatUtils;
+import org.junit.Test;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi.function
+ * @Description:
+ * @date 2023/7/2413:55
+ */
+public class TopDomainTest {
+
+
+ @Test
+ public void getTopDomainTest() {
+ String host = "heartsofsteel-.tumblr.com";
+ System.out.println(FormatUtils.getTopPrivateDomain(host));
+ host = "heartsofsteel.tumblr.com";
+ System.out.println(FormatUtils.getTopPrivateDomain(host));
+
+
+// String sni = "www.googleapis.com";
+ String sni = "juicebox-.tumblr.com";
+ String domain = FormatUtils.getDomain(sni);
+ System.out.println(domain);
+ System.out.println(FormatUtils.getTopPrivateDomain(sni));
+
+ System.out.println(InternetDomainName.isValid(sni));
+ System.out.println(InternetDomainName.isValid(domain));
+
+ InternetDomainName internetDomainName = InternetDomainName.from(domain);
+ if (internetDomainName.isUnderPublicSuffix()) {
+ System.out.println(internetDomainName.topPrivateDomain().toString());
+ }
+ }
+
+}
diff --git a/src/test/java/com/zdjizhi/hdfs/FileUtilsTest.java b/src/test/java/com/zdjizhi/hdfs/FileUtilsTest.java
new file mode 100644
index 0000000..bd5a33d
--- /dev/null
+++ b/src/test/java/com/zdjizhi/hdfs/FileUtilsTest.java
@@ -0,0 +1,75 @@
+package com.zdjizhi.hdfs;
+
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
+import com.zdjizhi.common.FlowWriteConfig;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi.tools.connections.hadoop
+ * @Description:
+ * @date 2022/11/217:57
+ */
+public class FileUtilsTest {
+ private static final Log logger = LogFactory.get();
+
+ private static FileSystem fileSystem;
+
+ static {
+ Configuration configuration = new Configuration();
+ try {
+ configuration.set("fs.defaultFS","hdfs://ns1");
+ configuration.set("hadoop.proxyuser.root.hosts","*");
+ configuration.set("hadoop.proxyuser.root.groups","*");
+ configuration.set("ha.zookeeper.quorum","192.168.44.83:2181,192.168.44.84:2181,192.168.44.85:2181");
+ configuration.set("dfs.nameservices","ns1");
+ configuration.set("dfs.ha.namenodes.ns1","nn1,nn2");
+ configuration.set("dfs.namenode.rpc-address.ns1.nn1","192.168.44.85:9000");
+ configuration.set("dfs.namenode.rpc-address.ns1.nn2","192.168.44.86:9000");
+ configuration.set("dfs.client.failover.proxy.provider.ns1","org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
+ //创建fileSystem,用于连接hdfs
+ fileSystem = FileSystem.get(configuration);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void mkdir() throws Exception{
+ fileSystem.mkdirs(new Path("/knowledgebase/test"));
+ }
+
+ @Test
+ public void create() throws Exception{
+ FSDataOutputStream outputStream = fileSystem.create(new Path("/knowledgebase/test/test.txt"));
+ outputStream.write("Hello World".getBytes());
+ outputStream.flush();
+ outputStream.close();
+ }
+
+ @Test
+ public void cat() throws Exception{
+ FSDataInputStream inputStream = fileSystem.open(new Path("/knowledgebase/test/test.txt"));
+ IOUtils.copyBytes(inputStream, System.out, 1024);
+ inputStream.close();
+ }
+
+ @Test
+ public void rename() throws Exception{
+ fileSystem.rename(new Path("/knowledgebase/test/test.txt"), new Path("/knowledgebase/test/test1.txt"));
+ }
+
+ @Test
+ public void delete() throws Exception{
+ fileSystem.delete(new Path("/knowledgebase/test"),true);//是否递归删除
+ }
+}
+
diff --git a/src/test/java/com/zdjizhi/hos/hosUtilsTest.java b/src/test/java/com/zdjizhi/hos/hosUtilsTest.java
new file mode 100644
index 0000000..1459773
--- /dev/null
+++ b/src/test/java/com/zdjizhi/hos/hosUtilsTest.java
@@ -0,0 +1,100 @@
+package com.zdjizhi.hos;
+
+import cn.hutool.core.io.IoUtil;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Joiner;
+import com.maxmind.db.CHMCache;
+import com.maxmind.db.Reader;
+import com.zdjizhi.common.FlowWriteConfig;
+import com.zdjizhi.tools.connections.http.HttpClientService;
+import com.geedgenetworks.utils.StringUtil;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.util.Map;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi.hos
+ * @Description:
+ * @date 2022/11/713:55
+ */
+public class hosUtilsTest {
+
+ @Test
+ public void downloadToLocalTest() {
+ FileOutputStream outputStream = null;
+ InputStream inputStream = null;
+ try {
+ String url = "http://192.168.44.242:9098/hos/knowledge_base_hos_bucket/0773846a-87df-4869-a5b7-815eda384b92-aXBfdjRfYnVpbHRfaW4=.mmdb";
+
+ HttpClientService httpClientService = new HttpClientService();
+ inputStream = httpClientService.httpGetInputStream(url, 3000);
+
+ File file = new File(FlowWriteConfig.TOOLS_LIBRARY.concat(File.separator).concat("ip_v4_built_in.mmdb"));
+ if (!file.getParentFile().exists()) {
+ file.getParentFile().mkdir();
+ }
+ outputStream = new FileOutputStream(file);
+ IoUtil.copy(inputStream, outputStream);
+ } catch (IOException | RuntimeException e) {
+ e.printStackTrace();
+ } finally {
+ IoUtil.close(inputStream);
+ IoUtil.close(outputStream);
+ }
+ }
+
+
+ @Test
+ public void locationTest() {
+ InputStream inputStream = null;
+ try {
+// Header header = new BasicHeader("token", FlowWriteConfig.HOS_TOKEN);
+ String url = "http://192.168.44.12:9098/hos/knowledge_base_hos_bucket/ac0ef83d-f23a-41a0-9c66-51c4e2b4cbc0-aXBfdjRfYnVpbHRfaW4=.mmdb";
+ HttpClientService httpClientService = new HttpClientService();
+ inputStream = httpClientService.httpGetInputStream(url, 3000);
+ Reader reader = new Reader(inputStream, new CHMCache());
+ InetAddress ipAddress = InetAddress.getByName("114.64.231.114");
+ Map map = reader.get(ipAddress, Map.class);
+ if (map != null) {
+ System.out.println(StringUtil.setDefaultIfEmpty(map.toString(), "unkonw").toString());
+ System.out.println(Joiner.on(".").useForNull("").join(map.get("COUNTRY"),
+ map.get("SUPER_ADMINISTRATIVE_AREA"), map.get("ADMINISTRATIVE_AREA")));
+ System.out.println(Joiner.on(".").useForNull("").join(map.get("COUNTRY"),
+ map.get("SUPER_ADMINISTRATIVE_AREA"), map.get("ADMINISTRATIVE_AREA")).replace("\"", ""));
+ }
+ } catch (IOException | RuntimeException e) {
+ e.printStackTrace();
+ } finally {
+ IoUtil.close(inputStream);
+ }
+ }
+
+ @Test
+ public void asnTest() {
+ InputStream inputStream = null;
+ try {
+ String url = "http://192.168.44.12:9098/hos/knowledge_base_hos_bucket/95b20b83-e6bd-4d28-85d5-3b4e32da9a3f-aXBfdjRfYnVpbHRfaW4=.mmdb";
+ HttpClientService httpClientService = new HttpClientService();
+ inputStream = httpClientService.httpGetInputStream(url, 3000);
+ Reader reader = new Reader(inputStream, new CHMCache());
+ InetAddress ipAddress = InetAddress.getByName("23.200.74.224");
+ Map map = reader.get(ipAddress, Map.class);
+ if (map != null) {
+ System.out.println(StringUtil.setDefaultIfEmpty(map.get("ASN"), "unkonw").toString());
+ System.out.println(StringUtil.setDefaultIfEmpty(map.get("ASN"), "unkonw").toString().replace("\"", ""));
+ System.out.println(StringUtil.setDefaultIfEmpty(map.toString(), "unkonw").toString());
+ }
+ } catch (IOException | RuntimeException e) {
+ e.printStackTrace();
+ } finally {
+ IoUtil.close(inputStream);
+ }
+ }
+
+}
diff --git a/src/test/java/com/zdjizhi/json/FastJsonTest.java b/src/test/java/com/zdjizhi/json/FastJsonTest.java
new file mode 100644
index 0000000..13c8688
--- /dev/null
+++ b/src/test/java/com/zdjizhi/json/FastJsonTest.java
@@ -0,0 +1,98 @@
+package com.zdjizhi.json;
+
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONArray;
+import com.alibaba.fastjson2.JSONObject;
+import com.zdjizhi.json.pojo.UserList;
+import com.zdjizhi.json.pojo.UserMap;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi.json
+ * @Description:
+ * @date 2023/5/2014:02
+ */
+public class FastJsonTest {
+
+ @Test
+ public void pojoTest() {
+ //all right
+ String message = "{\"name\":\"zhangsan\",\"age\":50,\"idcard\":\"140303199999999999\",\"previousaddress\":{\"first\":\"北京\",\"second\":\"上海\"}}";
+ //int error (yes)
+// String message = "{\"name\":\"zhangsan\",\"age\":\"123\",\"idCard\":\"140303199999999999\",\"previousaddress\":{\"first\":\"北京\",\"second\":\"上海\"}}";
+ //string error (yes)
+// String message = "{\"name\":123,\"age\":123,\"idCard\":\"140303199999999999\",\"previousaddress\":{\"first\":\"北京\",\"second\":\"上海\"}}";
+ //json error (no)
+// String message = "{\"name\":\"zhangsan\",\"age\":50,\"idCard\":\"140303199999999999\",\"previousaddress\":\"{\\\"first\\\":\\\"北京\\\",\\\"second\\\":\\\"上海\\\"}\"}";
+
+ UserMap user = JSON.parseObject(message, UserMap.class);
+ System.out.println(user.getName());
+ System.out.println(user.getDevicetag());
+ System.out.println(JSON.toJSONString(user));
+ }
+
+ @Test
+ public void typeCheckTest() {
+ //jsonobject
+ String message = "{\"name\":\"zhangsan\",\"age\":50,\"idcard\":\"123456789\",\"devicetag\":\"{\\\"tags\\\":[{\\\"tag\\\":\\\"group\\\",\\\"value\\\":\\\"7400\\\"},{\\\"tag\\\":\\\"center\\\",\\\"value\\\":\\\"7400\\\"}]}\"}";
+ JSONObject objectTest = JSONObject.parseObject(message);
+
+ for (Map.Entry entry : objectTest.entrySet()) {
+ System.out.println("key:" + entry.getKey() + "————value:" + entry.getValue() + "————class: " + entry.getValue().getClass());
+ }
+ System.out.println("\n输出原始日志：" + objectTest.toString());
+
+ Object previousMap = objectTest.get("devicetag");
+ if (previousMap.getClass() != Map.class) {
+ JSONObject previousObject = JSONObject.parseObject(previousMap.toString());
+ objectTest.put("devicetag", previousObject);
+ }
+ System.out.println("输出转换map类型后的日志：" + objectTest.toString());
+
+ UserMap userMap = objectTest.toJavaObject(UserMap.class);
+ System.out.println(JSON.toJSONString(userMap));
+
+ System.out.println("\n-----------------------------------------------\n");
+
+ //jsonarray
+ message = "{\"name\":\"zhangsan\",\"age\":50,\"idcard\":\"123456789\",\"devicetag\":\"[{\\\"tag\\\":\\\"group\\\",\\\"value\\\":\\\"7400\\\"},{\\\"tag\\\":\\\"center\\\",\\\"value\\\":\\\"7400\\\"}]\"}";
+ JSONObject arrayTest = JSONObject.parseObject(message);
+ for (Map.Entry entry : arrayTest.entrySet()) {
+ System.out.println("key:" + entry.getKey() + "————value:" + entry.getValue() + "————class: " + entry.getValue().getClass());
+ }
+ System.out.println("\n输出原始日志：" + arrayTest.toString());
+
+ Object previousList = arrayTest.get("devicetag");
+ if (previousList.getClass() != List.class) {
+ JSONArray previousArray = JSONArray.parseArray(previousList.toString());
+ arrayTest.put("devicetag", previousArray);
+ }
+ System.out.println("输出转换list类型后的日志：" + arrayTest.toString());
+
+ UserList userList = arrayTest.toJavaObject(UserList.class);
+ System.out.println(JSON.toJSONString(userList));
+
+ }
+
+ @Test
+ public void typeTest() {
+ String message = "{\"name\":\"zhangsan\",\"age\":12,\"object\":{\"name\":\"a\",\"age\":12},\"array\":[{\"one\":1},{\"two\":2}]}";
+ JSONObject objectTest = JSONObject.parseObject(message);
+
+ for (Map.Entry entry : objectTest.entrySet()) {
+ String key = entry.getKey();
+ Class aClass = entry.getValue().getClass();
+
+ System.out.println(key + "---------" + aClass.getSimpleName());
+ }
+ Object bbb = objectTest.get("bbb");
+ if (bbb == null){
+ System.out.println("null");
+ }
+ }
+
+}
diff --git a/src/test/java/com/zdjizhi/json/JsonPathTest.java b/src/test/java/com/zdjizhi/json/JsonPathTest.java
new file mode 100644
index 0000000..64f7350
--- /dev/null
+++ b/src/test/java/com/zdjizhi/json/JsonPathTest.java
@@ -0,0 +1,102 @@
+package com.zdjizhi.json;
+
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
+import com.alibaba.fastjson2.*;
+import com.alibaba.nacos.api.NacosFactory;
+import com.alibaba.nacos.api.PropertyKeyConst;
+import com.alibaba.nacos.api.config.ConfigService;
+import com.alibaba.nacos.api.exception.NacosException;
+import com.zdjizhi.common.CommonConfig;
+import com.zdjizhi.common.FlowWriteConfig;
+import org.junit.Test;
+
+import java.util.Properties;
+
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi.json
+ * @Description:
+ * @date 2022/3/2410:22
+ */
+public class JsonPathTest {
+
+ private static Properties properties = new Properties();
+
+
+ static {
+ properties.setProperty(PropertyKeyConst.SERVER_ADDR, "192.168.44.12:8848");
+ properties.setProperty(PropertyKeyConst.USERNAME, "nacos");
+ properties.setProperty(PropertyKeyConst.PASSWORD, "nacos");
+ }
+
+ @Test
+ public void konwledgeBaseTest() {
+ try {
+ ConfigService configService = NacosFactory.createConfigService(properties);
+ String schema = configService.getConfig("knowledge_base.json", "DEFAULT_GROUP", 5000);
+// String KNOWLEDGE_EXPR = "[?(@.version=='latest' && (@.name in ('ip_v4_built_in','ip_v6_built_in','ip_v4_user_defined','ip_v6_user_defined','asn_v4','asn_v6')))]";
+// String KNOWLEDGE_EXPR = "[?(@.version=='latest' && (@.type in ('ip_location','asn','ip')))]";
+// String KNOWLEDGE_EXPR = "[?(@.version=='latest' && (@.name in ('ip_v4_built_in','ip_v6_built_in','ip_v4_user_defined','ip_v6_user_defined','asn_v4','asn_v6')) && (@.type in ('ip_location','asn','ip')))]";
+// String types = "[?(@.type in ('ip_location','asn','ip_user_defined'))]";
+// String names = "[?(@.name in ('ip_v4_built_in','ip_v6_built_in','asn_v4','asn_v6','内置IP定位库'))]";
+// String expr = "[?(@.version=='latest')][?(@.type in ('ip_location','asn','ip_user_defined'))]";
+
+// String expr = "[?(@.version=='latest')][?(@.name == 'QQQ' || (@.type == 'ip_user_defined'))]";
+ String expr = "[?(@.version=='latest')][@.type in ('ip_location','asn','ip')][?(@.name in ('QQQ'))]";
+
+// JSONPath jsonPath = JSONPath.of(combinationFilterList());
+ JSONPath jsonPath = JSONPath.of(expr);
+
+ String extract = jsonPath.extract(JSONReader.of(schema)).toString();
+
+
+ JSONArray jsonArray = JSON.parseArray(extract);
+ for (int i = 0; i < jsonArray.size(); i++) {
+ System.out.println(jsonArray.getString(i));
+// KnowlegeBaseMeta knowlegeBaseMeta = JSONObject.parseObject(jsonArray.getString(i), KnowlegeBaseMeta.class);
+// System.out.println(knowlegeBaseMeta.toString());
+ }
+
+ } catch (NacosException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public static String combinationFilterList() {
+ String[] typeList = CommonConfig.KNOWLEDGEBASE_TYPE_LIST.split(",");
+ String[] nameList = CommonConfig.KNOWLEDGEBASE_NAME_LIST.split(",");
+ String expr = "[?(@.version=='latest')]";
+ // ip_location > 'ip_location'
+
+ if (typeList.length > 1) {
+ StringBuilder typeBuilder = new StringBuilder();
+ typeBuilder.append("[?(@.type in (");
+ for (int i = 0; i < typeList.length; i++) {
+ if (i == typeList.length - 1) {
+ typeBuilder.append("'").append(typeList[i]).append("'))]");
+ } else {
+ typeBuilder.append("'").append(typeList[i]).append("',");
+ }
+ }
+ expr = expr + typeBuilder.toString();
+ }
+
+ if (nameList.length > 1) {
+ StringBuilder nameBuilder = new StringBuilder();
+ nameBuilder.append("[?(@.name in (");
+ for (int i = 0; i < nameList.length; i++) {
+ if (i == nameList.length - 1) {
+ nameBuilder.append("'").append(nameList[i]).append("'))]");
+ } else {
+ nameBuilder.append("'").append(nameList[i]).append("',");
+ }
+ }
+ expr = expr + nameBuilder.toString();
+ }
+
+ return expr;
+ }
+
+}
diff --git a/src/test/java/com/zdjizhi/json/NewSchemaTest.java b/src/test/java/com/zdjizhi/json/NewSchemaTest.java
new file mode 100644
index 0000000..c3cf1be
--- /dev/null
+++ b/src/test/java/com/zdjizhi/json/NewSchemaTest.java
@@ -0,0 +1,223 @@
+package com.zdjizhi.json;
+
+import com.alibaba.fastjson2.JSONArray;
+import com.alibaba.fastjson2.JSONObject;
+import com.alibaba.nacos.api.NacosFactory;
+import com.alibaba.nacos.api.PropertyKeyConst;
+import com.alibaba.nacos.api.config.ConfigService;
+import com.alibaba.nacos.api.exception.NacosException;
+import com.zdjizhi.common.FlowWriteConfig;
+import org.junit.Test;
+
+import java.util.*;
+
+/**
+ * Applicable to schemas >= TSG22.08
+ *
+ * @author qidaijie
+ * @Package com.zdjizhi.nacos
+ * @Description:
+ * @date 2022/3/1714:57
+ */
+public class NewSchemaTest {
+
+ private static Properties properties = new Properties();
+
+ /**
+ * 获取需要删除字段的列表
+ */
+ private static ArrayList dropList = new ArrayList<>();
+
+ /**
+ * 获取schema指定的有效字段及类型
+ */
+ private static HashMap jsonFieldsMap;
+
+ /**
+ * 获取包含默认值的字段
+ */
+ private static HashMap defaultFieldsMap = new HashMap<>(16);
+
+
+
+
+ static {
+ properties.setProperty(PropertyKeyConst.SERVER_ADDR, "192.168.44.11:8848");
+ properties.setProperty(PropertyKeyConst.NAMESPACE, "f507879a-8b1b-4330-913e-83d4fcdc14bb");
+ properties.setProperty(PropertyKeyConst.USERNAME, "nacos");
+ properties.setProperty(PropertyKeyConst.PASSWORD, "nacos");
+ }
+
+
+ @Test
+ public void newSchemaTest() {
+ try {
+
+ ConfigService configService = NacosFactory.createConfigService(properties);
+ String dataId = "session_record.json";
+ String group = "Galaxy";
+ String schema = configService.getConfig(dataId, group, 5000);
+
+ ArrayList newJobList = getNewJobList(schema);
+ for (String[] job : newJobList) {
+ System.out.println(Arrays.toString(job));
+ }
+
+ HashMap fieldsFromSchema = getFieldsFromSchema(schema);
+ for (String key : fieldsFromSchema.keySet()) {
+ System.out.println("fileName:" + key + " Class:" + fieldsFromSchema.get(key));
+
+ }
+ } catch (NacosException e) {
+ e.printStackTrace();
+ }
+ }
+
+
+ /**
+ * 模式匹配，给定一个类型字符串返回一个类类型
+ *
+ * @param type 类型
+ * @return 类类型
+ */
+
+ private static Class getClassName(String type) {
+ Class clazz;
+
+ switch (type) {
+ case "int":
+ clazz = Integer.class;
+ break;
+ case "string":
+ clazz = String.class;
+ break;
+ case "long":
+ clazz = long.class;
+ break;
+ case "array":
+ clazz = List.class;
+ break;
+ case "double":
+ clazz = double.class;
+ break;
+ case "float":
+ clazz = float.class;
+ break;
+ case "char":
+ clazz = char.class;
+ break;
+ case "byte":
+ clazz = byte.class;
+ break;
+ case "boolean":
+ clazz = boolean.class;
+ break;
+ case "short":
+ clazz = short.class;
+ break;
+ default:
+ clazz = String.class;
+ }
+ return clazz;
+ }
+
+ /**
+ * 通过schema来获取所需的字段及字段类型。
+ *
+ * @return 用于反射生成schema类型的对象的一个map集合
+ */
+ private static HashMap getFieldsFromSchema(String schema) {
+ HashMap map = new HashMap<>(256);
+
+ //获取fields，并转化为数组，数组的每个元素都是一个name doc type
+ com.alibaba.fastjson2.JSONObject schemaJson = com.alibaba.fastjson2.JSONObject.parseObject(schema);
+ com.alibaba.fastjson2.JSONArray fields = schemaJson.getJSONArray("fields");
+
+ for (Object field : fields) {
+ String filedStr = field.toString();
+ if (checkKeepField(filedStr)) {
+ com.alibaba.fastjson2.JSONObject fieldJson = com.alibaba.fastjson2.JSONObject.parseObject(filedStr);
+ String name = fieldJson.getString("name");
+ String type = fieldJson.getString("type");
+ if (type.contains("{")) {
+ com.alibaba.fastjson2.JSONObject types = com.alibaba.fastjson2.JSONObject.parseObject(type);
+ type = types.getString("type");
+ }
+
+ if (fieldJson.containsKey("default")) {
+ defaultFieldsMap.put(name, fieldJson.get("default"));
+ }
+ //组合用来生成实体类的map
+ map.put(name, getClassName(type));
+ } else {
+ dropList.add(filedStr);
+ }
+ }
+ return map;
+ }
+
+ /**
+ * 判断字段是否需要保留
+ *
+ * @param message 单个field-json
+ * @return true or false
+ */
+ private static boolean checkKeepField(String message) {
+ boolean isKeepField = true;
+ com.alibaba.fastjson2.JSONObject fieldJson = com.alibaba.fastjson2.JSONObject.parseObject(message);
+ boolean hasDoc = fieldJson.containsKey("doc");
+ if (hasDoc) {
+ com.alibaba.fastjson2.JSONObject doc = com.alibaba.fastjson2.JSONObject.parseObject(fieldJson.getString("doc"));
+ if (doc.containsKey("visibility")) {
+ String visibility = doc.getString("visibility");
+ if ("disabled".equals(visibility)) {
+ isKeepField = false;
+ }
+ }
+ }
+ return isKeepField;
+ }
+
+
+ /**
+ * 根据http链接获取schema，解析之后返回一个任务列表 (useList toList funcList paramlist)
+ *
+ * @return 任务列表
+ */
+ private static ArrayList getNewJobList(String schema) {
+ ArrayList list = new ArrayList<>();
+
+ JSONObject schemaJson = JSONObject.parseObject(schema);
+ JSONArray fields = schemaJson.getJSONArray("fields");
+ for (Object field : fields) {
+ JSONObject fieldJson = JSONObject.parseObject(field.toString());
+ boolean hasDoc = fieldJson.containsKey("doc");
+ if (hasDoc) {
+ JSONObject docJson = JSONObject.parseObject(fieldJson.getString("doc"));
+ boolean hasFormat = docJson.containsKey("format");
+ if (hasFormat) {
+ String name = fieldJson.getString("name");
+ JSONArray formatList = docJson.getJSONArray("format");
+ for (Object format : formatList) {
+ JSONObject formatJson = JSONObject.parseObject(format.toString());
+ String function = formatJson.getString("function");
+ String appendTo;
+ String params = null;
+
+ if (formatJson.containsKey("appendTo")) {
+ appendTo = formatJson.getString("appendTo");
+ } else {
+ appendTo = name;
+ }
+ if (formatJson.containsKey("param")) {
+ params = formatJson.getString("param");
+ }
+ list.add(new String[]{name, appendTo, function, params});
+ }
+ }
+ }
+ }
+ return list;
+ }
+
+}
diff --git a/src/test/java/com/zdjizhi/json/OldSchemaTest.java b/src/test/java/com/zdjizhi/json/OldSchemaTest.java
new file mode 100644
index 0000000..546a157
--- /dev/null
+++ b/src/test/java/com/zdjizhi/json/OldSchemaTest.java
@@ -0,0 +1,121 @@
+package com.zdjizhi.json;
+
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.nacos.api.NacosFactory;
+import com.alibaba.nacos.api.PropertyKeyConst;
+import com.alibaba.nacos.api.config.ConfigService;
+import com.alibaba.nacos.api.exception.NacosException;
+import com.zdjizhi.common.FlowWriteConfig;
+import com.geedgenetworks.utils.StringUtil;
+import org.junit.Test;
+
+import java.util.*;
+
+/**
+ * Applicable to schemas < TSG22.08
+ *
+ * @author qidaijie
+ * @Package com.zdjizhi.nacos
+ * @Description:
+ * @date 2022/3/1714:57
+ */
+public class OldSchemaTest {
+
+ private static Properties properties = new Properties();
+
+
+ static {
+ properties.setProperty(PropertyKeyConst.SERVER_ADDR, "192.168.44.12:8848");
+ properties.setProperty(PropertyKeyConst.NAMESPACE, "prod");
+ properties.setProperty(PropertyKeyConst.USERNAME, "nacos");
+ properties.setProperty(PropertyKeyConst.PASSWORD, "nacos");
+ }
+
+
+ @Test
+ public void oldSchemaTest() {
+
+ try {
+ ConfigService configService = NacosFactory.createConfigService(properties);
+ String dataId = "session_record.json";
+ String group = "Galaxy";
+ String schema = configService.getConfig(dataId, group, 5000);
+
+ ArrayList oldJobList = getOldJobList(schema);
+
+ for (String[] job : oldJobList) {
+ System.out.println(Arrays.toString(job));
+ }
+
+ } catch (NacosException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * 解析schema，解析之后返回一个任务列表 (useList toList funcList paramlist)
+ *
+ * @param schema 日志schema
+ * @return 任务列表
+ */
+ private static ArrayList getOldJobList(String schema) {
+ ArrayList list = new ArrayList<>();
+
+ //获取fields，并转化为数组，数组的每个元素都是一个name doc type
+ JSONObject schemaJson = JSON.parseObject(schema);
+ JSONArray fields = (JSONArray) schemaJson.get("fields");
+
+ for (Object field : fields) {
+
+ if (JSON.parseObject(field.toString()).containsKey("doc")) {
+ Object doc = JSON.parseObject(field.toString()).get("doc");
+
+ if (JSON.parseObject(doc.toString()).containsKey("format")) {
+ String name = JSON.parseObject(field.toString()).get("name").toString();
+ Object format = JSON.parseObject(doc.toString()).get("format");
+ JSONObject formatObject = JSON.parseObject(format.toString());
+
+ String functions = formatObject.get("transform").toString();
+ String appendTo = null;
+ String params = null;
+
+ if (formatObject.containsKey("appendTo")) {
+ appendTo = formatObject.get("appendTo").toString();
+ }
+
+ if (formatObject.containsKey("param")) {
+ params = formatObject.get("param").toString();
+ }
+
+
+ if (StringUtil.isNotBlank(appendTo) && StringUtil.isBlank(params)) {
+ String[] functionArray = functions.split(FlowWriteConfig.FORMAT_SPLITTER);
+ String[] appendToArray = appendTo.split(FlowWriteConfig.FORMAT_SPLITTER);
+
+ for (int i = 0; i < functionArray.length; i++) {
+ list.add(new String[]{name, appendToArray[i], functionArray[i], null});
+ }
+
+ } else if (StringUtil.isNotBlank(appendTo) && StringUtil.isNotBlank(params)) {
+ String[] functionArray = functions.split(FlowWriteConfig.FORMAT_SPLITTER);
+ String[] appendToArray = appendTo.split(FlowWriteConfig.FORMAT_SPLITTER);
+ String[] paramArray = params.split(FlowWriteConfig.FORMAT_SPLITTER);
+
+ for (int i = 0; i < functionArray.length; i++) {
+ list.add(new String[]{name, appendToArray[i], functionArray[i], paramArray[i]});
+
+ }
+ } else {
+ list.add(new String[]{name, name, functions, params});
+ }
+
+ }
+ }
+
+ }
+ return list;
+ }
+}
diff --git a/src/test/java/com/zdjizhi/json/pojo/KnowlegeBaseMeta.java b/src/test/java/com/zdjizhi/json/pojo/KnowlegeBaseMeta.java
new file mode 100644
index 0000000..7afb87c
--- /dev/null
+++ b/src/test/java/com/zdjizhi/json/pojo/KnowlegeBaseMeta.java
@@ -0,0 +1,63 @@
+package com.zdjizhi.json.pojo;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi.json
+ * @Description:
+ * @date 2023/5/1918:42
+ */
+public class KnowlegeBaseMeta {
+ private String name;
+ private String sha256;
+ private String format;
+ private String path;
+
+ public KnowlegeBaseMeta(String name, String sha256, String format, String path) {
+ this.name = name;
+ this.sha256 = sha256;
+ this.format = format;
+ this.path = path;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getSha256() {
+ return sha256;
+ }
+
+ public void setSha256(String sha256) {
+ this.sha256 = sha256;
+ }
+
+ public String getFormat() {
+ return format;
+ }
+
+ public void setFormat(String format) {
+ this.format = format;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ @Override
+ public String toString() {
+ return "KnowlegeBaseMeta{" +
+ "name='" + name + '\'' +
+ ", sha256='" + sha256 + '\'' +
+ ", format='" + format + '\'' +
+ ", path='" + path + '\'' +
+ '}';
+ }
+}
diff --git a/src/test/java/com/zdjizhi/json/pojo/UserList.java b/src/test/java/com/zdjizhi/json/pojo/UserList.java
new file mode 100644
index 0000000..fe7631c
--- /dev/null
+++ b/src/test/java/com/zdjizhi/json/pojo/UserList.java
@@ -0,0 +1,56 @@
+package com.zdjizhi.json.pojo;
+
+
+import java.util.List;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi.json.pojo
+ * @Description:
+ * @date 2023/5/2014:06
+ */
+public class UserList {
+ private String name;
+ private Integer age;
+ private Long idcard;
+ private List devicetag;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Integer getAge() {
+ return age;
+ }
+
+ public void setAge(Integer age) {
+ this.age = age;
+ }
+
+ public Long getIdcard() {
+ return idcard;
+ }
+
+ public void setIdcard(Long idcard) {
+ this.idcard = idcard;
+ }
+
+ public List getDevicetag() {
+ return devicetag;
+ }
+
+ public void setDevicetag(List devicetag) {
+ this.devicetag = devicetag;
+ }
+
+ public UserList(String name, Integer age, Long idcard, List devicetag) {
+ this.name = name;
+ this.age = age;
+ this.idcard = idcard;
+ this.devicetag = devicetag;
+ }
+}
diff --git a/src/test/java/com/zdjizhi/json/pojo/UserMap.java b/src/test/java/com/zdjizhi/json/pojo/UserMap.java
new file mode 100644
index 0000000..26dae2c
--- /dev/null
+++ b/src/test/java/com/zdjizhi/json/pojo/UserMap.java
@@ -0,0 +1,55 @@
+package com.zdjizhi.json.pojo;
+
+import java.util.Map;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi.json.pojo
+ * @Description:
+ * @date 2023/5/2014:06
+ */
+public class UserMap {
+ private String name;
+ private Integer age;
+ private Long idcard;
+ private Map devicetag;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Integer getAge() {
+ return age;
+ }
+
+ public void setAge(Integer age) {
+ this.age = age;
+ }
+
+ public Long getIdcard() {
+ return idcard;
+ }
+
+ public void setIdcard(Long idcard) {
+ this.idcard = idcard;
+ }
+
+ public Map getDevicetag() {
+ return devicetag;
+ }
+
+ public void setDevicetag(Map devicetag) {
+ this.devicetag = devicetag;
+ }
+
+ public UserMap(String name, Integer age, Long idcard, Map devicetag) {
+ this.name = name;
+ this.age = age;
+ this.idcard = idcard;
+ this.devicetag = devicetag;
+ }
+}
diff --git a/src/test/java/com/zdjizhi/nacos/NacosTest.java b/src/test/java/com/zdjizhi/nacos/NacosTest.java
new file mode 100644
index 0000000..0cfe883
--- /dev/null
+++ b/src/test/java/com/zdjizhi/nacos/NacosTest.java
@@ -0,0 +1,102 @@
+package com.zdjizhi.nacos;
+
+import com.alibaba.nacos.api.NacosFactory;
+import com.alibaba.nacos.api.PropertyKeyConst;
+import com.alibaba.nacos.api.config.ConfigService;
+import com.alibaba.nacos.api.config.listener.Listener;
+import com.alibaba.nacos.api.exception.NacosException;
+import com.alibaba.nacos.client.config.impl.LocalConfigInfoProcessor;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Properties;
+import java.util.concurrent.Executor;
+
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi
+ * @Description:
+ * @date 2022/3/1016:58
+ */
+public class NacosTest {
+
+ /**
+ *
+ * com.alibaba.nacos
+ * nacos-client
+ * 1.2.0
+ *
+ */
+
+ private static Properties properties = new Properties();
+ /**
+ * config data id = config name
+ */
+ private static final String DATA_ID = "dos_detection.properties";
+ /**
+ * config group
+ */
+ private static final String GROUP = "Galaxy";
+
+ private void getProperties() {
+ properties.setProperty(PropertyKeyConst.SERVER_ADDR, "192.168.44.241:8848");
+ properties.setProperty(PropertyKeyConst.NAMESPACE, "prod");
+ properties.setProperty(PropertyKeyConst.USERNAME, "nacos");
+ properties.setProperty(PropertyKeyConst.PASSWORD, "nacos");
+ }
+
+
+ @Test
+ public void GetConfigurationTest() {
+ try {
+ getProperties();
+ ConfigService configService = NacosFactory.createConfigService(properties);
+ String content = configService.getConfig(DATA_ID, GROUP, 5000);
+ Properties nacosConfigMap = new Properties();
+ nacosConfigMap.load(new StringReader(content));
+ System.out.println(nacosConfigMap.getProperty("source.kafka.servers"));
+ System.out.println(content);
+ } catch (NacosException | IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ }
+
+ @Test
+ public void ListenerConfigurationTest() {
+ getProperties();
+ try {
+ //first get config
+ ConfigService configService = NacosFactory.createConfigService(properties);
+ String config = configService.getConfig(DATA_ID, GROUP, 5000);
+ System.out.println(config);
+
+ //start listenner
+ configService.addListener(DATA_ID, GROUP, new Listener() {
+ @Override
+ public Executor getExecutor() {
+ return null;
+ }
+
+ @Override
+ public void receiveConfigInfo(String configMsg) {
+ System.out.println(configMsg);
+ }
+ });
+ } catch (NacosException e) {
+ e.printStackTrace();
+ }
+
+ //keep running,change nacos config,print new config
+ for (int i = 0; i < 3; i++) {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+}
diff --git a/src/test/java/com/zdjizhi/nacos/SchemaListenerTest.java b/src/test/java/com/zdjizhi/nacos/SchemaListenerTest.java
new file mode 100644
index 0000000..992bb3f
--- /dev/null
+++ b/src/test/java/com/zdjizhi/nacos/SchemaListenerTest.java
@@ -0,0 +1,117 @@
+package com.zdjizhi.nacos;
+
+import cn.hutool.json.JSONArray;
+import cn.hutool.json.JSONObject;
+import com.alibaba.nacos.api.NacosFactory;
+import com.alibaba.nacos.api.PropertyKeyConst;
+import com.alibaba.nacos.api.config.ConfigService;
+import com.alibaba.nacos.api.config.listener.Listener;
+import com.alibaba.nacos.api.exception.NacosException;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.concurrent.Executor;
+
+/**
+ * @author qidaijie
+ * @Package com.zdjizhi.nacos
+ * @Description:
+ * @date 2022/3/1714:57
+ */
+public class SchemaListenerTest {
+
+ private static Properties properties = new Properties();
+ private static ArrayList jobList;
+
+
+ static {
+ properties.setProperty(PropertyKeyConst.SERVER_ADDR, "192.168.44.12:8848");
+ properties.setProperty(PropertyKeyConst.NAMESPACE, "prod");
+ properties.setProperty(PropertyKeyConst.USERNAME, "nacos");
+ properties.setProperty(PropertyKeyConst.PASSWORD, "nacos");
+
+ try {
+ ConfigService configService = NacosFactory.createConfigService(properties);
+ String dataId = "session_record.json";
+ String group = "Galaxy";
+ jobList = getJobListFromHttp(configService.getConfig(dataId, group, 5000));
+ System.out.println(jobList);
+ configService.addListener(dataId, group, new Listener() {
+ @Override
+ public Executor getExecutor() {
+ return null;
+ }
+
+ @Override
+ public void receiveConfigInfo(String configMsg) {
+ jobList = getJobListFromHttp(configMsg);
+ }
+ });
+ } catch (NacosException e) {
+ e.printStackTrace();
+ }
+ }
+
+
+ @Test
+ public void dealCommonMessage() {
+ //keep running,change nacos config,print new config
+ for (int i = 0; i < 1; i++) {
+ try {
+ for (String[] job : jobList) {
+ System.out.println(Arrays.toString(job));
+ }
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ /**
+ * 根据http链接获取schema，解析之后返回一个任务列表 (useList toList funcList paramlist)
+ *
+ * @return 任务列表
+ */
+ private static ArrayList getJobListFromHttp(String schema) {
+ ArrayList list = new ArrayList<>();
+
+ JSONObject schemaJson = new JSONObject(schema, false, true);
+ JSONArray fields = schemaJson.getJSONArray("fields");
+ for (Object field : fields) {
+ JSONObject fieldJson = new JSONObject(field, false, true);
+ boolean hasDoc = fieldJson.containsKey("doc");
+ if (hasDoc) {
+ JSONObject docJson = fieldJson.getJSONObject("doc");
+ boolean hasFormat = docJson.containsKey("format");
+ if (hasFormat) {
+ String name = fieldJson.getStr("name");
+ JSONArray formatList = docJson.getJSONArray("format");
+ for (Object format : formatList) {
+ JSONObject formatJson = new JSONObject(format, false, true);
+ String function = formatJson.getStr("function");
+ String appendTo = null;
+ String params = null;
+
+ if (formatJson.containsKey("appendTo")) {
+ appendTo = formatJson.getStr("appendTo");
+ } else {
+ appendTo = name;
+ }
+
+ if (formatJson.containsKey("param")) {
+ params = formatJson.getStr("param");
+ }
+
+ list.add(new String[]{name, appendTo, function, params});
+
+ }
+ }
+ }
+ }
+
+ return list;
+ }
+}
diff --git a/src/test/java/com/zdjizhi/testingDataset/gtpc b/src/test/java/com/zdjizhi/testingDataset/gtpc
new file mode 100644
index 0000000..35d57c0
--- /dev/null
+++ b/src/test/java/com/zdjizhi/testingDataset/gtpc
@@ -0,0 +1,8 @@
+{"common_schema_type":"GTP-C","gtp_version":"v2","gtp_msg_type":"delete","gtp_downlink_teid":392955378,"gtp_uplink_teid":579533063,"gtp_phone_number":"8617239723344","gtp_imei":"86989005126503","gtp_imsi":"460077015061987","common_direction":69,"common_l7_protocol":"UNCATEGORIZED","common_app_label":"gtpv2","common_app_id":{"THIRD":[{"app_name":"gtpv2","app_id":735,"surrogate_id":0,"packet_sequence":1}]},"common_app_identify_info":[{"app_name":"gtpv2","packet_sequence":1}],"common_server_ip":"172.50.0.100","common_client_ip":"172.50.0.1","common_server_port":2123,"common_client_port":12564,"common_stream_dir":2,"common_address_type":4,"common_address_list":"12564-2123-172.50.0.1-172.50.0.100","common_start_time":1663752297,"common_end_time":1663752711,"common_con_duration_ms":305,"common_s2c_pkt_num":107,"common_s2c_byte_num":9951,"common_c2s_pkt_num":0,"common_c2s_byte_num":0,"common_client_location":"United States.Other.Other","common_server_location":"United States.Other.Other","common_stream_trace_id":"579013265830096219","common_l4_protocol":"IPv4_UDP","common_sled_ip":"192.168.40.161","common_device_id":"unknown","common_device_tag":"{\"tags\":[{\"tag\":\"device_group\",\"value\":\"group-xxg\"},{\"tag\":\"data_center\",\"value\":\"center-xxg\"}]}","common_vsys_id":1,"common_policy_id":0,"common_service":8,"common_action":0}
+
+{"common_schema_type":"GTP-C","gtp_version":"v2","gtp_msg_type":"modify","gtp_downlink_teid":247749709,"gtp_uplink_teid":226022706,"gtp_phone_number":"8615859199999","gtp_imei":"86941104639999","gtp_imsi":"460028591549999","common_direction":69,"common_l7_protocol":"UNCATEGORIZED","common_app_label":"gtpv2","common_app_id":{"THIRD":[{"app_name":"gtpv2","app_id":735,"surrogate_id":0,"packet_sequence":1}]},"common_app_identify_info":[{"app_name":"gtpv2","packet_sequence":1}],"common_server_ip":"172.50.0.100","common_client_ip":"172.50.0.1","common_server_port":2123,"common_client_port":12564,"common_stream_dir":2,"common_address_type":4,"common_address_list":"12564-2123-172.50.0.1-172.50.0.100","common_start_time":1663752297,"common_end_time":1663752711,"common_con_duration_ms":305,"common_s2c_pkt_num":107,"common_s2c_byte_num":9951,"common_c2s_pkt_num":0,"common_c2s_byte_num":0,"common_client_location":"United States.Other.Other","common_server_location":"United States.Other.Other","common_stream_trace_id":"579013265830096219","common_l4_protocol":"IPv4_UDP","common_sled_ip":"192.168.40.161","common_device_id":"unknown","common_device_tag":"{\"tags\":[{\"tag\":\"device_group\",\"value\":\"group-xxg\"},{\"tag\":\"data_center\",\"value\":\"center-xxg\"}]}","common_vsys_id":1,"common_policy_id":0,"common_service":8,"common_action":0,"common_vsys_id":2}
+
+{"common_schema_type":"GTP-C","gtp_version":"v2","gtp_msg_type":"delete","gtp_downlink_teid":247749709,"gtp_uplink_teid":226022706,"gtp_phone_number":"8615859199999","gtp_imei":"86941104639999","gtp_imsi":"460028591549999","common_direction":69,"common_l7_protocol":"UNCATEGORIZED","common_app_label":"gtpv2","common_app_id":{"THIRD":[{"app_name":"gtpv2","app_id":735,"surrogate_id":0,"packet_sequence":1}]},"common_app_identify_info":[{"app_name":"gtpv2","packet_sequence":1}],"common_server_ip":"172.50.0.100","common_client_ip":"172.50.0.1","common_server_port":2123,"common_client_port":12564,"common_stream_dir":2,"common_address_type":4,"common_address_list":"12564-2123-172.50.0.1-172.50.0.100","common_start_time":1663752297,"common_end_time":1663753228,"common_con_duration_ms":305,"common_s2c_pkt_num":107,"common_s2c_byte_num":9951,"common_c2s_pkt_num":0,"common_c2s_byte_num":0,"common_client_location":"United States.Other.Other","common_server_location":"United States.Other.Other","common_stream_trace_id":"579013265830096219","common_l4_protocol":"IPv4_UDP","common_sled_ip":"192.168.40.161","common_device_id":"unknown","common_device_tag":"{\"tags\":[{\"tag\":\"device_group\",\"value\":\"group-xxg\"},{\"tag\":\"data_center\",\"value\":\"center-xxg\"}]}","common_vsys_id":1,"common_policy_id":0,"common_service":8,"common_action":0,"common_vsys_id":3}
+
+{"common_schema_type":"GTP-C","gtp_version":"v2","gtp_msg_type":"modify","gtp_downlink_teid":247749709,"gtp_uplink_teid":226022706,"gtp_phone_number":"8615859199999","gtp_imei":"86941104639999","gtp_imsi":"460028591549999","common_direction":69,"common_l7_protocol":"UNCATEGORIZED","common_app_label":"gtpv2","common_app_id":{"THIRD":[{"app_name":"gtpv2","app_id":735,"surrogate_id":0,"packet_sequence":1}]},"common_app_identify_info":[{"app_name":"gtpv2","packet_sequence":1}],"common_server_ip":"172.50.0.100","common_client_ip":"172.50.0.1","common_server_port":2123,"common_client_port":12564,"common_stream_dir":2,"common_address_type":4,"common_address_list":"12564-2123-172.50.0.1-172.50.0.100","common_start_time":1663752297,"common_end_time":1663752711,"common_con_duration_ms":305,"common_s2c_pkt_num":107,"common_s2c_byte_num":9951,"common_c2s_pkt_num":0,"common_c2s_byte_num":0,"common_client_location":"United States.Other.Other","common_server_location":"United States.Other.Other","common_stream_trace_id":"579013265830096219","common_l4_protocol":"IPv4_UDP","common_sled_ip":"192.168.40.161","common_device_id":"unknown","common_device_tag":"{\"tags\":[{\"tag\":\"device_group\",\"value\":\"group-xxg\"},{\"tag\":\"data_center\",\"value\":\"center-xxg\"}]}","common_vsys_id":1,"common_policy_id":0,"common_service":8,"common_action":0,"common_vsys_id":4}
+{"common_schema_type":"GTP-C","gtp_version":"v2","gtp_msg_type":"modify","gtp_downlink_teid":247749709,"gtp_uplink_teid":226022706,"gtp_phone_number":"8615859199999","gtp_imei":"86941104639999","gtp_imsi":"460028591549999","common_direction":69,"common_l7_protocol":"UNCATEGORIZED","common_app_label":"gtpv2","common_app_id":{"THIRD":[{"app_name":"gtpv2","app_id":735,"surrogate_id":0,"packet_sequence":1}]},"common_app_identify_info":[{"app_name":"gtpv2","packet_sequence":1}],"common_server_ip":"172.50.0.100","common_client_ip":"172.50.0.1","common_server_port":2123,"common_client_port":12564,"common_stream_dir":2,"common_address_type":4,"common_address_list":"12564-2123-172.50.0.1-172.50.0.100","common_start_time":1663752297,"common_end_time":1663752711,"common_con_duration_ms":305,"common_s2c_pkt_num":107,"common_s2c_byte_num":9951,"common_c2s_pkt_num":0,"common_c2s_byte_num":0,"common_client_location":"United States.Other.Other","common_server_location":"United States.Other.Other","common_stream_trace_id":"579013265830096219","common_l4_protocol":"IPv4_UDP","common_sled_ip":"192.168.40.161","common_device_id":"unknown","common_device_tag":"{\"tags\":[{\"tag\":\"device_group\",\"value\":\"group-xxg\"},{\"tag\":\"data_center\",\"value\":\"center-xxg\"}]}","common_vsys_id":1,"common_policy_id":0,"common_service":8,"common_action":0,"common_vsys_id":3}
\ No newline at end of file
diff --git a/src/test/java/com/zdjizhi/testingDataset/radius b/src/test/java/com/zdjizhi/testingDataset/radius
new file mode 100644
index 0000000..149aa1b
--- /dev/null
+++ b/src/test/java/com/zdjizhi/testingDataset/radius
@@ -0,0 +1,7 @@
+{"common_stream_dir":1,"common_address_type":4,"common_client_ip":"192.168.50.26","common_server_ip":"192.168.40.190","common_client_port":62228,"common_server_port":1813,"common_c2s_pkt_num":1,"common_s2c_pkt_num":0,"common_c2s_byte_num":52,"common_s2c_byte_num":0,"common_start_time":1663754402,"common_end_time":1663754402,"common_con_duration_ms":0,"common_stream_trace_id":107327506993450,"common_l4_protocol":"IPv4_UDP","common_address_list":"62228-1813-192.168.50.26-192.168.40.190","common_policy_id":0,"common_service":162,"common_sled_ip":"192.168.40.161","common_schema_type":"RADIUS","common_device_tag":"{\"tags\":[{\"tag\":\"device_group\",\"value\":\"group-xxg\"},{\"tag\":\"data_center\",\"value\":\"center-xxg\"}]}","radius_packet_type":4,"radius_account":"test1","radius_acct_status_type":1,"radius_acct_session_id":"10964","radius_framed_ip":"192.168.50.26","radius_event_timestamp":1663580387}
+{"common_stream_dir":1,"common_address_type":4,"common_client_ip":"192.168.50.26","common_server_ip":"192.168.40.190","common_client_port":62229,"common_server_port":1813,"common_c2s_pkt_num":1,"common_s2c_pkt_num":0,"common_c2s_byte_num":52,"common_s2c_byte_num":0,"common_start_time":1663754402,"common_end_time":1663754402,"common_con_duration_ms":0,"common_stream_trace_id":107327506993452,"common_l4_protocol":"IPv4_UDP","common_address_list":"62229-1813-192.168.50.26-192.168.40.190","common_policy_id":0,"common_service":162,"common_sled_ip":"192.168.40.161","common_schema_type":"RADIUS","common_device_tag":"{\"tags\":[{\"tag\":\"device_group\",\"value\":\"group-xxg\"},{\"tag\":\"data_center\",\"value\":\"center-xxg\"}]}","radius_packet_type":4,"radius_account":"test2","radius_acct_status_type":1,"radius_acct_session_id":"10964","radius_framed_ip":"192.168.50.16","radius_event_timestamp":1663580387,"common_vsys_id":3}
+{"common_stream_dir":1,"common_address_type":4,"common_client_ip":"192.168.50.26","common_server_ip":"192.168.40.190","common_client_port":62229,"common_server_port":1813,"common_c2s_pkt_num":1,"common_s2c_pkt_num":0,"common_c2s_byte_num":52,"common_s2c_byte_num":0,"common_start_time":1663754402,"common_end_time":1663754402,"common_con_duration_ms":0,"common_stream_trace_id":107327506993452,"common_l4_protocol":"IPv4_UDP","common_address_list":"62229-1813-192.168.50.26-192.168.40.190","common_policy_id":0,"common_service":162,"common_sled_ip":"192.168.40.161","common_schema_type":"RADIUS","common_device_tag":"{\"tags\":[{\"tag\":\"device_group\",\"value\":\"group-xxg\"},{\"tag\":\"data_center\",\"value\":\"center-xxg\"}]}","radius_packet_type":4,"radius_account":"test2","radius_acct_status_type":1,"radius_acct_session_id":"10964","radius_framed_ip":"192.168.50.16","radius_event_timestamp":1663580387,"common_vsys_id":4}
+
+
+
+{"common_stream_dir":1,"common_address_type":4,"common_client_ip":"192.168.50.26","common_server_ip":"192.168.40.190","common_client_port":62229,"common_server_port":1813,"common_c2s_pkt_num":1,"common_s2c_pkt_num":0,"common_c2s_byte_num":52,"common_s2c_byte_num":0,"common_start_time":1663754402,"common_end_time":1663754653,"common_con_duration_ms":0,"common_stream_trace_id":107327506993452,"common_l4_protocol":"IPv4_UDP","common_address_list":"62229-1813-192.168.50.26-192.168.40.190","common_policy_id":0,"common_service":162,"common_sled_ip":"192.168.40.161","common_schema_type":"RADIUS","common_device_tag":"{\"tags\":[{\"tag\":\"device_group\",\"value\":\"group-xxg\"},{\"tag\":\"data_center\",\"value\":\"center-xxg\"}]}","radius_packet_type":4,"radius_account":"test2","radius_acct_status_type":2,"radius_acct_session_id":"10964","radius_framed_ip":"192.168.50.16","radius_event_timestamp":1663580387,"common_vsys_id":3}
\ No newline at end of file
diff --git a/src/test/java/com/zdjizhi/testingDataset/session b/src/test/java/com/zdjizhi/testingDataset/session
new file mode 100644
index 0000000..ae93c4c
--- /dev/null
+++ b/src/test/java/com/zdjizhi/testingDataset/session
@@ -0,0 +1,2 @@
+{"common_schema_type":"HTTP","common_sessions":1,"http_request_line":"GET sampleFile.html HTTP/1.1","http_host":"www.texaslotto.com","http_url":"www.texaslotto.com/sampleFile.html","http_user_agent":"xPTS/2.0","http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html","http_isn":1953597368,"http_proxy_flag":0,"http_version":"http1","http_response_latency_ms":1,"http_session_duration_ms":2,"http_sequence":80,"common_protocol_label":"ETHERNET.IPv4.UDP.GTP.IPv4.TCP","common_c2s_byte_diff":17110,"common_c2s_pkt_diff":119,"common_s2c_byte_diff":16490,"common_s2c_pkt_diff":81,"common_c2s_ipfrag_num":0,"common_s2c_ipfrag_num":0,"common_first_ttl":64,"common_c2s_tcp_unorder_num":0,"common_s2c_tcp_unorder_num":0,"common_c2s_tcp_lostlen":0,"common_s2c_tcp_lostlen":0,"common_c2s_pkt_retrans":240,"common_s2c_pkt_retrans":162,"common_c2s_byte_retrans":12800,"common_s2c_byte_retrans":18400,"common_direction":69,"common_l7_protocol":"HTTP","common_app_label":"unknown","common_app_id":{"UNKNOWN":[{"app_name":"unknown","app_id":4,"surrogate_id":0,"packet_sequence":29}]},"common_app_identify_info":[{"app_name":"unknown","packet_sequence":29}],"common_tcp_client_isn":1953597368,"common_tcp_server_isn":1950649408,"common_server_ip":"10.201.35.10","common_client_ip":"1.1.1.27","common_server_port":80,"common_client_port":2000,"common_stream_dir":3,"common_address_type":4,"common_address_list":"IPv4_TCP<2000-80-1.1.1.27-10.201.35.10>|GTP<111534000-665547833>|IPv4_UDP<2152-2152-192.56.5.2-192.56.10.20>|MAC<000c299b2fa4-000c2915b4f4>","common_start_time":1660272209,"common_end_time":1660272424,"common_con_duration_ms":215201,"common_s2c_pkt_num":243,"common_s2c_byte_num":49470,"common_c2s_pkt_num":360,"common_c2s_byte_num":51600,"common_establish_latency_ms":1,"common_client_location":"china1.beijing.beijing1","common_tunnels":[{"tunnels_schema_type":"GTP","gtp_endpoint_a2b_teid":247749709,"gtp_endpoint_b2a_teid":665547833,"gtp_sgw_ip":"192.56.5.2","gtp_pgw_ip":"192.56.10.20","gtp_sgw_port":2152,"gtp_pgw_port":2152},{"tunnels_schema_type":"MULTIPATH_ETHERNET","c2s_source_mac":"00:0c:29:9b:2f:a4","c2s_destination_mac":"00:0c:29:15:b4:f4","s2c_source_mac":"00:0c:29:15:b4:f4","s2c_destination_mac":"00:0c:29:9b:2f:a4"}],"common_stream_trace_id":"869231578438992199","common_l4_protocol":"IPv4_TCP","common_sled_ip":"192.168.40.81","common_device_id":"21426003","common_device_tag":"{\"tags\":[{\"tag\":\"device_group\",\"value\":\"group-xxg-three\"},{\"tag\":\"data_center\",\"value\":\"center-xxg-three\"}]}","common_policy_id":0,"common_service":2,"common_action":0}
+{"common_schema_type":"HTTP","common_sessions":1,"http_request_line":"GET sampleFile.html HTTP/1.1","http_host":"www.texaslotto.com","http_url":"www.texaslotto.com/sampleFile.html","http_user_agent":"xPTS/2.0","http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html","http_isn":1953597368,"http_proxy_flag":0,"http_version":"http1","http_response_latency_ms":1,"http_session_duration_ms":2,"http_sequence":80,"common_protocol_label":"ETHERNET.IPv4.UDP.GTP.IPv4.TCP","common_c2s_byte_diff":17110,"common_c2s_pkt_diff":119,"common_s2c_byte_diff":16490,"common_s2c_pkt_diff":81,"common_c2s_ipfrag_num":0,"common_s2c_ipfrag_num":0,"common_first_ttl":64,"common_c2s_tcp_unorder_num":0,"common_s2c_tcp_unorder_num":0,"common_c2s_tcp_lostlen":0,"common_s2c_tcp_lostlen":0,"common_c2s_pkt_retrans":240,"common_s2c_pkt_retrans":162,"common_c2s_byte_retrans":12800,"common_s2c_byte_retrans":18400,"common_direction":69,"common_l7_protocol":"HTTP","common_app_label":"unknown","common_app_id":{"UNKNOWN":[{"app_name":"unknown","app_id":4,"surrogate_id":0,"packet_sequence":29}]},"common_app_identify_info":[{"app_name":"unknown","packet_sequence":29}],"common_tcp_client_isn":1953597368,"common_tcp_server_isn":1950649408,"common_server_ip":"10.201.35.10","common_client_ip":"1.1.1.27","common_server_port":80,"common_client_port":2000,"common_stream_dir":3,"common_address_type":4,"common_address_list":"IPv4_TCP<2000-80-1.1.1.27-10.201.35.10>|GTP<111534000-665547833>|IPv4_UDP<2152-2152-192.56.5.2-192.56.10.20>|MAC<000c299b2fa4-000c2915b4f4>","common_start_time":1660272209,"common_end_time":1660272424,"common_con_duration_ms":215201,"common_s2c_pkt_num":243,"common_s2c_byte_num":49470,"common_c2s_pkt_num":360,"common_c2s_byte_num":51600,"common_establish_latency_ms":1,"common_client_location":"china1.beijing.beijing1","common_tunnels":[{"tunnels_schema_type":"GTP","gtp_endpoint_a2b_teid":247749709,"gtp_endpoint_b2a_teid":665547833,"gtp_sgw_ip":"192.56.5.2","gtp_pgw_ip":"192.56.10.20","gtp_sgw_port":2152,"gtp_pgw_port":2152},{"tunnels_schema_type":"MULTIPATH_ETHERNET","c2s_source_mac":"00:0c:29:9b:2f:a4","c2s_destination_mac":"00:0c:29:15:b4:f4","s2c_source_mac":"00:0c:29:15:b4:f4","s2c_destination_mac":"00:0c:29:9b:2f:a4"}],"common_stream_trace_id":"869231578438992199","common_l4_protocol":"IPv4_TCP","common_sled_ip":"192.168.40.81","common_device_id":"21426003","common_device_tag":"{\"tags\":[{\"tag\":\"device_group\",\"value\":\"group-xxg-three\"},{\"tag\":\"data_center\",\"value\":\"center-xxg-three\"}]}","common_policy_id":0,"common_service":2,"common_action":0,"common_vsys_id":4}