gfwleak/galaxy-tsg-olap-log-stream-doublewrite
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1.适配TSG 23.07及以上功能，添加数据传输统计指标，并输出至pushgateway。(GAL-409)

Browse Source 
2.原URL参数domain从http_domain字段取值，更新为从common_server_domain字段取值。(GAL-410)
This commit is contained in:
wangchengcheng
2023-09-28 15:59:26 +08:00
parent c3ad8140e8
commit 5c0a108393
77 changed files with 4726 additions and 2543 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
src/test/java/com/zdjizhi/json/FastJsonTest.java Normal file
View File

@@ -0,0 +1,98 @@
package com.zdjizhi.json;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.zdjizhi.json.pojo.UserList;
import com.zdjizhi.json.pojo.UserMap;
import org.junit.Test;
import java.util.List;
import java.util.Map;
/**
* @author qidaijie
* @Package com.zdjizhi.json
* @Description:
* @date 2023/5/2014:02
*/
public class FastJsonTest {
@Test
public void pojoTest() {
//all right
String message = "{\"name\":\"zhangsan\",\"age\":50,\"idcard\":\"140303199999999999\",\"previousaddress\":{\"first\":\"北京\",\"second\":\"上海\"}}";
//int error (yes)
// String message = "{\"name\":\"zhangsan\",\"age\":\"123\",\"idCard\":\"140303199999999999\",\"previousaddress\":{\"first\":\"北京\",\"second\":\"上海\"}}";
//string error (yes)
// String message = "{\"name\":123,\"age\":123,\"idCard\":\"140303199999999999\",\"previousaddress\":{\"first\":\"北京\",\"second\":\"上海\"}}";
//json error (no)
// String message = "{\"name\":\"zhangsan\",\"age\":50,\"idCard\":\"140303199999999999\",\"previousaddress\":\"{\\\"first\\\":\\\"北京\\\",\\\"second\\\":\\\"上海\\\"}\"}";
UserMap user = JSON.parseObject(message, UserMap.class);
System.out.println(user.getName());
System.out.println(user.getDevicetag());
System.out.println(JSON.toJSONString(user));
}
@Test
public void typeCheckTest() {
//jsonobject
String message = "{\"name\":\"zhangsan\",\"age\":50,\"idcard\":\"123456789\",\"devicetag\":\"{\\\"tags\\\":[{\\\"tag\\\":\\\"group\\\",\\\"value\\\":\\\"7400\\\"},{\\\"tag\\\":\\\"center\\\",\\\"value\\\":\\\"7400\\\"}]}\"}";
JSONObject objectTest = JSONObject.parseObject(message);
for (Map.Entry<String, Object> entry : objectTest.entrySet()) {
System.out.println("key:" + entry.getKey() + "————value:" + entry.getValue() + "————class: " + entry.getValue().getClass());
}
System.out.println("\n输出原始日志：" + objectTest.toString());
Object previousMap = objectTest.get("devicetag");
if (previousMap.getClass() != Map.class) {
JSONObject previousObject = JSONObject.parseObject(previousMap.toString());
objectTest.put("devicetag", previousObject);
}
System.out.println("输出转换map类型后的日志：" + objectTest.toString());
UserMap userMap = objectTest.toJavaObject(UserMap.class);
System.out.println(JSON.toJSONString(userMap));
System.out.println("\n-----------------------------------------------\n");
//jsonarray
message = "{\"name\":\"zhangsan\",\"age\":50,\"idcard\":\"123456789\",\"devicetag\":\"[{\\\"tag\\\":\\\"group\\\",\\\"value\\\":\\\"7400\\\"},{\\\"tag\\\":\\\"center\\\",\\\"value\\\":\\\"7400\\\"}]\"}";
JSONObject arrayTest = JSONObject.parseObject(message);
for (Map.Entry<String, Object> entry : arrayTest.entrySet()) {
System.out.println("key:" + entry.getKey() + "————value:" + entry.getValue() + "————class: " + entry.getValue().getClass());
}
System.out.println("\n输出原始日志：" + arrayTest.toString());
Object previousList = arrayTest.get("devicetag");
if (previousList.getClass() != List.class) {
JSONArray previousArray = JSONArray.parseArray(previousList.toString());
arrayTest.put("devicetag", previousArray);
}
System.out.println("输出转换list类型后的日志：" + arrayTest.toString());
UserList userList = arrayTest.toJavaObject(UserList.class);
System.out.println(JSON.toJSONString(userList));
}
@Test
public void typeTest() {
String message = "{\"name\":\"zhangsan\",\"age\":12,\"object\":{\"name\":\"a\",\"age\":12},\"array\":[{\"one\":1},{\"two\":2}]}";
JSONObject objectTest = JSONObject.parseObject(message);
for (Map.Entry<String, Object> entry : objectTest.entrySet()) {
String key = entry.getKey();
Class<?> aClass = entry.getValue().getClass();
System.out.println(key + "---------" + aClass.getSimpleName());
}
Object bbb = objectTest.get("bbb");
if (bbb == null){
System.out.println("null");
}
}
}

102
src/test/java/com/zdjizhi/json/JsonPathTest.java Normal file
View File

@@ -0,0 +1,102 @@
package com.zdjizhi.json;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
import com.alibaba.fastjson2.*;
import com.alibaba.nacos.api.NacosFactory;
import com.alibaba.nacos.api.PropertyKeyConst;
import com.alibaba.nacos.api.config.ConfigService;
import com.alibaba.nacos.api.exception.NacosException;
import com.zdjizhi.common.CommonConfig;
import com.zdjizhi.common.FlowWriteConfig;
import org.junit.Test;
import java.util.Properties;
/**
* @author qidaijie
* @Package com.zdjizhi.json
* @Description:
* @date 2022/3/2410:22
*/
public class JsonPathTest {
private static Properties properties = new Properties();
static {
properties.setProperty(PropertyKeyConst.SERVER_ADDR, "192.168.44.12:8848");
properties.setProperty(PropertyKeyConst.USERNAME, "nacos");
properties.setProperty(PropertyKeyConst.PASSWORD, "nacos");
}
@Test
public void konwledgeBaseTest() {
try {
ConfigService configService = NacosFactory.createConfigService(properties);
String schema = configService.getConfig("knowledge_base.json", "DEFAULT_GROUP", 5000);
// String KNOWLEDGE_EXPR = "[?(@.version=='latest' && (@.name in ('ip_v4_built_in','ip_v6_built_in','ip_v4_user_defined','ip_v6_user_defined','asn_v4','asn_v6')))]";
// String KNOWLEDGE_EXPR = "[?(@.version=='latest' && (@.type in ('ip_location','asn','ip')))]";
// String KNOWLEDGE_EXPR = "[?(@.version=='latest' && (@.name in ('ip_v4_built_in','ip_v6_built_in','ip_v4_user_defined','ip_v6_user_defined','asn_v4','asn_v6')) && (@.type in ('ip_location','asn','ip')))]";
// String types = "[?(@.type in ('ip_location','asn','ip_user_defined'))]";
// String names = "[?(@.name in ('ip_v4_built_in','ip_v6_built_in','asn_v4','asn_v6','内置IP定位库'))]";
// String expr = "[?(@.version=='latest')][?(@.type in ('ip_location','asn','ip_user_defined'))]";
// String expr = "[?(@.version=='latest')][?(@.name == 'QQQ' || (@.type == 'ip_user_defined'))]";
String expr = "[?(@.version=='latest')][@.type in ('ip_location','asn','ip')][?(@.name in ('QQQ'))]";
// JSONPath jsonPath = JSONPath.of(combinationFilterList());
JSONPath jsonPath = JSONPath.of(expr);
String extract = jsonPath.extract(JSONReader.of(schema)).toString();
JSONArray jsonArray = JSON.parseArray(extract);
for (int i = 0; i < jsonArray.size(); i++) {
System.out.println(jsonArray.getString(i));
// KnowlegeBaseMeta knowlegeBaseMeta = JSONObject.parseObject(jsonArray.getString(i), KnowlegeBaseMeta.class);
// System.out.println(knowlegeBaseMeta.toString());
}
} catch (NacosException e) {
e.printStackTrace();
}
}
public static String combinationFilterList() {
String[] typeList = CommonConfig.KNOWLEDGEBASE_TYPE_LIST.split(",");
String[] nameList = CommonConfig.KNOWLEDGEBASE_NAME_LIST.split(",");
String expr = "[?(@.version=='latest')]";
// ip_location > 'ip_location'
if (typeList.length > 1) {
StringBuilder typeBuilder = new StringBuilder();
typeBuilder.append("[?(@.type in (");
for (int i = 0; i < typeList.length; i++) {
if (i == typeList.length - 1) {
typeBuilder.append("'").append(typeList[i]).append("'))]");
} else {
typeBuilder.append("'").append(typeList[i]).append("',");
}
}
expr = expr + typeBuilder.toString();
}
if (nameList.length > 1) {
StringBuilder nameBuilder = new StringBuilder();
nameBuilder.append("[?(@.name in (");
for (int i = 0; i < nameList.length; i++) {
if (i == nameList.length - 1) {
nameBuilder.append("'").append(nameList[i]).append("'))]");
} else {
nameBuilder.append("'").append(nameList[i]).append("',");
}
}
expr = expr + nameBuilder.toString();
}
return expr;
}
}

223
src/test/java/com/zdjizhi/json/NewSchemaTest.java Normal file
View File

@@ -0,0 +1,223 @@
package com.zdjizhi.json;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.alibaba.nacos.api.NacosFactory;
import com.alibaba.nacos.api.PropertyKeyConst;
import com.alibaba.nacos.api.config.ConfigService;
import com.alibaba.nacos.api.exception.NacosException;
import com.zdjizhi.common.FlowWriteConfig;
import org.junit.Test;
import java.util.*;
/**
* Applicable to schemas >= TSG22.08
*
* @author qidaijie
* @Package com.zdjizhi.nacos
* @Description:
* @date 2022/3/1714:57
*/
public class NewSchemaTest {
private static Properties properties = new Properties();
/**
* 获取需要删除字段的列表
*/
private static ArrayList<String> dropList = new ArrayList<>();
/**
* 获取schema指定的有效字段及类型
*/
private static HashMap<String, Class> jsonFieldsMap;
/**
* 获取包含默认值的字段
*/
private static HashMap<String, Object> defaultFieldsMap = new HashMap<>(16);
static {
properties.setProperty(PropertyKeyConst.SERVER_ADDR, "192.168.44.11:8848");
properties.setProperty(PropertyKeyConst.NAMESPACE, "f507879a-8b1b-4330-913e-83d4fcdc14bb");
properties.setProperty(PropertyKeyConst.USERNAME, "nacos");
properties.setProperty(PropertyKeyConst.PASSWORD, "nacos");
}
@Test
public void newSchemaTest() {
try {
ConfigService configService = NacosFactory.createConfigService(properties);
String dataId = "session_record.json";
String group = "Galaxy";
String schema = configService.getConfig(dataId, group, 5000);
ArrayList<String[]> newJobList = getNewJobList(schema);
for (String[] job : newJobList) {
System.out.println(Arrays.toString(job));
}
HashMap<String, Class> fieldsFromSchema = getFieldsFromSchema(schema);
for (String key : fieldsFromSchema.keySet()) {
System.out.println("fileName:" + key + " Class:" + fieldsFromSchema.get(key));
}
} catch (NacosException e) {
e.printStackTrace();
}
}
/**
* 模式匹配，给定一个类型字符串返回一个类类型
*
* @param type 类型
* @return 类类型
*/
private static Class getClassName(String type) {
Class clazz;
switch (type) {
case "int":
clazz = Integer.class;
break;
case "string":
clazz = String.class;
break;
case "long":
clazz = long.class;
break;
case "array":
clazz = List.class;
break;
case "double":
clazz = double.class;
break;
case "float":
clazz = float.class;
break;
case "char":
clazz = char.class;
break;
case "byte":
clazz = byte.class;
break;
case "boolean":
clazz = boolean.class;
break;
case "short":
clazz = short.class;
break;
default:
clazz = String.class;
}
return clazz;
}
/**
* 通过schema来获取所需的字段及字段类型。
*
* @return 用于反射生成schema类型的对象的一个map集合
*/
private static HashMap<String, Class> getFieldsFromSchema(String schema) {
HashMap<String, Class> map = new HashMap<>(256);
//获取fields，并转化为数组，数组的每个元素都是一个name doc type
com.alibaba.fastjson2.JSONObject schemaJson = com.alibaba.fastjson2.JSONObject.parseObject(schema);
com.alibaba.fastjson2.JSONArray fields = schemaJson.getJSONArray("fields");
for (Object field : fields) {
String filedStr = field.toString();
if (checkKeepField(filedStr)) {
com.alibaba.fastjson2.JSONObject fieldJson = com.alibaba.fastjson2.JSONObject.parseObject(filedStr);
String name = fieldJson.getString("name");
String type = fieldJson.getString("type");
if (type.contains("{")) {
com.alibaba.fastjson2.JSONObject types = com.alibaba.fastjson2.JSONObject.parseObject(type);
type = types.getString("type");
}
if (fieldJson.containsKey("default")) {
defaultFieldsMap.put(name, fieldJson.get("default"));
}
//组合用来生成实体类的map
map.put(name, getClassName(type));
} else {
dropList.add(filedStr);
}
}
return map;
}
/**
* 判断字段是否需要保留
*
* @param message 单个field-json
* @return true or false
*/
private static boolean checkKeepField(String message) {
boolean isKeepField = true;
com.alibaba.fastjson2.JSONObject fieldJson = com.alibaba.fastjson2.JSONObject.parseObject(message);
boolean hasDoc = fieldJson.containsKey("doc");
if (hasDoc) {
com.alibaba.fastjson2.JSONObject doc = com.alibaba.fastjson2.JSONObject.parseObject(fieldJson.getString("doc"));
if (doc.containsKey("visibility")) {
String visibility = doc.getString("visibility");
if ("disabled".equals(visibility)) {
isKeepField = false;
}
}
}
return isKeepField;
}
/**
* 根据http链接获取schema，解析之后返回一个任务列表 (useList toList funcList paramlist)
*
* @return 任务列表
*/
private static ArrayList<String[]> getNewJobList(String schema) {
ArrayList<String[]> list = new ArrayList<>();
JSONObject schemaJson = JSONObject.parseObject(schema);
JSONArray fields = schemaJson.getJSONArray("fields");
for (Object field : fields) {
JSONObject fieldJson = JSONObject.parseObject(field.toString());
boolean hasDoc = fieldJson.containsKey("doc");
if (hasDoc) {
JSONObject docJson = JSONObject.parseObject(fieldJson.getString("doc"));
boolean hasFormat = docJson.containsKey("format");
if (hasFormat) {
String name = fieldJson.getString("name");
JSONArray formatList = docJson.getJSONArray("format");
for (Object format : formatList) {
JSONObject formatJson = JSONObject.parseObject(format.toString());
String function = formatJson.getString("function");
String appendTo;
String params = null;
if (formatJson.containsKey("appendTo")) {
appendTo = formatJson.getString("appendTo");
} else {
appendTo = name;
}
if (formatJson.containsKey("param")) {
params = formatJson.getString("param");
}
list.add(new String[]{name, appendTo, function, params});
}
}
}
}
return list;
}
}

121
src/test/java/com/zdjizhi/json/OldSchemaTest.java Normal file
View File

@@ -0,0 +1,121 @@
package com.zdjizhi.json;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.nacos.api.NacosFactory;
import com.alibaba.nacos.api.PropertyKeyConst;
import com.alibaba.nacos.api.config.ConfigService;
import com.alibaba.nacos.api.exception.NacosException;
import com.zdjizhi.common.FlowWriteConfig;
import com.geedgenetworks.utils.StringUtil;
import org.junit.Test;
import java.util.*;
/**
* Applicable to schemas < TSG22.08
*
* @author qidaijie
* @Package com.zdjizhi.nacos
* @Description:
* @date 2022/3/1714:57
*/
public class OldSchemaTest {
private static Properties properties = new Properties();
static {
properties.setProperty(PropertyKeyConst.SERVER_ADDR, "192.168.44.12:8848");
properties.setProperty(PropertyKeyConst.NAMESPACE, "prod");
properties.setProperty(PropertyKeyConst.USERNAME, "nacos");
properties.setProperty(PropertyKeyConst.PASSWORD, "nacos");
}
@Test
public void oldSchemaTest() {
try {
ConfigService configService = NacosFactory.createConfigService(properties);
String dataId = "session_record.json";
String group = "Galaxy";
String schema = configService.getConfig(dataId, group, 5000);
ArrayList<String[]> oldJobList = getOldJobList(schema);
for (String[] job : oldJobList) {
System.out.println(Arrays.toString(job));
}
} catch (NacosException e) {
e.printStackTrace();
}
}
/**
* 解析schema，解析之后返回一个任务列表 (useList toList funcList paramlist)
*
* @param schema 日志schema
* @return 任务列表
*/
private static ArrayList<String[]> getOldJobList(String schema) {
ArrayList<String[]> list = new ArrayList<>();
//获取fields，并转化为数组，数组的每个元素都是一个name doc type
JSONObject schemaJson = JSON.parseObject(schema);
JSONArray fields = (JSONArray) schemaJson.get("fields");
for (Object field : fields) {
if (JSON.parseObject(field.toString()).containsKey("doc")) {
Object doc = JSON.parseObject(field.toString()).get("doc");
if (JSON.parseObject(doc.toString()).containsKey("format")) {
String name = JSON.parseObject(field.toString()).get("name").toString();
Object format = JSON.parseObject(doc.toString()).get("format");
JSONObject formatObject = JSON.parseObject(format.toString());
String functions = formatObject.get("transform").toString();
String appendTo = null;
String params = null;
if (formatObject.containsKey("appendTo")) {
appendTo = formatObject.get("appendTo").toString();
}
if (formatObject.containsKey("param")) {
params = formatObject.get("param").toString();
}
if (StringUtil.isNotBlank(appendTo) && StringUtil.isBlank(params)) {
String[] functionArray = functions.split(FlowWriteConfig.FORMAT_SPLITTER);
String[] appendToArray = appendTo.split(FlowWriteConfig.FORMAT_SPLITTER);
for (int i = 0; i < functionArray.length; i++) {
list.add(new String[]{name, appendToArray[i], functionArray[i], null});
}
} else if (StringUtil.isNotBlank(appendTo) && StringUtil.isNotBlank(params)) {
String[] functionArray = functions.split(FlowWriteConfig.FORMAT_SPLITTER);
String[] appendToArray = appendTo.split(FlowWriteConfig.FORMAT_SPLITTER);
String[] paramArray = params.split(FlowWriteConfig.FORMAT_SPLITTER);
for (int i = 0; i < functionArray.length; i++) {
list.add(new String[]{name, appendToArray[i], functionArray[i], paramArray[i]});
}
} else {
list.add(new String[]{name, name, functions, params});
}
}
}
}
return list;
}
}

63
src/test/java/com/zdjizhi/json/pojo/KnowlegeBaseMeta.java Normal file
View File

@@ -0,0 +1,63 @@
package com.zdjizhi.json.pojo;
/**
* @author qidaijie
* @Package com.zdjizhi.json
* @Description:
* @date 2023/5/1918:42
*/
public class KnowlegeBaseMeta {
private String name;
private String sha256;
private String format;
private String path;
public KnowlegeBaseMeta(String name, String sha256, String format, String path) {
this.name = name;
this.sha256 = sha256;
this.format = format;
this.path = path;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getSha256() {
return sha256;
}
public void setSha256(String sha256) {
this.sha256 = sha256;
}
public String getFormat() {
return format;
}
public void setFormat(String format) {
this.format = format;
}
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
@Override
public String toString() {
return "KnowlegeBaseMeta{" +
"name='" + name + '\'' +
", sha256='" + sha256 + '\'' +
", format='" + format + '\'' +
", path='" + path + '\'' +
'}';
}
}

56
src/test/java/com/zdjizhi/json/pojo/UserList.java Normal file
View File

@@ -0,0 +1,56 @@
package com.zdjizhi.json.pojo;
import java.util.List;
/**
* @author qidaijie
* @Package com.zdjizhi.json.pojo
* @Description:
* @date 2023/5/2014:06
*/
public class UserList {
private String name;
private Integer age;
private Long idcard;
private List devicetag;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Integer getAge() {
return age;
}
public void setAge(Integer age) {
this.age = age;
}
public Long getIdcard() {
return idcard;
}
public void setIdcard(Long idcard) {
this.idcard = idcard;
}
public List getDevicetag() {
return devicetag;
}
public void setDevicetag(List devicetag) {
this.devicetag = devicetag;
}
public UserList(String name, Integer age, Long idcard, List devicetag) {
this.name = name;
this.age = age;
this.idcard = idcard;
this.devicetag = devicetag;
}
}

55
src/test/java/com/zdjizhi/json/pojo/UserMap.java Normal file
View File

@@ -0,0 +1,55 @@
package com.zdjizhi.json.pojo;
import java.util.Map;
/**
* @author qidaijie
* @Package com.zdjizhi.json.pojo
* @Description:
* @date 2023/5/2014:06
*/
public class UserMap {
private String name;
private Integer age;
private Long idcard;
private Map devicetag;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Integer getAge() {
return age;
}
public void setAge(Integer age) {
this.age = age;
}
public Long getIdcard() {
return idcard;
}
public void setIdcard(Long idcard) {
this.idcard = idcard;
}
public Map getDevicetag() {
return devicetag;
}
public void setDevicetag(Map devicetag) {
this.devicetag = devicetag;
}
public UserMap(String name, Integer age, Long idcard, Map devicetag) {
this.name = name;
this.age = age;
this.idcard = idcard;
this.devicetag = devicetag;
}
}