gfwleak/wangwei-cn-object-scheduler
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

适配 CMv23.10版本

Browse Source
This commit is contained in:
wangwei
2024-01-18 10:14:21 +08:00
parent b1fd5dc815
commit 01e97642bc
6 changed files with 284 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
config/application.yml
View File

@@ -12,9 +12,11 @@ logging:
config: ./config/logback-spring.xml
tsg:
system:
url: http://192.168.44.3:80
token: aa2bdec5518ad131f71944b13ce5c298&1&
url: http://192.168.44.29:8080
##token: aa2bdec5518ad131f71944b13ce5c298&1&
token: 1ca0350bc499a208a0edadb74d76a0ee&1&
httpTimeout: 3600000
isLatestVersion: false
object:
ip:
#指定排除的IP. 1.如果没有可不写值，2.若有多个用英文逗号,分隔如: 127.0.0.1,127.0.0.2，3.可适用IP段匹配，如：127.*

22
src/main/java/com/geedge/common/constant/TsgObject.java
View File

@@ -38,5 +38,27 @@ public class TsgObject {
public static final Integer VALUE_OBJECT_MEMBER_ITEMS_STRING_EXPR_TYPE_0 = 0;
public static final Integer VALUE_OBJECT_MEMBER_ITEMS_STRING_IS_HEXBIN_0 = 0;
public static String TSG_OBJECT_ID_V2310 = "objectId";
public static String TSG_OBJECT_TYPE_V2310 = "objectType";
public static String TSG_IP_ADDR_OBJECT_V2310 = "ip";
public static String TSG_FQDN_OBJECT_V2310 = "fqdn";
public static String TSG_OBJECT_NAME_V2310 = "objectName";
public static String TSG_IS_BUILTIN_V2310 = "isBuiltin";
public static String TSG_IS_EXCLUSION_V2310 = "isExclusion";
public static String TSG_IS_VALID_V2310 = "isValid";
public static String TSG_ITEM_IP_V2310 = "ip";
public static String TSG_ITEM_PORT_V2310 = "port";
public static String TSG_VALUE_OBJECT_MEMBER_ITEMS_IP_PORT_RANGE_V2310 = "0-65535";
public static String TSG_ITEM_ISSESSION_V2310 = "isSession";
public static String TSG_ITEM_ENDPOINT_V2310 = "endpoint";
public static String TSG_OP_ACTION_V2310 = "opAction";
public static String TSG_UPDATE_V2310 = "update";
public static String TSG_OBJECT_LIST_V2310 = "objectList";
public static String TSG_VSYS_ID_V2310 = "vsysId";
public static String TSG_TSG_OBJECT_IDS_V2310 = "objectIds";
public static String TSG_ITEM_TYPE_V2310 = "itemType";
public static String TSG_KEYWORDARRAY_V2310 = "keywordArray";
public static String TSG_DELETE_ITEMS_BYL_TTIME_V2310 = "deleteItemsByLtTime";
}

2
src/main/java/com/geedge/common/PrometheusMetrics.java → src/main/java/com/geedge/common/monitor/PrometheusMetrics.java
View File

@@ -1,4 +1,4 @@
package com.geedge.common;
package com.geedge.common.monitor;
import io.prometheus.client.exporter.MetricsServlet;
import org.springframework.web.bind.annotation.GetMapping;

47
src/main/java/com/geedge/common/util/TsgUtil.java
View File

@@ -6,7 +6,6 @@ import cn.hutool.http.HttpRequest;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Maps;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@@ -26,8 +25,14 @@ import java.util.concurrent.TimeUnit;
public class TsgUtil {
public static String TSG_URL;
private static Integer httpTimeOut;
private static String TSG_TOKEN;
private static Integer httpTimeOut;
public static Boolean isLatestVersion;
@Value("${tsg.system.isLatestVersion}")
public void isLatestVersion(Boolean latestVersion) {
isLatestVersion = latestVersion;
}
@Value("${tsg.system.httpTimeout}")
public void setTsgUrl(Integer timeOut) {
@@ -71,6 +76,25 @@ public class TsgUtil {
}
}
public static void updateObjectOld(Map<String, Object> body) {
Stopwatch watch = Stopwatch.createStarted();
String response = HttpRequest.put(TSG_URL + "/v1/policy/object")
.header(Header.AUTHORIZATION, getToken())
.body(JSONUtil.toJsonStr(body))
.timeout(httpTimeOut)
.execute().body();
log.info("update tsg-api, cost {} seconds", watch.elapsed(TimeUnit.SECONDS));
if (StrUtil.isBlank(response)) {
log.error("update {} Object error, response: {}", body, response);
throw new RuntimeException("update " + body + " Object error, response: " + response);
}
JSONObject jsonObject = JSONUtil.parseObj(response);
if (!"200".equals(jsonObject.get("code").toString())) {
log.error("update {} Object error, response: {}", body, response);
throw new RuntimeException("update " + body + " Object error, response: " + response);
}
}
public static void deleteItemOfObjectById(Integer id, Map<String, Object> form) {
Stopwatch watch = Stopwatch.createStarted();
String response = HttpRequest.delete(TSG_URL + "/v1/policy/object/" + id + "/item")
@@ -89,4 +113,23 @@ public class TsgUtil {
throw new RuntimeException("update " + id + " Object error, response: " + response);
}
}
public static void deleteItemOfObjectOld(Map<String, Object> body) {
Stopwatch watch = Stopwatch.createStarted();
String response = HttpRequest.delete(TSG_URL + "/v1/policy/items")
.header(Header.AUTHORIZATION, getToken())
.body(JSONUtil.toJsonStr(body))
.timeout(httpTimeOut)
.execute().body();
log.info("delete tsg-api, cost {} seconds", watch.elapsed(TimeUnit.SECONDS));
if (StrUtil.isBlank(response)) {
log.error("update {} Object error, response: {}", body, response);
throw new RuntimeException("update " + body + " Object error, response: " + response);
}
JSONObject jsonObject = JSONUtil.parseObj(response);
if (!"200".equals(jsonObject.get("code").toString())) {
log.error("update {} Object error, response: {}", body, response);
throw new RuntimeException("update " + body + " Object error, response: " + response);
}
}
}

100
src/main/java/com/geedge/scheduler/FqdnScheduler.java
View File

@@ -4,6 +4,8 @@ import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.BooleanUtil;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import com.geedge.common.constant.TsgObject;
import com.geedge.common.util.TsgUtil;
import com.google.common.base.Stopwatch;
@@ -18,10 +20,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.*;
import java.util.concurrent.TimeUnit;
/**
@@ -157,6 +156,22 @@ public class FqdnScheduler {
}
private static void executeUpdate(Integer id, String name, Boolean enable, String sql, Counter counter) {
if (TsgUtil.isLatestVersion) {
executeUpdateLatest(id, name, enable, sql, counter);
} else {
executeUpdateOld(id, name, enable, sql, counter);
}
}
private static void executeDelete(Integer id, Boolean enable, Integer offsetSecond) {
if (TsgUtil.isLatestVersion) {
executeDeleteLatest(id, enable, offsetSecond);
} else {
executeDeleteOld(id, enable, offsetSecond);
}
}
private static void executeUpdateLatest(Integer id, String name, Boolean enable, String sql, Counter counter) {
log.info("{}: started update task.", id);
if (BooleanUtil.isFalse(enable)) {
log.warn("{}: interrupted update task. enable: {}", id, enable);
@@ -212,7 +227,58 @@ public class FqdnScheduler {
}
}
private static void executeDelete(Integer id, Boolean enable, Integer offsetSecond) {
private static void executeUpdateOld(Integer id, String name, Boolean enable, String sql, Counter counter) {
log.info("{}: started update task.", id);
if (BooleanUtil.isFalse(enable)) {
log.warn("{}: interrupted update task. enable: {}", id, enable);
return;
}
try {
Stopwatch watch = Stopwatch.createStarted();
List<Record> data = Db.find(sql);
log.info("{}: query knowledge base content, cost {} seconds", id, watch.elapsed(TimeUnit.SECONDS));
watch.reset().start();
List<Map<String, Object>> items = Lists.newArrayList();
for (Record record : data) {
String domain = record.get("domain");
List<String> keywordArray = new ArrayList<>();
keywordArray.add(domain);
JSONObject obj = new JSONObject();
obj.set("isHexbin", 0);
obj.set(TsgObject.TSG_IS_BUILTIN_V2310, 0);
obj.set(TsgObject.TSG_KEYWORDARRAY_V2310, keywordArray);
items.add(obj);
}
JSONArray array = new JSONArray();
JSONObject jsonObject = new JSONObject();
jsonObject.set(TsgObject.TSG_OBJECT_ID_V2310, id);
jsonObject.set(TsgObject.TSG_OBJECT_TYPE_V2310, TsgObject.TSG_FQDN_OBJECT_V2310);
jsonObject.set(TsgObject.TSG_OBJECT_NAME_V2310, name);
jsonObject.set(TsgObject.TSG_IS_BUILTIN_V2310, 0);
jsonObject.set(TsgObject.TSG_IS_EXCLUSION_V2310, 0);
jsonObject.set(TsgObject.TSG_IS_VALID_V2310, 1);
jsonObject.set("addItemList", items);
log.info("{}: build api params, items size: {}, cost {} seconds", id, items.size(), watch.elapsed(TimeUnit.SECONDS));
if (items.isEmpty()) {
return;
}
array.add(jsonObject);
JSONObject body = new JSONObject();
body.set(TsgObject.TSG_OP_ACTION_V2310, TsgObject.TSG_UPDATE_V2310);
body.set(TsgObject.TSG_OBJECT_LIST_V2310, array);
TsgUtil.updateObjectOld(body);
counter.inc(items.size());
} catch (Exception e) {
log.error("{}: failed to execute update task. message: {}", id, e.getMessage());
throw new RuntimeException(e);
}
}
private static void executeDeleteLatest(Integer id, Boolean enable, Integer offsetSecond) {
log.info("{}: started delete task.", id);
if (BooleanUtil.isFalse(enable)) {
log.warn("{}: interrupted delete task. enable: {}", id, enable);
@@ -231,4 +297,28 @@ public class FqdnScheduler {
throw new RuntimeException(e);
}
}
private static void executeDeleteOld(Integer id, Boolean enable, Integer offsetSecond) {
log.info("{}: started delete task.", id);
if (BooleanUtil.isFalse(enable)) {
log.warn("{}: interrupted delete task. enable: {}", id, enable);
return;
}
try {
DateTime dateTime = DateUtil.offsetSecond(new Date(), offsetSecond).setTimeZone(TimeZone.getTimeZone("UTC"));
String datetimeZ = DateUtil.format(dateTime, DatePattern.UTC_PATTERN);
Map<String, Object> body = new HashMap<>(16);
body.put(TsgObject.TSG_DELETE_ITEMS_BYL_TTIME_V2310, datetimeZ);
body.put(TsgObject.TSG_ITEM_TYPE_V2310, TsgObject.TSG_FQDN_OBJECT_V2310);
body.put(TsgObject.TSG_VSYS_ID_V2310, 1);
List<Object> objectIds = new ArrayList<>();
objectIds.add(id);
body.put(TsgObject.TSG_TSG_OBJECT_IDS_V2310, objectIds);
TsgUtil.deleteItemOfObjectOld(body);
} catch (Exception e) {
log.error("{}: failed to execute delete task. message: {}", id, e.getMessage());
throw new RuntimeException(e);
}
}
}

119
src/main/java/com/geedge/scheduler/IpScheduler.java
View File

@@ -7,6 +7,8 @@ import cn.hutool.core.net.Ipv4Util;
import cn.hutool.core.net.MaskBit;
import cn.hutool.core.util.BooleanUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import com.geedge.common.constant.TsgObject;
import com.geedge.common.enums.AddressFormat;
import com.geedge.common.util.TsgUtil;
@@ -337,8 +339,24 @@ public class IpScheduler {
executeDelete(psiphon3vpnServeripId, psiphon3vpnServeripDeleteEnable, psiphon3vpnServeripOffsetSecond);
}
private static void executeUpdate(Integer id, String name, Boolean enable, String sql, Counter counter) {
if (TsgUtil.isLatestVersion) {
executeUpdateLatest(id, name, enable, sql, counter);
} else {
executeUpdateOld(id, name, enable, sql, counter);
}
}
private static void executeDelete(Integer id, Boolean enable, Integer offsetSecond) {
if (TsgUtil.isLatestVersion) {
executeDeleteLatest(id, enable, offsetSecond);
} else {
executeDeleteOld(id, enable, offsetSecond);
}
}
private static void executeUpdateLatest(Integer id, String name, Boolean enable, String sql, Counter counter) {
log.info("{}: started update task.", id);
if (BooleanUtil.isFalse(enable)) {
log.warn("{}: interrupted update task. enable: {}", id, enable);
@@ -413,7 +431,81 @@ public class IpScheduler {
}
}
private static void executeDelete(Integer id, Boolean enable, Integer offsetSecond) {
private static void executeUpdateOld(Integer id, String name, Boolean enable, String sql, Counter counter) {
log.info("{}: started update task.", id);
if (BooleanUtil.isFalse(enable)) {
log.warn("{}: interrupted update task. enable: {}", id, enable);
return;
}
try {
Stopwatch watch = Stopwatch.createStarted();
List<Record> data = Db.find(sql);
log.info("{}: query knowledge base content, cost {} seconds", id, watch.elapsed(TimeUnit.SECONDS));
watch.reset().start();
List<Map<String, Object>> items = Lists.newArrayList();
row:
for (Record record : data) {
String addressFormat = record.get("addrFormat");
String ip1 = record.get("ip1");
if (StrUtil.isBlank(ip1)) {
continue;
}
for (String excludeItem : excludeList) {
if (excludeItem.equals(ip1)) {
continue row;
}
if (excludeItem.endsWith("*") && ip1.startsWith(excludeItem.replace("*", ""))) {
continue row;
}
}
String ip2 = record.get("ip2");
Map<String, Object> ip = Maps.newHashMap();
if (AddressFormat.CIDR.getValue().equalsIgnoreCase(addressFormat)) {
int maskBit = Integer.parseInt(ip2);
String mask = MaskBit.get(Integer.parseInt(ip2));
String beginIpStr = Ipv4Util.getBeginIpStr(ip1, maskBit);
String ipMask = Ipv4Util.formatIpBlock(beginIpStr, mask);
ip.put(TsgObject.TSG_ITEM_IP_V2310, ipMask);
} else if (AddressFormat.RANGE.getValue().equalsIgnoreCase(addressFormat)) {
ip.put(TsgObject.TSG_ITEM_IP_V2310, ip1 + "-" + ip2);
} else if (AddressFormat.SINGLE.getValue().equalsIgnoreCase(addressFormat)) {
ip.put(TsgObject.TSG_ITEM_IP_V2310, ip1);
} else {
log.warn("address format parse error: {}", record);
continue;
}
ip.put(TsgObject.TSG_ITEM_PORT_V2310, TsgObject.TSG_VALUE_OBJECT_MEMBER_ITEMS_IP_PORT_RANGE_V2310);
ip.put(TsgObject.TSG_ITEM_ISSESSION_V2310, TsgObject.TSG_ITEM_ENDPOINT_V2310);
items.add(ip);
}
JSONArray array = new JSONArray();
JSONObject jsonObject = new JSONObject();
jsonObject.set(TsgObject.TSG_OBJECT_ID_V2310, id);
jsonObject.set(TsgObject.TSG_OBJECT_TYPE_V2310, TsgObject.TSG_IP_ADDR_OBJECT_V2310);
jsonObject.set(TsgObject.TSG_OBJECT_NAME_V2310, name);
jsonObject.set(TsgObject.TSG_IS_BUILTIN_V2310, 0);
jsonObject.set(TsgObject.TSG_IS_EXCLUSION_V2310, 0);
jsonObject.set(TsgObject.TSG_IS_VALID_V2310, 1);
jsonObject.set("addItemList", items);
log.info("{}: build api params, items size: {}, cost {} seconds", id, items.size(), watch.elapsed(TimeUnit.SECONDS));
if (items.isEmpty()) {
return;
}
array.add(jsonObject);
JSONObject body = new JSONObject();
body.set(TsgObject.TSG_OP_ACTION_V2310, TsgObject.TSG_UPDATE_V2310);
body.set(TsgObject.TSG_OBJECT_LIST_V2310, array);
TsgUtil.updateObjectOld(body);
counter.inc(items.size());
} catch (Exception e) {
log.error("{}: failed to execute update task. message: {}", id, e.getMessage());
throw new RuntimeException(e);
}
}
private static void executeDeleteLatest(Integer id, Boolean enable, Integer offsetSecond) {
log.info("{}: started delete task.", id);
if (BooleanUtil.isFalse(enable)) {
log.warn("{}: interrupted delete task. enable: {}", id, enable);
@@ -432,4 +524,27 @@ public class IpScheduler {
throw new RuntimeException(e);
}
}
private static void executeDeleteOld(Integer id, Boolean enable, Integer offsetSecond) {
log.info("{}: started delete task.", id);
if (BooleanUtil.isFalse(enable)) {
log.warn("{}: interrupted delete task. enable: {}", id, enable);
return;
}
try {
DateTime dateTime = DateUtil.offsetSecond(new Date(), offsetSecond).setTimeZone(TimeZone.getTimeZone("UTC"));
String datetimeZ = DateUtil.format(dateTime, DatePattern.UTC_PATTERN);
Map<String, Object> body = new HashMap<>(16);
body.put(TsgObject.TSG_DELETE_ITEMS_BYL_TTIME_V2310, datetimeZ);
body.put(TsgObject.TSG_ITEM_TYPE_V2310, TsgObject.TSG_ITEM_IP_V2310);
body.put(TsgObject.TSG_VSYS_ID_V2310, 1);
List<Object> objectIds = new ArrayList<>();
objectIds.add(id);
body.put(TsgObject.TSG_TSG_OBJECT_IDS_V2310, objectIds);
TsgUtil.deleteItemOfObjectOld(body);
} catch (Exception e) {
log.error("{}: failed to execute delete task. message: {}", id, e.getMessage());
throw new RuntimeException(e);
}
}
}