diff --git a/config/application.yml b/config/application.yml index 96b0dcb..dda11b5 100644 --- a/config/application.yml +++ b/config/application.yml @@ -12,9 +12,11 @@ logging: config: ./config/logback-spring.xml tsg: system: - url: http://192.168.44.3:80 - token: aa2bdec5518ad131f71944b13ce5c298&1& + url: http://192.168.44.29:8080 + ##token: aa2bdec5518ad131f71944b13ce5c298&1& + token: 1ca0350bc499a208a0edadb74d76a0ee&1& httpTimeout: 3600000 + isLatestVersion: false object: ip: #指定排除的IP. 1.如果没有可不写值，2.若有多个用英文逗号,分隔如: 127.0.0.1,127.0.0.2，3.可适用IP段匹配，如：127.* diff --git a/src/main/java/com/geedge/common/constant/TsgObject.java b/src/main/java/com/geedge/common/constant/TsgObject.java index 9a01eda..a672d29 100644 --- a/src/main/java/com/geedge/common/constant/TsgObject.java +++ b/src/main/java/com/geedge/common/constant/TsgObject.java @@ -38,5 +38,27 @@ public class TsgObject { public static final Integer VALUE_OBJECT_MEMBER_ITEMS_STRING_EXPR_TYPE_0 = 0; public static final Integer VALUE_OBJECT_MEMBER_ITEMS_STRING_IS_HEXBIN_0 = 0; + public static String TSG_OBJECT_ID_V2310 = "objectId"; + public static String TSG_OBJECT_TYPE_V2310 = "objectType"; + public static String TSG_IP_ADDR_OBJECT_V2310 = "ip"; + public static String TSG_FQDN_OBJECT_V2310 = "fqdn"; + public static String TSG_OBJECT_NAME_V2310 = "objectName"; + public static String TSG_IS_BUILTIN_V2310 = "isBuiltin"; + public static String TSG_IS_EXCLUSION_V2310 = "isExclusion"; + public static String TSG_IS_VALID_V2310 = "isValid"; + public static String TSG_ITEM_IP_V2310 = "ip"; + public static String TSG_ITEM_PORT_V2310 = "port"; + public static String TSG_VALUE_OBJECT_MEMBER_ITEMS_IP_PORT_RANGE_V2310 = "0-65535"; + public static String TSG_ITEM_ISSESSION_V2310 = "isSession"; + public static String TSG_ITEM_ENDPOINT_V2310 = "endpoint"; + public static String TSG_OP_ACTION_V2310 = "opAction"; + public static String TSG_UPDATE_V2310 = "update"; + public static String TSG_OBJECT_LIST_V2310 = "objectList"; + public static String TSG_VSYS_ID_V2310 = "vsysId"; + public static String TSG_TSG_OBJECT_IDS_V2310 = "objectIds"; + public static String TSG_ITEM_TYPE_V2310 = "itemType"; + public static String TSG_KEYWORDARRAY_V2310 = "keywordArray"; + public static String TSG_DELETE_ITEMS_BYL_TTIME_V2310 = "deleteItemsByLtTime"; + } diff --git a/src/main/java/com/geedge/common/PrometheusMetrics.java b/src/main/java/com/geedge/common/monitor/PrometheusMetrics.java similarity index 94% rename from src/main/java/com/geedge/common/PrometheusMetrics.java rename to src/main/java/com/geedge/common/monitor/PrometheusMetrics.java index 57beb83..9c6d079 100644 --- a/src/main/java/com/geedge/common/PrometheusMetrics.java +++ b/src/main/java/com/geedge/common/monitor/PrometheusMetrics.java @@ -1,4 +1,4 @@ -package com.geedge.common; +package com.geedge.common.monitor; import io.prometheus.client.exporter.MetricsServlet; import org.springframework.web.bind.annotation.GetMapping; diff --git a/src/main/java/com/geedge/common/util/TsgUtil.java b/src/main/java/com/geedge/common/util/TsgUtil.java index ac0d11b..3e5b995 100644 --- a/src/main/java/com/geedge/common/util/TsgUtil.java +++ b/src/main/java/com/geedge/common/util/TsgUtil.java @@ -6,7 +6,6 @@ import cn.hutool.http.HttpRequest; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; import com.google.common.base.Stopwatch; -import com.google.common.collect.Maps; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -26,8 +25,14 @@ import java.util.concurrent.TimeUnit; public class TsgUtil { public static String TSG_URL; - private static Integer httpTimeOut; private static String TSG_TOKEN; + private static Integer httpTimeOut; + public static Boolean isLatestVersion; + + @Value("${tsg.system.isLatestVersion}") + public void isLatestVersion(Boolean latestVersion) { + isLatestVersion = latestVersion; + } @Value("${tsg.system.httpTimeout}") public void setTsgUrl(Integer timeOut) { @@ -71,6 +76,25 @@ public class TsgUtil { } } + public static void updateObjectOld(Map body) { + Stopwatch watch = Stopwatch.createStarted(); + String response = HttpRequest.put(TSG_URL + "/v1/policy/object") + .header(Header.AUTHORIZATION, getToken()) + .body(JSONUtil.toJsonStr(body)) + .timeout(httpTimeOut) + .execute().body(); + log.info("update tsg-api, cost {} seconds", watch.elapsed(TimeUnit.SECONDS)); + if (StrUtil.isBlank(response)) { + log.error("update {} Object error, response: {}", body, response); + throw new RuntimeException("update " + body + " Object error, response: " + response); + } + JSONObject jsonObject = JSONUtil.parseObj(response); + if (!"200".equals(jsonObject.get("code").toString())) { + log.error("update {} Object error, response: {}", body, response); + throw new RuntimeException("update " + body + " Object error, response: " + response); + } + } + public static void deleteItemOfObjectById(Integer id, Map form) { Stopwatch watch = Stopwatch.createStarted(); String response = HttpRequest.delete(TSG_URL + "/v1/policy/object/" + id + "/item") @@ -89,4 +113,23 @@ public class TsgUtil { throw new RuntimeException("update " + id + " Object error, response: " + response); } } + + public static void deleteItemOfObjectOld(Map body) { + Stopwatch watch = Stopwatch.createStarted(); + String response = HttpRequest.delete(TSG_URL + "/v1/policy/items") + .header(Header.AUTHORIZATION, getToken()) + .body(JSONUtil.toJsonStr(body)) + .timeout(httpTimeOut) + .execute().body(); + log.info("delete tsg-api, cost {} seconds", watch.elapsed(TimeUnit.SECONDS)); + if (StrUtil.isBlank(response)) { + log.error("update {} Object error, response: {}", body, response); + throw new RuntimeException("update " + body + " Object error, response: " + response); + } + JSONObject jsonObject = JSONUtil.parseObj(response); + if (!"200".equals(jsonObject.get("code").toString())) { + log.error("update {} Object error, response: {}", body, response); + throw new RuntimeException("update " + body + " Object error, response: " + response); + } + } } diff --git a/src/main/java/com/geedge/scheduler/FqdnScheduler.java b/src/main/java/com/geedge/scheduler/FqdnScheduler.java index ec34a6e..b92c6a1 100644 --- a/src/main/java/com/geedge/scheduler/FqdnScheduler.java +++ b/src/main/java/com/geedge/scheduler/FqdnScheduler.java @@ -4,6 +4,8 @@ import cn.hutool.core.date.DatePattern; import cn.hutool.core.date.DateTime; import cn.hutool.core.date.DateUtil; import cn.hutool.core.util.BooleanUtil; +import cn.hutool.json.JSONArray; +import cn.hutool.json.JSONObject; import com.geedge.common.constant.TsgObject; import com.geedge.common.util.TsgUtil; import com.google.common.base.Stopwatch; @@ -18,10 +20,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; +import java.util.*; import java.util.concurrent.TimeUnit; /** @@ -157,6 +156,22 @@ public class FqdnScheduler { } private static void executeUpdate(Integer id, String name, Boolean enable, String sql, Counter counter) { + if (TsgUtil.isLatestVersion) { + executeUpdateLatest(id, name, enable, sql, counter); + } else { + executeUpdateOld(id, name, enable, sql, counter); + } + } + + private static void executeDelete(Integer id, Boolean enable, Integer offsetSecond) { + if (TsgUtil.isLatestVersion) { + executeDeleteLatest(id, enable, offsetSecond); + } else { + executeDeleteOld(id, enable, offsetSecond); + } + } + + private static void executeUpdateLatest(Integer id, String name, Boolean enable, String sql, Counter counter) { log.info("{}: started update task.", id); if (BooleanUtil.isFalse(enable)) { log.warn("{}: interrupted update task. enable: {}", id, enable); @@ -212,7 +227,58 @@ public class FqdnScheduler { } } - private static void executeDelete(Integer id, Boolean enable, Integer offsetSecond) { + + private static void executeUpdateOld(Integer id, String name, Boolean enable, String sql, Counter counter) { + log.info("{}: started update task.", id); + if (BooleanUtil.isFalse(enable)) { + log.warn("{}: interrupted update task. enable: {}", id, enable); + return; + } + try { + Stopwatch watch = Stopwatch.createStarted(); + List data = Db.find(sql); + log.info("{}: query knowledge base content, cost {} seconds", id, watch.elapsed(TimeUnit.SECONDS)); + watch.reset().start(); + + List> items = Lists.newArrayList(); + for (Record record : data) { + String domain = record.get("domain"); + List keywordArray = new ArrayList<>(); + keywordArray.add(domain); + JSONObject obj = new JSONObject(); + obj.set("isHexbin", 0); + obj.set(TsgObject.TSG_IS_BUILTIN_V2310, 0); + obj.set(TsgObject.TSG_KEYWORDARRAY_V2310, keywordArray); + items.add(obj); + } + + JSONArray array = new JSONArray(); + JSONObject jsonObject = new JSONObject(); + jsonObject.set(TsgObject.TSG_OBJECT_ID_V2310, id); + jsonObject.set(TsgObject.TSG_OBJECT_TYPE_V2310, TsgObject.TSG_FQDN_OBJECT_V2310); + jsonObject.set(TsgObject.TSG_OBJECT_NAME_V2310, name); + jsonObject.set(TsgObject.TSG_IS_BUILTIN_V2310, 0); + jsonObject.set(TsgObject.TSG_IS_EXCLUSION_V2310, 0); + jsonObject.set(TsgObject.TSG_IS_VALID_V2310, 1); + jsonObject.set("addItemList", items); + log.info("{}: build api params, items size: {}, cost {} seconds", id, items.size(), watch.elapsed(TimeUnit.SECONDS)); + if (items.isEmpty()) { + return; + } + array.add(jsonObject); + JSONObject body = new JSONObject(); + body.set(TsgObject.TSG_OP_ACTION_V2310, TsgObject.TSG_UPDATE_V2310); + body.set(TsgObject.TSG_OBJECT_LIST_V2310, array); + TsgUtil.updateObjectOld(body); + counter.inc(items.size()); + } catch (Exception e) { + log.error("{}: failed to execute update task. message: {}", id, e.getMessage()); + throw new RuntimeException(e); + } + } + + + private static void executeDeleteLatest(Integer id, Boolean enable, Integer offsetSecond) { log.info("{}: started delete task.", id); if (BooleanUtil.isFalse(enable)) { log.warn("{}: interrupted delete task. enable: {}", id, enable); @@ -231,4 +297,28 @@ public class FqdnScheduler { throw new RuntimeException(e); } } + + private static void executeDeleteOld(Integer id, Boolean enable, Integer offsetSecond) { + log.info("{}: started delete task.", id); + if (BooleanUtil.isFalse(enable)) { + log.warn("{}: interrupted delete task. enable: {}", id, enable); + return; + } + try { + DateTime dateTime = DateUtil.offsetSecond(new Date(), offsetSecond).setTimeZone(TimeZone.getTimeZone("UTC")); + String datetimeZ = DateUtil.format(dateTime, DatePattern.UTC_PATTERN); + Map body = new HashMap<>(16); + body.put(TsgObject.TSG_DELETE_ITEMS_BYL_TTIME_V2310, datetimeZ); + body.put(TsgObject.TSG_ITEM_TYPE_V2310, TsgObject.TSG_FQDN_OBJECT_V2310); + body.put(TsgObject.TSG_VSYS_ID_V2310, 1); + List objectIds = new ArrayList<>(); + objectIds.add(id); + body.put(TsgObject.TSG_TSG_OBJECT_IDS_V2310, objectIds); + TsgUtil.deleteItemOfObjectOld(body); + } catch (Exception e) { + log.error("{}: failed to execute delete task. message: {}", id, e.getMessage()); + throw new RuntimeException(e); + } + } + } diff --git a/src/main/java/com/geedge/scheduler/IpScheduler.java b/src/main/java/com/geedge/scheduler/IpScheduler.java index dcf7fef..bd41728 100644 --- a/src/main/java/com/geedge/scheduler/IpScheduler.java +++ b/src/main/java/com/geedge/scheduler/IpScheduler.java @@ -7,6 +7,8 @@ import cn.hutool.core.net.Ipv4Util; import cn.hutool.core.net.MaskBit; import cn.hutool.core.util.BooleanUtil; import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSONArray; +import cn.hutool.json.JSONObject; import com.geedge.common.constant.TsgObject; import com.geedge.common.enums.AddressFormat; import com.geedge.common.util.TsgUtil; @@ -337,8 +339,24 @@ public class IpScheduler { executeDelete(psiphon3vpnServeripId, psiphon3vpnServeripDeleteEnable, psiphon3vpnServeripOffsetSecond); } - private static void executeUpdate(Integer id, String name, Boolean enable, String sql, Counter counter) { + if (TsgUtil.isLatestVersion) { + executeUpdateLatest(id, name, enable, sql, counter); + } else { + executeUpdateOld(id, name, enable, sql, counter); + } + } + + private static void executeDelete(Integer id, Boolean enable, Integer offsetSecond) { + if (TsgUtil.isLatestVersion) { + executeDeleteLatest(id, enable, offsetSecond); + } else { + executeDeleteOld(id, enable, offsetSecond); + } + } + + + private static void executeUpdateLatest(Integer id, String name, Boolean enable, String sql, Counter counter) { log.info("{}: started update task.", id); if (BooleanUtil.isFalse(enable)) { log.warn("{}: interrupted update task. enable: {}", id, enable); @@ -413,7 +431,81 @@ public class IpScheduler { } } - private static void executeDelete(Integer id, Boolean enable, Integer offsetSecond) { + private static void executeUpdateOld(Integer id, String name, Boolean enable, String sql, Counter counter) { + log.info("{}: started update task.", id); + if (BooleanUtil.isFalse(enable)) { + log.warn("{}: interrupted update task. enable: {}", id, enable); + return; + } + try { + Stopwatch watch = Stopwatch.createStarted(); + List data = Db.find(sql); + log.info("{}: query knowledge base content, cost {} seconds", id, watch.elapsed(TimeUnit.SECONDS)); + watch.reset().start(); + List> items = Lists.newArrayList(); + row: + for (Record record : data) { + String addressFormat = record.get("addrFormat"); + String ip1 = record.get("ip1"); + if (StrUtil.isBlank(ip1)) { + continue; + } + for (String excludeItem : excludeList) { + if (excludeItem.equals(ip1)) { + continue row; + } + if (excludeItem.endsWith("*") && ip1.startsWith(excludeItem.replace("*", ""))) { + continue row; + } + } + String ip2 = record.get("ip2"); + Map ip = Maps.newHashMap(); + if (AddressFormat.CIDR.getValue().equalsIgnoreCase(addressFormat)) { + int maskBit = Integer.parseInt(ip2); + String mask = MaskBit.get(Integer.parseInt(ip2)); + String beginIpStr = Ipv4Util.getBeginIpStr(ip1, maskBit); + String ipMask = Ipv4Util.formatIpBlock(beginIpStr, mask); + ip.put(TsgObject.TSG_ITEM_IP_V2310, ipMask); + } else if (AddressFormat.RANGE.getValue().equalsIgnoreCase(addressFormat)) { + ip.put(TsgObject.TSG_ITEM_IP_V2310, ip1 + "-" + ip2); + } else if (AddressFormat.SINGLE.getValue().equalsIgnoreCase(addressFormat)) { + ip.put(TsgObject.TSG_ITEM_IP_V2310, ip1); + } else { + log.warn("address format parse error: {}", record); + continue; + } + ip.put(TsgObject.TSG_ITEM_PORT_V2310, TsgObject.TSG_VALUE_OBJECT_MEMBER_ITEMS_IP_PORT_RANGE_V2310); + ip.put(TsgObject.TSG_ITEM_ISSESSION_V2310, TsgObject.TSG_ITEM_ENDPOINT_V2310); + items.add(ip); + } + JSONArray array = new JSONArray(); + + JSONObject jsonObject = new JSONObject(); + jsonObject.set(TsgObject.TSG_OBJECT_ID_V2310, id); + jsonObject.set(TsgObject.TSG_OBJECT_TYPE_V2310, TsgObject.TSG_IP_ADDR_OBJECT_V2310); + jsonObject.set(TsgObject.TSG_OBJECT_NAME_V2310, name); + jsonObject.set(TsgObject.TSG_IS_BUILTIN_V2310, 0); + jsonObject.set(TsgObject.TSG_IS_EXCLUSION_V2310, 0); + jsonObject.set(TsgObject.TSG_IS_VALID_V2310, 1); + jsonObject.set("addItemList", items); + log.info("{}: build api params, items size: {}, cost {} seconds", id, items.size(), watch.elapsed(TimeUnit.SECONDS)); + if (items.isEmpty()) { + return; + } + array.add(jsonObject); + JSONObject body = new JSONObject(); + body.set(TsgObject.TSG_OP_ACTION_V2310, TsgObject.TSG_UPDATE_V2310); + body.set(TsgObject.TSG_OBJECT_LIST_V2310, array); + + TsgUtil.updateObjectOld(body); + counter.inc(items.size()); + } catch (Exception e) { + log.error("{}: failed to execute update task. message: {}", id, e.getMessage()); + throw new RuntimeException(e); + } + } + + private static void executeDeleteLatest(Integer id, Boolean enable, Integer offsetSecond) { log.info("{}: started delete task.", id); if (BooleanUtil.isFalse(enable)) { log.warn("{}: interrupted delete task. enable: {}", id, enable); @@ -432,4 +524,27 @@ public class IpScheduler { throw new RuntimeException(e); } } + + private static void executeDeleteOld(Integer id, Boolean enable, Integer offsetSecond) { + log.info("{}: started delete task.", id); + if (BooleanUtil.isFalse(enable)) { + log.warn("{}: interrupted delete task. enable: {}", id, enable); + return; + } + try { + DateTime dateTime = DateUtil.offsetSecond(new Date(), offsetSecond).setTimeZone(TimeZone.getTimeZone("UTC")); + String datetimeZ = DateUtil.format(dateTime, DatePattern.UTC_PATTERN); + Map body = new HashMap<>(16); + body.put(TsgObject.TSG_DELETE_ITEMS_BYL_TTIME_V2310, datetimeZ); + body.put(TsgObject.TSG_ITEM_TYPE_V2310, TsgObject.TSG_ITEM_IP_V2310); + body.put(TsgObject.TSG_VSYS_ID_V2310, 1); + List objectIds = new ArrayList<>(); + objectIds.add(id); + body.put(TsgObject.TSG_TSG_OBJECT_IDS_V2310, objectIds); + TsgUtil.deleteItemOfObjectOld(body); + } catch (Exception e) { + log.error("{}: failed to execute delete task. message: {}", id, e.getMessage()); + throw new RuntimeException(e); + } + } }