diff --git a/src/main/java/com/nis/util/JedisUtils.java b/src/main/java/com/nis/util/JedisUtils.java index 4c1d949..86f59fe 100644 --- a/src/main/java/com/nis/util/JedisUtils.java +++ b/src/main/java/com/nis/util/JedisUtils.java @@ -306,9 +306,10 @@ public class JedisUtils { * @return 是否获取成功 */ public static Boolean lock(String requestId) { + Jedis resource = getResource(0); String key = "redisDistributedLock"; - String var1 = getResource(0).set(key, requestId, SET_IF_NOT_EXIST, SET_WITH_EXPIRE_TIME, - Constants.REDISLOCKTIME); + String var1 = resource.set(key, requestId, SET_IF_NOT_EXIST, SET_WITH_EXPIRE_TIME, Constants.REDISLOCKTIME); + returnResource(resource); if (LOCK_SUCCESS.equals(var1)) { return true; } @@ -322,11 +323,13 @@ public class JedisUtils { * @return */ public static Boolean unLock(String value) { + Jedis resource = getResource(0); String key = "redisDistributedLock"; // 这个字符串是个lua脚本，代表的意思是如果根据key拿到的value跟传入的value相同就执行del，否则就返回0【保证安全性】 String luaScript = "if redis.call(\"get\",KEYS[1]) == ARGV[1] then return redis.call(\"del\",KEYS[1]) else return 0 end"; // 这个命令就是去执行lua脚本，KEYS的集合就是第二个参数，ARGV的集合就是第三参数【保证解锁的原子操作】 - Object var2 = getResource(0).eval(luaScript, Collections.singletonList(key), Collections.singletonList(value)); + Object var2 = resource.eval(luaScript, Collections.singletonList(key), Collections.singletonList(value)); + returnResource(resource); if (UNLOCK_SUCCESS == var2) { return true; } diff --git a/src/main/java/com/nis/web/controller/restful/ConfigSourcesController.java b/src/main/java/com/nis/web/controller/restful/ConfigSourcesController.java index be4e954..ad5c56c 100644 --- a/src/main/java/com/nis/web/controller/restful/ConfigSourcesController.java +++ b/src/main/java/com/nis/web/controller/restful/ConfigSourcesController.java @@ -18,6 +18,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; @@ -30,6 +31,7 @@ import com.nis.restful.RestBusinessCode; import com.nis.restful.RestServiceException; import com.nis.restful.ServiceRuntimeException; import com.nis.util.Constants; +import com.nis.util.ExceptionUtil; import com.nis.util.FileUtils; import com.nis.util.MinioUtil; import com.nis.util.StringUtils; @@ -81,9 +83,10 @@ public class ConfigSourcesController extends BaseRestController { if (getLock(requestId)) { if (null != configSource && null != configSource.getConfigCompileList() && configSource.getConfigCompileList().size() > 0) { - if (configSource.getConfigCompileList()!=null&&configSource.getConfigCompileList().size()>Constants.MAX_LIST_SIZE) { - thread.setContent("批量提交数量超过"+Constants.MAX_LIST_SIZE+"条,不记录请求内容"); - } + if (configSource.getConfigCompileList() != null + && configSource.getConfigCompileList().size() > Constants.MAX_LIST_SIZE) { + thread.setContent("批量提交数量超过" + Constants.MAX_LIST_SIZE + "条,不记录请求内容"); + } checkOpAction(thread, System.currentTimeMillis() - start, configSource.getOpAction(), Constants.OPACTION_POST); // 验证配置编译数据 @@ -134,8 +137,9 @@ public class ConfigSourcesController extends BaseRestController { if (null != configSource && null != configSource.getConfigCompileList() && configSource.getConfigCompileList().size() > 0) { int opAction = configSource.getOpAction(); - if (configSource.getConfigCompileList()!=null&&configSource.getConfigCompileList().size()>Constants.MAX_LIST_SIZE) { - thread.setContent("批量提交数量超过"+Constants.MAX_LIST_SIZE+"条,不记录请求内容"); + if (configSource.getConfigCompileList() != null + && configSource.getConfigCompileList().size() > Constants.MAX_LIST_SIZE) { + thread.setContent("批量提交数量超过" + Constants.MAX_LIST_SIZE + "条,不记录请求内容"); } checkOpAction(thread, System.currentTimeMillis() - start, opAction, 2); configSourcesService.updateConfigSources(thread, start, configSource.getConfigCompileList(), @@ -551,6 +555,28 @@ public class ConfigSourcesController extends BaseRestController { "MAAT规则分组复用域配置删除成功" + sb.toString(), Constants.IS_DEBUG ? groupReuseSource : null); } + @RequestMapping(value = "/cfg/v1/getAllKVByCompileId", method = RequestMethod.GET) + @ApiOperation(value = "根据配置id获取对应的编译,组,域等信息", httpMethod = "GET", response = Map.class, notes = "根据配置id获取对应的编译,组,域等信息") + @ApiParam(value = "配置id", name = "getAllKVByCompileId", required = true) + public Map getAllKVByCompileId(@RequestParam Long compileId, HttpServletRequest request, + HttpServletResponse response) { + long start = System.currentTimeMillis(); + AuditLogThread thread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + compileId); + Map map=null; + try { + map=configRedisService.getAllKVByCompileId(compileId); + } catch (Exception e) { + thread.setExceptionInfo("根据配置id获取对应的编译,组,域等信息时出现异常:" + e.getMessage()); + logger.error("根据配置id获取对应的编译,组,域等信息时出现异常:" +ExceptionUtil.getExceptionMsg(e)); + throw new RestServiceException("根据配置id获取对应的编译,组,域等信息失败", + RestBusinessCode.ConfigSourceIsNull.getValue()); + + } + return compileServiceResponse(thread, System.currentTimeMillis() - start, request, response, + "根据配置id获取对应的编译,组,域等信息成功", map); + } + private void validateGroupReuseSource(AuditLogThread thread, long start, GroupReuseSource groupReuseSource) { String errorInfo = ""; diff --git a/src/main/java/com/nis/web/controller/restful/DashboardServiceController.java b/src/main/java/com/nis/web/controller/restful/DashboardServiceController.java index d97dd14..2f261a5 100644 --- a/src/main/java/com/nis/web/controller/restful/DashboardServiceController.java +++ b/src/main/java/com/nis/web/controller/restful/DashboardServiceController.java @@ -166,12 +166,12 @@ public class DashboardServiceController extends BaseRestController { */ @RequestMapping(value="trafficProtocolList", method = RequestMethod.GET) @ApiOperation(value = "协议统计占比与报表", httpMethod = "GET", notes = "对应协议统计详情占比与报表") - public Map trafficProtocolList(Model model, HttpServletRequest request, HttpServletResponse response){ + public Map trafficProtocolList(String searchFoundStartTime,String searchFoundEndTime,Model model, HttpServletRequest request, HttpServletResponse response){ long start = System.currentTimeMillis(); AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); List list = new ArrayList(); try { - List ipActiveList = dashboardService.getProtocolList(); + List ipActiveList = dashboardService.getProtocolList(searchFoundStartTime,searchFoundEndTime); if (ipActiveList!=null&&ipActiveList.size() > 0) { String jsonString = JsonMapper.toJsonString(ipActiveList); list = (java.util.List) JsonMapper.fromJsonList(jsonString,HashMap.class); @@ -289,12 +289,12 @@ public class DashboardServiceController extends BaseRestController { */ @RequestMapping(value="trafficAppList", method = RequestMethod.GET) @ApiOperation(value = "App统计占比与报表", httpMethod = "GET", notes = "对应App统计详情占比与报表") - public Map trafficAppList(Model model, HttpServletRequest request, HttpServletResponse response){ + public Map trafficAppList(String searchFoundStartTime,String searchFoundEndTime,Model model, HttpServletRequest request, HttpServletResponse response){ long start = System.currentTimeMillis(); AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); List list = new ArrayList(); try { - List appList = dashboardService.getAppList(); + List appList = dashboardService.getAppList(searchFoundStartTime,searchFoundEndTime); if (appList!=null&&appList.size() > 0) { String jsonString = JsonMapper.toJsonString(appList); list = (java.util.List) JsonMapper.fromJsonList(jsonString,HashMap.class); diff --git a/src/main/java/com/nis/web/controller/restful/NmsInfoController.java b/src/main/java/com/nis/web/controller/restful/NmsInfoController.java index c0d9415..cc3d9ca 100644 --- a/src/main/java/com/nis/web/controller/restful/NmsInfoController.java +++ b/src/main/java/com/nis/web/controller/restful/NmsInfoController.java @@ -55,6 +55,7 @@ public class NmsInfoController extends BaseRestController { if (trafficNmsServerStatisticList != null && trafficNmsServerStatisticList.getTrafficNmsServerList() != null && trafficNmsServerStatisticList.getTrafficNmsServerList().size() > 0) { + thread.setContent("NMS系统上报服务器数量为"+trafficNmsServerStatisticList.getTrafficNmsServerList().size()); trafficNmsServerStatisticService.saveNmsInfo(trafficNmsServerStatisticList.getTrafficNmsServerList()); } else { throw new RestServiceException(thread, System.currentTimeMillis() - start, "参数trafficNmsServerList不能为空", @@ -71,7 +72,8 @@ public class NmsInfoController extends BaseRestController { } } - return testServiceResponse(System.currentTimeMillis() - start, request, response, "上报服务器状态信息成功", + + return compileServiceResponse(thread, System.currentTimeMillis() - start, request, response, "上报服务器状态信息成功", Constants.IS_DEBUG ? trafficNmsServerStatisticList : null); } @@ -86,6 +88,7 @@ public class NmsInfoController extends BaseRestController { try { if (nmsDiRuleList != null && nmsDiRuleList.getNmsDiRuleList() != null && nmsDiRuleList.getNmsDiRuleList().size() > 0) { + thread.setContent("NMS系统上报的di信息数量为"+nmsDiRuleList.getNmsDiRuleList().size()); nmsDiRuleService.saveNmsDiRuleInfo(nmsDiRuleList.getNmsDiRuleList()); } else { throw new RestServiceException(thread, System.currentTimeMillis() - start, "参数nmsDiRuleList不能为空", @@ -100,7 +103,7 @@ public class NmsInfoController extends BaseRestController { "上报NmsDiRule信息异常:" + e.getMessage(), RestBusinessCode.unknow_error.getValue()); } } - return testServiceResponse(System.currentTimeMillis() - start, request, response, "上报NmsDiRule信息成功", + return compileServiceResponse(thread, System.currentTimeMillis() - start, request, response, "上报NmsDiRule信息成功", Constants.IS_DEBUG ? nmsDiRuleList : null); } @@ -116,6 +119,7 @@ public class NmsInfoController extends BaseRestController { try { if (trafficNetflowPortInfoList != null && trafficNetflowPortInfoList.getTrafficNetflowPortInfoList() != null && trafficNetflowPortInfoList.getTrafficNetflowPortInfoList().size() > 0) { + thread.setContent("NMS系统上报上报trafficNetflowPort信息数量为"+trafficNetflowPortInfoList.getTrafficNetflowPortInfoList().size()); trafficNetflowPortInfoService .saveTrafficNetflowPortInfo(trafficNetflowPortInfoList.getTrafficNetflowPortInfoList()); } else { @@ -131,7 +135,7 @@ public class NmsInfoController extends BaseRestController { "上报trafficNetflowPort信息异常:" + e.getMessage(), RestBusinessCode.unknow_error.getValue()); } } - return testServiceResponse(System.currentTimeMillis() - start, request, response, + return compileServiceResponse(thread, System.currentTimeMillis() - start, request, response, "上报trafficNetflowPort信息成功", Constants.IS_DEBUG ? trafficNetflowPortInfoList : null); } diff --git a/src/main/java/com/nis/web/dao/dashboard/NtcTotalReportDao.xml b/src/main/java/com/nis/web/dao/dashboard/NtcTotalReportDao.xml index f122204..f5fd6fe 100644 --- a/src/main/java/com/nis/web/dao/dashboard/NtcTotalReportDao.xml +++ b/src/main/java/com/nis/web/dao/dashboard/NtcTotalReportDao.xml @@ -16,24 +16,24 @@ - SELECT MAX(report_time) reportTime FROM ntc_entrance_report + SELECT report_time reportTime FROM ntc_entrance_report order by report_time desc limit 1 - SELECT MAX(stat_time) statTime FROM traffic_app_statistic WHERE app_type != 0 + SELECT stat_time statTime FROM traffic_app_statistic order by stat_time desc limit 1 - SELECT app_type appType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_app_statistic - WHERE app_type !=0 and stat_time = #{statTime} + WHERE stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour) GROUP BY app_type order by count desc limit 0,10 + \ No newline at end of file diff --git a/src/main/java/com/nis/web/dao/dashboard/TrafficHttpStatisticDao.xml b/src/main/java/com/nis/web/dao/dashboard/TrafficHttpStatisticDao.xml index f0ab688..54f3684 100644 --- a/src/main/java/com/nis/web/dao/dashboard/TrafficHttpStatisticDao.xml +++ b/src/main/java/com/nis/web/dao/dashboard/TrafficHttpStatisticDao.xml @@ -17,31 +17,32 @@ - SELECT SUM(c2s_byte_len+s2c_byte_len) count, IFNULL( website_service_id, 268435455 ) websiteServiceId FROM TRAFFIC_HTTP_STATISTIC t LEFT JOIN ui_website_domain_topic u ON t.web_id = u.id - where t.web_id!=0 - and t.stat_time = #{statTime} + where t.stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour) GROUP BY u.website_service_id ORDER BY count desc limit 0,10 - - + + + @@ -56,7 +57,7 @@ FROM traffic_http_statistic t LEFT JOIN ui_website_domain_topic u ON t.web_id=u.id where u.website_service_id=#{websiteServiceId} - and t.stat_time =#{statTime} + and t.stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour) GROUP BY t.web_id ORDER BY count desc limit 0,10 @@ -65,7 +66,7 @@ FROM traffic_http_statistic t LEFT JOIN ui_website_domain_topic u ON t.web_id=u.id where u.topic_id=#{topicId} - and t.stat_time = #{statTime} + and t.stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour) GROUP BY t.web_id ORDER BY count desc limit 0,10 @@ -74,13 +75,13 @@ SELECT SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_http_statistic t LEFT JOIN ui_website_domain_topic u ON t.web_id=u.id where u.website_service_id=#{websiteServiceId} - and t.web_id!=0 + and t.web_id not in #{singleType} - and stat_time =#{statTime} + and stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour) \ No newline at end of file diff --git a/src/main/java/com/nis/web/dao/dashboard/TrafficIpActiveStatisticDao.xml b/src/main/java/com/nis/web/dao/dashboard/TrafficIpActiveStatisticDao.xml index bd5bf13..84f3821 100644 --- a/src/main/java/com/nis/web/dao/dashboard/TrafficIpActiveStatisticDao.xml +++ b/src/main/java/com/nis/web/dao/dashboard/TrafficIpActiveStatisticDao.xml @@ -24,12 +24,12 @@ --> diff --git a/src/main/java/com/nis/web/dao/dashboard/TrafficPortActiveStatisticDao.xml b/src/main/java/com/nis/web/dao/dashboard/TrafficPortActiveStatisticDao.xml index 688cdb0..6fbdf04 100644 --- a/src/main/java/com/nis/web/dao/dashboard/TrafficPortActiveStatisticDao.xml +++ b/src/main/java/com/nis/web/dao/dashboard/TrafficPortActiveStatisticDao.xml @@ -11,18 +11,19 @@ \ No newline at end of file diff --git a/src/main/java/com/nis/web/dao/dashboard/TrafficProtocolStatisticDao.java b/src/main/java/com/nis/web/dao/dashboard/TrafficProtocolStatisticDao.java index 4db61e8..ccafd43 100644 --- a/src/main/java/com/nis/web/dao/dashboard/TrafficProtocolStatisticDao.java +++ b/src/main/java/com/nis/web/dao/dashboard/TrafficProtocolStatisticDao.java @@ -13,5 +13,5 @@ public interface TrafficProtocolStatisticDao { TrafficProtocolStatistic getMaxStatTime(); List protocolChart(@Param("statTime")Date statTime); - List getProtocolList(@Param("statTime")Date statTime); + List getProtocolList(@Param("statTime")Date statTime,@Param("beginTime")String beginTime,@Param("endTime")String endTime); } \ No newline at end of file diff --git a/src/main/java/com/nis/web/dao/dashboard/TrafficProtocolStatisticDao.xml b/src/main/java/com/nis/web/dao/dashboard/TrafficProtocolStatisticDao.xml index 0f6f16c..2d27d3a 100644 --- a/src/main/java/com/nis/web/dao/dashboard/TrafficProtocolStatisticDao.xml +++ b/src/main/java/com/nis/web/dao/dashboard/TrafficProtocolStatisticDao.xml @@ -17,18 +17,25 @@ - + SELECT proto_type protoType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_protocol_statistic WHERE + stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour) GROUP BY proto_type ORDER BY count DESC LIMIT 0, 10 \ No newline at end of file diff --git a/src/main/java/com/nis/web/dao/dashboard/TrafficUaStatisticDao.xml b/src/main/java/com/nis/web/dao/dashboard/TrafficUaStatisticDao.xml index 2f16fdf..6d37544 100644 --- a/src/main/java/com/nis/web/dao/dashboard/TrafficUaStatisticDao.xml +++ b/src/main/java/com/nis/web/dao/dashboard/TrafficUaStatisticDao.xml @@ -18,24 +18,25 @@ - SELECT os_type osType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic - WHERE os_type !=0 and stat_time = #{statTime} + WHERE stat_time > DATE_SUB(#{statTime}, INTERVAL 1 HOUR) GROUP BY os_type ORDER BY count DESC limit 0,10 @@ -48,25 +49,24 @@ #{singleType} - and stat_time = #{statTime} + and stat_time > DATE_SUB(#{statTime},INTERVAL 1 HOUR) - - SELECT os_type osType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic - WHERE os_type !=0 and bs_type=#{bsType} and stat_time = #{statTime} + WHERE bs_type=#{bsType} and stat_time > DATE_SUB(#{statTime},INTERVAL 1 HOUR) GROUP BY os_type ORDER BY count DESC limit 0,10 @@ -79,6 +79,6 @@ #{singleType} - and stat_time = #{statTime} + and stat_time > DATE_SUB(#{statTime},INTERVAL 1 HOUR) \ No newline at end of file diff --git a/src/main/java/com/nis/web/service/restful/ConfigJedisServiceimpl.java b/src/main/java/com/nis/web/service/restful/ConfigJedisServiceimpl.java index 5655265..da0b3b2 100644 --- a/src/main/java/com/nis/web/service/restful/ConfigJedisServiceimpl.java +++ b/src/main/java/com/nis/web/service/restful/ConfigJedisServiceimpl.java @@ -30,6 +30,8 @@ import com.nis.util.ReadMaatXmlUtil; import com.nis.util.ServiceAndRDBIndexReal; import redis.clients.jedis.Jedis; +import redis.clients.jedis.ScanParams; +import redis.clients.jedis.ScanResult; import redis.clients.jedis.Transaction; import redis.clients.jedis.exceptions.JedisConnectionException; @@ -568,8 +570,7 @@ public class ConfigJedisServiceimpl implements ConfigRedisService { * @param redisTemplate * @param redisDBIndex * @param compileId - * @param isReuse - * 是否是分组复用 + * @param isReuse 是否是分组复用 */ private void setCommonConfig(MaatXmlConfig maatXmlConfig, Map map, int type, Double maatVersion, int service, Transaction transaction, Integer redisDBIndex, String compileId, boolean isReuse) { @@ -1028,17 +1029,12 @@ public class ConfigJedisServiceimpl implements ConfigRedisService { /** * 删除maat类配置 * - * @param id - * 配置id - * @param maatXmlConfig - * maat.xml中关于当前业务类型的key和value写法的对象 - * @param maatVersion - * 版本号 - * @param service - * 业务类型 + * @param id 配置id + * @param maatXmlConfig maat.xml中关于当前业务类型的key和value写法的对象 + * @param maatVersion 版本号 + * @param service 业务类型 * @param redisTemplate - * @param maatRelation - * id对应关系对象 + * @param maatRelation id对应关系对象 */ private void removeConfig(Long id, MaatXmlConfig maatXmlConfig, Double maatVersion, int service, Transaction transaction, int redisDBIndex, int idRelaRedisDBIndex) { @@ -1700,20 +1696,13 @@ public class ConfigJedisServiceimpl implements ConfigRedisService { /** * 保存域配置信息,供分组复用单独添加域和普通maat配置添加域 * - * @param maatConfig - * 配置信息 - * @param maatXmlConfig - * 配置的表达式 - * @param maatVersion - * 版本号 - * @param service - * 业务类型 - * @param transaction - * redis连接 - * @param redisDBIndex - * redis编号 - * @param isReuseSaveRegion - * 是否是分组复用单独添加域配置 + * @param maatConfig 配置信息 + * @param maatXmlConfig 配置的表达式 + * @param maatVersion 版本号 + * @param service 业务类型 + * @param transaction redis连接 + * @param redisDBIndex redis编号 + * @param isReuseSaveRegion 是否是分组复用单独添加域配置 */ private Integer addRegion(MaatConfig maatConfig, MaatXmlConfig maatXmlConfig, Double maatVersion, int service, Transaction transaction, Integer redisDBIndex, boolean isReuseSaveRegion) { @@ -1957,6 +1946,7 @@ public class ConfigJedisServiceimpl implements ConfigRedisService { JedisUtils.returnResource(resource); } } + private void updateMaatInfo(List list, String maatKey, Transaction transaction, Double maatVersion, int redisDBIndex, boolean idDel) { if (list != null && list.size() > 0) { @@ -1991,4 +1981,78 @@ public class ConfigJedisServiceimpl implements ConfigRedisService { } } + public Map> getAllKVByCompileId(Long id) { + Jedis resource = JedisUtils.getResource(1); + try { + Map> map = new HashMap<>(); + String pattern = "*," + id; + + for (int index = 2; index < 6; index++) { + resource.select(index); + List keyByScan = getKeyByScan(pattern, resource); + for (String redisKey : keyByScan) { + if (resource.exists(redisKey)) { + + String statusMebmer = null; + if (redisKey.startsWith("EFFECTIVE_RULE:")) { + statusMebmer = redisKey.replace("EFFECTIVE_RULE:", "ADD,"); + } else if (redisKey.startsWith("OBSOLETE_RULE:")) { + statusMebmer = redisKey.replace("OBSOLETE_RULE:", "DEL,"); + } + if (statusMebmer != null) { + Double zscore = resource.zscore("MAAT_UPDATE_STATUS", statusMebmer); + if (zscore != null) { + Set zrangeByScore = resource.zrangeByScore("MAAT_UPDATE_STATUS", zscore, + zscore); + for (String statusKey : zrangeByScore) { + String replaceKey = null; + if (statusKey.startsWith("ADD,")) { + replaceKey = statusKey.replace("ADD,", "EFFECTIVE_RULE:"); + + } else if (statusKey.startsWith("DEL,")) { + replaceKey = statusKey.replace("DEL,", "OBSOLETE_RULE:"); + } + if (replaceKey != null) { + String val = resource.get(replaceKey); + if (val != null) { + if (map.containsKey(index)) { + Map map2 = map.get(index); + map2.put(replaceKey, val); + } else { + Map map2 = new HashMap<>(); + map2.put(replaceKey, val); + map.put(index, map2); + } + } + } + } + + } + } + } + + } + } + return map; + } finally { + resource.close(); + } + } + + public List getKeyByScan(String pattern, Jedis resource) { + List list = new ArrayList<>(); + int count = 1000; + String cursor = ScanParams.SCAN_POINTER_START; + ScanParams scanParams = new ScanParams(); + scanParams.count(count); + scanParams.match(pattern); + do { + ScanResult scanResult = resource.scan(cursor, scanParams); + list.addAll(scanResult.getResult()); + cursor = scanResult.getStringCursor(); + } while ("!0".equals(cursor)); + return list; + + } + } diff --git a/src/main/java/com/nis/web/service/restful/ConfigRedisService.java b/src/main/java/com/nis/web/service/restful/ConfigRedisService.java index 5473542..3427baf 100644 --- a/src/main/java/com/nis/web/service/restful/ConfigRedisService.java +++ b/src/main/java/com/nis/web/service/restful/ConfigRedisService.java @@ -7,16 +7,24 @@ import com.nis.domain.restful.MaatConfig; /** * - *

Title: ConfigRedisService

- *

Description: 保存配置到redis中

- *

Company: IIE

+ *

+ * Title: ConfigRedisService + *

+ *

+ * Description: 保存配置到redis中 + *

+ *

+ * Company: IIE + *

+ * * @author rkg - * @date 2018年5月17日 + * @date 2018年5月17日 * */ public interface ConfigRedisService { /** * 下发非maat配置接口,key是redisDBIndex,value是配置集合 + * * @param configMap * @return 成功返回true,失败返回false或抛出异常 */ @@ -24,6 +32,7 @@ public interface ConfigRedisService { /** * 保存 maat配置接口,key是redisDBIndex,value是配置集合 + * * @param configMap * @return 成功返回true,失败返回false或抛出异常 */ @@ -31,27 +40,32 @@ public interface ConfigRedisService { /** * 分组复用域配置新增接口,value是配置集合 + * * @param configList * @return 成功返回true,失败返回false或抛出异常 */ - public boolean saveGroupReuseConfig( List configList); + public boolean saveGroupReuseConfig(List configList); /** * 获取指定key的自增长值 + * * @param key 需要自增的key * @return */ public Long getIncrId(String key); + /** * 批量获取接口,利用java进行遍历,比从redis中自增要快很多 + * * @param key * @param num * @return */ - public Long getIncrById(String key,int num); + public Long getIncrById(String key, int num); /** * 删除非maat类配置,第一个key是redisDBIndex,第二个key是业务类型,value是配置id集合 + * * @param idMap * @param isInvalid 是否将配置置为无效 * @return 成功返回true,失败返回false或抛出异常 @@ -60,6 +74,7 @@ public interface ConfigRedisService { /** * 删除maat类配置,key是业务类型,value是配置id集合 + * * @param serviceConfigMap * @return 成功返回true,失败返回false或抛出异常 */ @@ -67,9 +82,17 @@ public interface ConfigRedisService { /** * 删除分组复用的域配置 + * * @param reuseMap key是groupId,value是"tableName,regionId" * @return */ - public boolean delGroupReuseConfig( Map> reuseMap); + public boolean delGroupReuseConfig(Map> reuseMap); + /** + * 根据配置id获取对应的编译,组,域等信息 + * + * @param id + * @return + */ + public Map> getAllKVByCompileId(Long id); } diff --git a/src/main/java/com/nis/web/service/restful/DashboardService.java b/src/main/java/com/nis/web/service/restful/DashboardService.java index 8356d24..47133f8 100644 --- a/src/main/java/com/nis/web/service/restful/DashboardService.java +++ b/src/main/java/com/nis/web/service/restful/DashboardService.java @@ -306,12 +306,12 @@ public class DashboardService extends BaseService{ * 协议统计报表 * @return */ - public List getProtocolList(){ + public List getProtocolList(String startTime,String endTime){ TrafficProtocolStatistic maxStatTime = trafficProtocolStatisticDao.getMaxStatTime(); List list=new ArrayList(); if(maxStatTime!=null&&maxStatTime.getStatTime()!=null) { Date statTime = maxStatTime.getStatTime(); - list = trafficProtocolStatisticDao.getProtocolList(statTime); + list = trafficProtocolStatisticDao.getProtocolList(statTime,startTime,endTime); } return list; } @@ -339,12 +339,12 @@ public class DashboardService extends BaseService{ * App统计报表 * @return */ - public List getAppList(){ + public List getAppList(String startTime,String endTime){ TrafficAppStatistic maxStatTime = trafficAppStatisticDao.getMaxStatTime(); List list=new ArrayList(); if(maxStatTime!=null&&maxStatTime.getStatTime()!=null) { Date statTime = maxStatTime.getStatTime(); - list = trafficAppStatisticDao.getAppList(statTime); + list = trafficAppStatisticDao.getAppList(statTime,startTime,endTime); } return list; } diff --git a/src/main/java/com/nis/web/task/SyncRedisToCluster.java b/src/main/java/com/nis/web/task/SyncRedisToCluster.java index e9dffc7..4b044cd 100644 --- a/src/main/java/com/nis/web/task/SyncRedisToCluster.java +++ b/src/main/java/com/nis/web/task/SyncRedisToCluster.java @@ -11,15 +11,17 @@ import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import com.nis.restful.RestBusinessCode; +import com.nis.restful.ServiceRuntimeException; import com.nis.util.Configurations; import com.nis.util.Constants; import com.nis.util.ExceptionUtil; import com.nis.util.JedisUtils; +import com.nis.web.service.SpringContextHolder; import com.nis.web.service.restful.ConfigJedisServiceimpl; import redis.clients.jedis.Jedis; @@ -28,6 +30,7 @@ import redis.clients.jedis.JedisPool; import redis.clients.jedis.ScanParams; import redis.clients.jedis.ScanResult; import redis.clients.jedis.Tuple; +import redis.clients.jedis.exceptions.JedisException; import redis.clients.util.JedisClusterCRC16; @Component @@ -36,14 +39,46 @@ public class SyncRedisToCluster { private static Logger logger = LoggerFactory.getLogger(ConfigJedisServiceimpl.class); private static int redisStatisticsRealDBIndex = Configurations.getIntProperty("redisStatisticsRealDBIndex", 14); - @Autowired - private JedisCluster jedisCluster; + /** + * 每次使用获取新连接,避免集群某些机器宕机后影响连接使用,注意需要在applicationContext-redis.xml中修改jedisCluster的scope为 + * prototype + * + * @return + * @throws JedisException + */ + public JedisCluster getResource() throws JedisException { + JedisCluster jedisCluster = SpringContextHolder.getBean(JedisCluster.class); + if (jedisCluster == null) { + throw new ServiceRuntimeException("无法获取redis-cluster连接,请联系管理员检查程序", + RestBusinessCode.CannotConnectionRedis.getValue()); + } + return jedisCluster; + + } + + /** + * 关闭集群连接 + * + * @param jedisCluster + */ + private void closeConn(JedisCluster jedisCluster) { + if (jedisCluster != null) { + try { + jedisCluster.close(); + } catch (Exception e) { + throw new ServiceRuntimeException("释放redis-cluster连接失败", + RestBusinessCode.CannotConnectionRedis.getValue()); + } + } + } @Scheduled(cron = "${syncRedisToClusterCron}") public void syncRedisToCluster() { + JedisCluster jedisCluster = getResource(); + System.out.println(jedisCluster); String requestId = UUID.randomUUID().toString(); try { - if (lock(requestId)) {// 避免集群环境下同一秒钟所有的机器都执行这个定时任务 + if (lock(jedisCluster, requestId)) {// 避免集群环境下同一秒钟所有的机器都执行这个定时任务 // keys("EFFECTIVE_RULE*"); // keys("OBSOLETE_RULE*"); String clusterMaatVersionStr = jedisCluster.get("MAAT_VERSION"); @@ -54,11 +89,11 @@ public class SyncRedisToCluster { Integer redisMaatVersion = Integer.valueOf(redisMaatVersionStr); if (redisMaatVersion < clusterMaatVersion) {// 如果主从库比集群库的版本号小则下发全量 logger.info("redis集群中的MAAT_VERSION为大于配置库中的MAAT_VERSION,开始执行全量同步"); - syncAllData(redisMaatVersionStr); + syncAllData(jedisCluster, redisMaatVersionStr); } else if (redisMaatVersion > clusterMaatVersion) {// 获取增量的数据 logger.info("redis集群中的MAAT_VERSION为小于配置库中的MAAT_VERSION,开始执行增量同步,score是{}-{}", clusterMaatVersion, +redisMaatVersion); - syncData(clusterMaatVersion.doubleValue(), redisMaatVersion.doubleValue(), + syncData(jedisCluster, clusterMaatVersion.doubleValue(), redisMaatVersion.doubleValue(), redisMaatVersionStr); } else { logger.info("redis集群中的MAAT_VERSION与配置库中的MAAT_VERSION相等,暂不执行配置同步操作"); @@ -66,13 +101,13 @@ public class SyncRedisToCluster { } else { logger.info("redis配置库中MAAT_VERSION为null,但是redis集群中的MAAT_VERSION为{},集群与配置库的数据不同步,开始删除集群中的配置", clusterMaatVersionStr); - delClusterData(); + delClusterData(jedisCluster); } } else { if (redisMaatVersionStr != null && !redisMaatVersionStr.trim().equals("")) { logger.info("redis配置库中的MAAT_VERSION为{},redis集群中的MAAT_VERSION为null,开始执行全量同步", redisMaatVersionStr); - syncAllData(redisMaatVersionStr); + syncAllData(jedisCluster, redisMaatVersionStr); } else { logger.info("redis配置库中和redis集群中的MAAT_VERSION都为null,暂时不执行全量同步"); } @@ -83,12 +118,13 @@ public class SyncRedisToCluster { } catch (Exception e) { logger.error("同步配置库配置到3A-redisCluster失败,失败原因:{}", ExceptionUtil.getExceptionMsg(e)); } finally { - unlock(requestId); + unlock(jedisCluster, requestId); + closeConn(jedisCluster); } } // 删除之前的key - public void deleteRedisKeyStartWith(String redisKeyStartWith) { + public void deleteRedisKeyStartWith(JedisCluster jedisCluster, String redisKeyStartWith) { try { Map clusterNodes = jedisCluster.getClusterNodes(); for (Map.Entry entry : clusterNodes.entrySet()) { @@ -126,7 +162,7 @@ public class SyncRedisToCluster { } } - public TreeSet keys(String pattern) { + public TreeSet keys(JedisCluster jedisCluster, String pattern) { TreeSet keys = new TreeSet<>(); // 获取所有的节点 Map clusterNodes = jedisCluster.getClusterNodes(); @@ -150,7 +186,7 @@ public class SyncRedisToCluster { return keys; } - public void delClusterData() { + public void delClusterData(JedisCluster jedisCluster) { // 清空集群库配置数据 if (jedisCluster.exists("MAAT_UPDATE_STATUS")) { jedisCluster.del("MAAT_UPDATE_STATUS"); @@ -160,18 +196,18 @@ public class SyncRedisToCluster { jedisCluster.del("MAAT_VERSION"); logger.info("删除MAAT_VERSION成功"); } - deleteRedisKeyStartWith("EFFECTIVE_RULE*"); + deleteRedisKeyStartWith(jedisCluster, "EFFECTIVE_RULE*"); logger.info("删除EFFECTIVE_RULE*成功"); - deleteRedisKeyStartWith("OBSOLETE_RULE*"); + deleteRedisKeyStartWith(jedisCluster, "OBSOLETE_RULE*"); logger.info("删除OBSOLETE_RULE*成功"); } - public void syncAllData(String version) { - delClusterData(); - syncData(null, null, version); + public void syncAllData(JedisCluster jedisCluster, String version) { + delClusterData(jedisCluster); + syncData(jedisCluster, null, null, version); } - public void syncData(Double min, Double max, String verionStr) { + public void syncData(JedisCluster jedisCluster, Double min, Double max, String verionStr) { Jedis resource = JedisUtils.getResource(redisStatisticsRealDBIndex); Set zrangeByScoreWithScores = null; if (min == null && max == null) { @@ -190,20 +226,20 @@ public class SyncRedisToCluster { } min = Double.parseDouble(start + ""); max = Double.parseDouble(end + ""); - //分批获取,避免一次性获取太多,造成redis阻塞 + // 分批获取,避免一次性获取太多,造成redis阻塞 zrangeByScoreWithScores = resource.zrangeByScoreWithScores("MAAT_UPDATE_STATUS", min, max);// 获取所有的maat_update_status - syncData(zrangeByScoreWithScores); + syncData(jedisCluster, zrangeByScoreWithScores); } } else { zrangeByScoreWithScores = resource.zrangeByScoreWithScores("MAAT_UPDATE_STATUS", min, max);// 获取增量的数据 - syncData(zrangeByScoreWithScores); + syncData(jedisCluster, zrangeByScoreWithScores); } jedisCluster.set("MAAT_VERSION", verionStr); logger.info("更新了redis集群中的MAAT_VERSION,更新后值是{}", verionStr); logger.info("向redis集群同步数据成功"); } - private void syncData(Set zrangeByScoreWithScores) { + private void syncData(JedisCluster jedisCluster, Set zrangeByScoreWithScores) { for (Tuple tuple : zrangeByScoreWithScores) { String key = tuple.getElement(); String zset = key; @@ -239,7 +275,7 @@ public class SyncRedisToCluster { * @param requestId 请求标识(UUID.randomUUID().toString()),正常情况下是谁加的锁,谁去解锁不能a加的锁,b去解锁 * @return 是否获取成功 */ - public Boolean lock(String requestId) { + public Boolean lock(JedisCluster jedisCluster, String requestId) { String key = "configSyncDistributedLock"; String var1 = jedisCluster.set(key, requestId, SET_IF_NOT_EXIST, SET_WITH_EXPIRE_TIME, Constants.REDISLOCKTIME); if (LOCK_SUCCESS.equals(var1)) { @@ -253,7 +289,7 @@ public class SyncRedisToCluster { * * @param requestId */ - protected boolean unlock(String requestId) { + protected boolean unlock(JedisCluster jedisCluster, String requestId) { String key = "configSyncDistributedLock"; // 这个字符串是个lua脚本，代表的意思是如果根据key拿到的value跟传入的value相同就执行del，否则就返回0【保证安全性】 String luaScript = "if redis.call(\"get\",KEYS[1]) == ARGV[1] then return redis.call(\"del\",KEYS[1]) else return 0 end"; diff --git a/src/main/resources/commonSources/applicationConfig-parser.xml b/src/main/resources/commonSources/applicationConfig-parser.xml index f04ea88..444abbd 100644 --- a/src/main/resources/commonSources/applicationConfig-parser.xml +++ b/src/main/resources/commonSources/applicationConfig-parser.xml @@ -233,8 +233,9 @@ + - + diff --git a/src/main/resources/maatXml/applicationConfig-maatRedis.xml b/src/main/resources/maatXml/applicationConfig-maatRedis.xml index ab465b1..137752e 100644 --- a/src/main/resources/maatXml/applicationConfig-maatRedis.xml +++ b/src/main/resources/maatXml/applicationConfig-maatRedis.xml @@ -184,7 +184,7 @@ EFFECTIVE_RULE;:;{un_maat_table_name};,;[cfg_id] - [cfg_id];\t;[original_dest_ip];\t;[original_dest_port];\t;[original_protocol];\t;[translate_dest_ip];\t;[translate_dest_port];\t;[do_log];\t;[action];\t;[service];\t;[is_valid];\t;[effective_range];\t;[op_time];&nbsp;0;\n + [cfg_id];\t;[original_addr_type];\t;[original_dest_ip];\t;[original_dest_port];\t;[original_protocol];\t;[translated_dest_ip];\t;[translated_dest_port];\t;[do_log];\t;[action];\t;[service];\t;[is_valid];\t;[effective_range];\t;[op_time];&nbsp;0;\n 1