1:为实时统计配置中jediscluster改为每次使用都获取一个新的

2:添加根据配置id获取编译,组,域等信息的接口
3:修改获取或释放redis分布式锁后,将redis连接释放
4:解决冲突
This commit is contained in:
renkaige
2018-12-09 17:30:24 +06:00
parent 09c5a06947
commit ef35100af5
19 changed files with 303 additions and 128 deletions

View File

@@ -306,9 +306,10 @@ public class JedisUtils {
* @return 是否获取成功 * @return 是否获取成功
*/ */
public static Boolean lock(String requestId) { public static Boolean lock(String requestId) {
Jedis resource = getResource(0);
String key = "redisDistributedLock"; String key = "redisDistributedLock";
String var1 = getResource(0).set(key, requestId, SET_IF_NOT_EXIST, SET_WITH_EXPIRE_TIME, String var1 = resource.set(key, requestId, SET_IF_NOT_EXIST, SET_WITH_EXPIRE_TIME, Constants.REDISLOCKTIME);
Constants.REDISLOCKTIME); returnResource(resource);
if (LOCK_SUCCESS.equals(var1)) { if (LOCK_SUCCESS.equals(var1)) {
return true; return true;
} }
@@ -322,11 +323,13 @@ public class JedisUtils {
* @return * @return
*/ */
public static Boolean unLock(String value) { public static Boolean unLock(String value) {
Jedis resource = getResource(0);
String key = "redisDistributedLock"; String key = "redisDistributedLock";
// 这个字符串是个lua脚本代表的意思是如果根据key拿到的value跟传入的value相同就执行del否则就返回0【保证安全性】 // 这个字符串是个lua脚本代表的意思是如果根据key拿到的value跟传入的value相同就执行del否则就返回0【保证安全性】
String luaScript = "if redis.call(\"get\",KEYS[1]) == ARGV[1] then return redis.call(\"del\",KEYS[1]) else return 0 end"; String luaScript = "if redis.call(\"get\",KEYS[1]) == ARGV[1] then return redis.call(\"del\",KEYS[1]) else return 0 end";
// 这个命令就是去执行lua脚本KEYS的集合就是第二个参数ARGV的集合就是第三参数【保证解锁的原子操作】 // 这个命令就是去执行lua脚本KEYS的集合就是第二个参数ARGV的集合就是第三参数【保证解锁的原子操作】
Object var2 = getResource(0).eval(luaScript, Collections.singletonList(key), Collections.singletonList(value)); Object var2 = resource.eval(luaScript, Collections.singletonList(key), Collections.singletonList(value));
returnResource(resource);
if (UNLOCK_SUCCESS == var2) { if (UNLOCK_SUCCESS == var2) {
return true; return true;
} }

View File

@@ -18,6 +18,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
@@ -30,6 +31,7 @@ import com.nis.restful.RestBusinessCode;
import com.nis.restful.RestServiceException; import com.nis.restful.RestServiceException;
import com.nis.restful.ServiceRuntimeException; import com.nis.restful.ServiceRuntimeException;
import com.nis.util.Constants; import com.nis.util.Constants;
import com.nis.util.ExceptionUtil;
import com.nis.util.FileUtils; import com.nis.util.FileUtils;
import com.nis.util.MinioUtil; import com.nis.util.MinioUtil;
import com.nis.util.StringUtils; import com.nis.util.StringUtils;
@@ -81,9 +83,10 @@ public class ConfigSourcesController extends BaseRestController {
if (getLock(requestId)) { if (getLock(requestId)) {
if (null != configSource && null != configSource.getConfigCompileList() if (null != configSource && null != configSource.getConfigCompileList()
&& configSource.getConfigCompileList().size() > 0) { && configSource.getConfigCompileList().size() > 0) {
if (configSource.getConfigCompileList()!=null&&configSource.getConfigCompileList().size()>Constants.MAX_LIST_SIZE) { if (configSource.getConfigCompileList() != null
thread.setContent("批量提交数量超过"+Constants.MAX_LIST_SIZE+"条,不记录请求内容"); && configSource.getConfigCompileList().size() > Constants.MAX_LIST_SIZE) {
} thread.setContent("批量提交数量超过" + Constants.MAX_LIST_SIZE + "条,不记录请求内容");
}
checkOpAction(thread, System.currentTimeMillis() - start, configSource.getOpAction(), checkOpAction(thread, System.currentTimeMillis() - start, configSource.getOpAction(),
Constants.OPACTION_POST); Constants.OPACTION_POST);
// 验证配置编译数据 // 验证配置编译数据
@@ -134,8 +137,9 @@ public class ConfigSourcesController extends BaseRestController {
if (null != configSource && null != configSource.getConfigCompileList() if (null != configSource && null != configSource.getConfigCompileList()
&& configSource.getConfigCompileList().size() > 0) { && configSource.getConfigCompileList().size() > 0) {
int opAction = configSource.getOpAction(); int opAction = configSource.getOpAction();
if (configSource.getConfigCompileList()!=null&&configSource.getConfigCompileList().size()>Constants.MAX_LIST_SIZE) { if (configSource.getConfigCompileList() != null
thread.setContent("批量提交数量超过"+Constants.MAX_LIST_SIZE+"条,不记录请求内容"); && configSource.getConfigCompileList().size() > Constants.MAX_LIST_SIZE) {
thread.setContent("批量提交数量超过" + Constants.MAX_LIST_SIZE + "条,不记录请求内容");
} }
checkOpAction(thread, System.currentTimeMillis() - start, opAction, 2); checkOpAction(thread, System.currentTimeMillis() - start, opAction, 2);
configSourcesService.updateConfigSources(thread, start, configSource.getConfigCompileList(), configSourcesService.updateConfigSources(thread, start, configSource.getConfigCompileList(),
@@ -551,6 +555,28 @@ public class ConfigSourcesController extends BaseRestController {
"MAAT规则分组复用域配置删除成功" + sb.toString(), Constants.IS_DEBUG ? groupReuseSource : null); "MAAT规则分组复用域配置删除成功" + sb.toString(), Constants.IS_DEBUG ? groupReuseSource : null);
} }
@RequestMapping(value = "/cfg/v1/getAllKVByCompileId", method = RequestMethod.GET)
@ApiOperation(value = "根据配置id获取对应的编译,组,域等信息", httpMethod = "GET", response = Map.class, notes = "根据配置id获取对应的编译,组,域等信息")
@ApiParam(value = "配置id", name = "getAllKVByCompileId", required = true)
public Map getAllKVByCompileId(@RequestParam Long compileId, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis();
AuditLogThread thread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request,
compileId);
Map map=null;
try {
map=configRedisService.getAllKVByCompileId(compileId);
} catch (Exception e) {
thread.setExceptionInfo("根据配置id获取对应的编译,组,域等信息时出现异常:" + e.getMessage());
logger.error("根据配置id获取对应的编译,组,域等信息时出现异常:" +ExceptionUtil.getExceptionMsg(e));
throw new RestServiceException("根据配置id获取对应的编译,组,域等信息失败",
RestBusinessCode.ConfigSourceIsNull.getValue());
}
return compileServiceResponse(thread, System.currentTimeMillis() - start, request, response,
"根据配置id获取对应的编译,组,域等信息成功", map);
}
private void validateGroupReuseSource(AuditLogThread thread, long start, GroupReuseSource groupReuseSource) { private void validateGroupReuseSource(AuditLogThread thread, long start, GroupReuseSource groupReuseSource) {
String errorInfo = ""; String errorInfo = "";

View File

@@ -166,12 +166,12 @@ public class DashboardServiceController extends BaseRestController {
*/ */
@RequestMapping(value="trafficProtocolList", method = RequestMethod.GET) @RequestMapping(value="trafficProtocolList", method = RequestMethod.GET)
@ApiOperation(value = "协议统计占比与报表", httpMethod = "GET", notes = "对应协议统计详情占比与报表") @ApiOperation(value = "协议统计占比与报表", httpMethod = "GET", notes = "对应协议统计详情占比与报表")
public Map<String,?> trafficProtocolList(Model model, HttpServletRequest request, HttpServletResponse response){ public Map<String,?> trafficProtocolList(String searchFoundStartTime,String searchFoundEndTime,Model model, HttpServletRequest request, HttpServletResponse response){
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null);
List<HashMap> list = new ArrayList<HashMap>(); List<HashMap> list = new ArrayList<HashMap>();
try { try {
List<Map> ipActiveList = dashboardService.getProtocolList(); List<Map> ipActiveList = dashboardService.getProtocolList(searchFoundStartTime,searchFoundEndTime);
if (ipActiveList!=null&&ipActiveList.size() > 0) { if (ipActiveList!=null&&ipActiveList.size() > 0) {
String jsonString = JsonMapper.toJsonString(ipActiveList); String jsonString = JsonMapper.toJsonString(ipActiveList);
list = (java.util.List<HashMap>) JsonMapper.fromJsonList(jsonString,HashMap.class); list = (java.util.List<HashMap>) JsonMapper.fromJsonList(jsonString,HashMap.class);
@@ -289,12 +289,12 @@ public class DashboardServiceController extends BaseRestController {
*/ */
@RequestMapping(value="trafficAppList", method = RequestMethod.GET) @RequestMapping(value="trafficAppList", method = RequestMethod.GET)
@ApiOperation(value = "App统计占比与报表", httpMethod = "GET", notes = "对应App统计详情占比与报表") @ApiOperation(value = "App统计占比与报表", httpMethod = "GET", notes = "对应App统计详情占比与报表")
public Map<String,?> trafficAppList(Model model, HttpServletRequest request, HttpServletResponse response){ public Map<String,?> trafficAppList(String searchFoundStartTime,String searchFoundEndTime,Model model, HttpServletRequest request, HttpServletResponse response){
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null);
List<HashMap> list = new ArrayList<HashMap>(); List<HashMap> list = new ArrayList<HashMap>();
try { try {
List<Map> appList = dashboardService.getAppList(); List<Map> appList = dashboardService.getAppList(searchFoundStartTime,searchFoundEndTime);
if (appList!=null&&appList.size() > 0) { if (appList!=null&&appList.size() > 0) {
String jsonString = JsonMapper.toJsonString(appList); String jsonString = JsonMapper.toJsonString(appList);
list = (java.util.List<HashMap>) JsonMapper.fromJsonList(jsonString,HashMap.class); list = (java.util.List<HashMap>) JsonMapper.fromJsonList(jsonString,HashMap.class);

View File

@@ -55,6 +55,7 @@ public class NmsInfoController extends BaseRestController {
if (trafficNmsServerStatisticList != null && trafficNmsServerStatisticList.getTrafficNmsServerList() != null if (trafficNmsServerStatisticList != null && trafficNmsServerStatisticList.getTrafficNmsServerList() != null
&& trafficNmsServerStatisticList.getTrafficNmsServerList().size() > 0) { && trafficNmsServerStatisticList.getTrafficNmsServerList().size() > 0) {
thread.setContent("NMS系统上报服务器数量为"+trafficNmsServerStatisticList.getTrafficNmsServerList().size());
trafficNmsServerStatisticService.saveNmsInfo(trafficNmsServerStatisticList.getTrafficNmsServerList()); trafficNmsServerStatisticService.saveNmsInfo(trafficNmsServerStatisticList.getTrafficNmsServerList());
} else { } else {
throw new RestServiceException(thread, System.currentTimeMillis() - start, "参数trafficNmsServerList不能为空", throw new RestServiceException(thread, System.currentTimeMillis() - start, "参数trafficNmsServerList不能为空",
@@ -71,7 +72,8 @@ public class NmsInfoController extends BaseRestController {
} }
} }
return testServiceResponse(System.currentTimeMillis() - start, request, response, "上报服务器状态信息成功",
return compileServiceResponse(thread, System.currentTimeMillis() - start, request, response, "上报服务器状态信息成功",
Constants.IS_DEBUG ? trafficNmsServerStatisticList : null); Constants.IS_DEBUG ? trafficNmsServerStatisticList : null);
} }
@@ -86,6 +88,7 @@ public class NmsInfoController extends BaseRestController {
try { try {
if (nmsDiRuleList != null && nmsDiRuleList.getNmsDiRuleList() != null if (nmsDiRuleList != null && nmsDiRuleList.getNmsDiRuleList() != null
&& nmsDiRuleList.getNmsDiRuleList().size() > 0) { && nmsDiRuleList.getNmsDiRuleList().size() > 0) {
thread.setContent("NMS系统上报的di信息数量为"+nmsDiRuleList.getNmsDiRuleList().size());
nmsDiRuleService.saveNmsDiRuleInfo(nmsDiRuleList.getNmsDiRuleList()); nmsDiRuleService.saveNmsDiRuleInfo(nmsDiRuleList.getNmsDiRuleList());
} else { } else {
throw new RestServiceException(thread, System.currentTimeMillis() - start, "参数nmsDiRuleList不能为空", throw new RestServiceException(thread, System.currentTimeMillis() - start, "参数nmsDiRuleList不能为空",
@@ -100,7 +103,7 @@ public class NmsInfoController extends BaseRestController {
"上报NmsDiRule信息异常:" + e.getMessage(), RestBusinessCode.unknow_error.getValue()); "上报NmsDiRule信息异常:" + e.getMessage(), RestBusinessCode.unknow_error.getValue());
} }
} }
return testServiceResponse(System.currentTimeMillis() - start, request, response, "上报NmsDiRule信息成功", return compileServiceResponse(thread, System.currentTimeMillis() - start, request, response, "上报NmsDiRule信息成功",
Constants.IS_DEBUG ? nmsDiRuleList : null); Constants.IS_DEBUG ? nmsDiRuleList : null);
} }
@@ -116,6 +119,7 @@ public class NmsInfoController extends BaseRestController {
try { try {
if (trafficNetflowPortInfoList != null && trafficNetflowPortInfoList.getTrafficNetflowPortInfoList() != null if (trafficNetflowPortInfoList != null && trafficNetflowPortInfoList.getTrafficNetflowPortInfoList() != null
&& trafficNetflowPortInfoList.getTrafficNetflowPortInfoList().size() > 0) { && trafficNetflowPortInfoList.getTrafficNetflowPortInfoList().size() > 0) {
thread.setContent("NMS系统上报上报trafficNetflowPort信息数量为"+trafficNetflowPortInfoList.getTrafficNetflowPortInfoList().size());
trafficNetflowPortInfoService trafficNetflowPortInfoService
.saveTrafficNetflowPortInfo(trafficNetflowPortInfoList.getTrafficNetflowPortInfoList()); .saveTrafficNetflowPortInfo(trafficNetflowPortInfoList.getTrafficNetflowPortInfoList());
} else { } else {
@@ -131,7 +135,7 @@ public class NmsInfoController extends BaseRestController {
"上报trafficNetflowPort信息异常:" + e.getMessage(), RestBusinessCode.unknow_error.getValue()); "上报trafficNetflowPort信息异常:" + e.getMessage(), RestBusinessCode.unknow_error.getValue());
} }
} }
return testServiceResponse(System.currentTimeMillis() - start, request, response, return compileServiceResponse(thread, System.currentTimeMillis() - start, request, response,
"上报trafficNetflowPort信息成功", Constants.IS_DEBUG ? trafficNetflowPortInfoList : null); "上报trafficNetflowPort信息成功", Constants.IS_DEBUG ? trafficNetflowPortInfoList : null);
} }

View File

@@ -16,24 +16,24 @@
<!-- 获取取阻断,监测等流量信息最近时间 --> <!-- 获取取阻断,监测等流量信息最近时间 -->
<select id="getMaxReportTime" resultType="com.nis.domain.restful.dashboard.NtcTotalReport"> <select id="getMaxReportTime" resultType="com.nis.domain.restful.dashboard.NtcTotalReport">
SELECT MAX(report_time) reportTime FROM ntc_total_report SELECT report_time reportTime FROM ntc_total_report order by report_time desc limit 1
</select> </select>
<!-- 根据最近的时间获取阻断,监测等流量信息 --> <!-- 根据最近的时间获取阻断,监测等流量信息 -->
<select id="getTotalReportList" resultType="java.util.HashMap"> <select id="getTotalReportList" resultType="java.util.HashMap">
SELECT SUM(reject_num) rejectNum,SUM(monitor_num) monitorNum,SUM(c2s_pkt_num) c2sPktNum,SUM(s2c_pkt_num) s2cPktNum,SUM(c2s_byte_len) c2sByteLen,SUM(s2c_byte_len) s2cByteLen,SUM(new_uni_conn_num) newUniConnNum,SUM(live_conn_num) liveConnNum, SELECT SUM(reject_num) rejectNum,SUM(monitor_num) monitorNum,SUM(c2s_pkt_num) c2sPktNum,SUM(s2c_pkt_num) s2cPktNum,SUM(c2s_byte_len) c2sByteLen,SUM(s2c_byte_len) s2cByteLen,SUM(new_uni_conn_num) newUniConnNum,SUM(live_conn_num) liveConnNum,
SUM(drop_conn_num) dropConnNum,SUM(loop_conn_num) loopConnNum FROM ntc_total_report SUM(drop_conn_num) dropConnNum,SUM(loop_conn_num) loopConnNum FROM ntc_total_report
where report_time = #{reportTime} where report_time > DATE_SUB(#{reportTime}, INTERVAL 1 HOUR)
</select> </select>
<!-- 获取带宽最近时间 --> <!-- 获取带宽最近时间 -->
<select id="getMaxStatTime" resultType="java.util.HashMap"> <select id="getMaxStatTime" resultType="java.util.HashMap">
SELECT MAX(stat_time) statTime FROM traffic_trans_statistic SELECT stat_time statTime FROM traffic_trans_statistic order by stat_time desc limit 1
</select> </select>
<!-- 根据最近时间条获取带宽,进出口流量 --> <!-- 根据最近时间条获取带宽,进出口流量 -->
<select id="getNetFlowPortInfoNew" resultType="java.util.HashMap"> <select id="getNetFlowPortInfoNew" resultType="java.util.HashMap">
SELECT IFNULL(SUM(c2s_byte_len),0) inoctets ,IFNULL(SUM(s2c_byte_len),0) outoctets FROM traffic_trans_statistic SELECT IFNULL(SUM(c2s_byte_len),0) AS inoctets ,IFNULL(SUM(s2c_byte_len),0) AS outoctets FROM traffic_trans_statistic
where stat_time = #{statTime} WHERE stat_time > DATE_SUB(#{statTime}, INTERVAL 1 HOUR)
</select> </select>
<!-- 获取近五分钟的带宽根据ip46协议 tcp,udp变化 --> <!-- 获取近五分钟的带宽根据ip46协议 tcp,udp变化 -->
<select id="getBandwidthTrans" resultMap="BandwidthResultMap"> <select id="getBandwidthTrans" resultMap="BandwidthResultMap">
@@ -63,7 +63,7 @@
<!-- 获取报表最近时间 --> <!-- 获取报表最近时间 -->
<select id="getEntranceMaxReportTime" resultType="java.util.HashMap"> <select id="getEntranceMaxReportTime" resultType="java.util.HashMap">
SELECT MAX(report_time) reportTime FROM ntc_entrance_report SELECT report_time reportTime FROM ntc_entrance_report order by report_time desc limit 1
</select> </select>
<!-- entrance 1,2 ,动作为阻断,近五分钟数据 --> <!-- entrance 1,2 ,动作为阻断,近五分钟数据 -->
<select id="getActionTrans" resultType="com.nis.domain.restful.NtcEntranceReport"> <select id="getActionTrans" resultType="com.nis.domain.restful.NtcEntranceReport">

View File

@@ -13,5 +13,5 @@ public interface TrafficAppStatisticDao {
TrafficAppStatistic getMaxStatTime(); TrafficAppStatistic getMaxStatTime();
List<Map> appChart(@Param("statTime")Date statTime); List<Map> appChart(@Param("statTime")Date statTime);
List<Map> getAppList(@Param("statTime")Date statTime); List<Map> getAppList(@Param("statTime")Date statTime,@Param("beginTime")String beginTime,@Param("endTime")String endTime);
} }

View File

@@ -17,19 +17,28 @@
</sql> </sql>
<!-- 查询最近时间 --> <!-- 查询最近时间 -->
<select id="getMaxStatTime" resultType="com.nis.domain.restful.dashboard.TrafficAppStatistic"> <select id="getMaxStatTime" resultType="com.nis.domain.restful.dashboard.TrafficAppStatistic">
SELECT MAX(stat_time) statTime FROM traffic_app_statistic WHERE app_type != 0 SELECT stat_time statTime FROM traffic_app_statistic order by stat_time desc limit 1
</select> </select>
<select id="appChart" resultType="java.util.HashMap">
<select id="appChart" resultType="java.util.HashMap">
SELECT app_type appType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_app_statistic SELECT app_type appType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_app_statistic
WHERE app_type !=0 and stat_time = #{statTime} WHERE stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour)
GROUP BY app_type order by count desc limit 0,10 GROUP BY app_type order by count desc limit 0,10
</select> </select>
<!-- 获取app统计详情 --> <!-- 获取app统计详情 -->
<select id="getAppList" resultType="java.util.HashMap"> <select id="getAppList" resultType="java.util.HashMap">
SELECT IFNULL(c.app_name,p.app_type) appName,p.app_type appType,p.link_num linkNum, p.packets packets, p.GByte GByte FROM ( SELECT IFNULL(c.app_name,p.app_type) appName,p.app_type appType,p.link_num linkNum, p.packets packets, p.GByte GByte FROM (
SELECT app_type,SUM(link_num) AS link_num,SUM(c2s_pkt_num+s2c_pkt_num) AS packets, SELECT app_type,SUM(link_num) AS link_num,SUM(c2s_pkt_num+s2c_pkt_num) AS packets,
(SUM(c2s_byte_len+s2c_byte_len)/1024/1024/1024) AS GByte (SUM(c2s_byte_len+s2c_byte_len)/1024/1024/1024) AS GByte
FROM traffic_app_statistic WHERE stat_time=#{statTime} GROUP BY app_type ) p FROM traffic_app_statistic WHERE
<if test="beginTime==null or endTime==null">
stat_time > DATE_SUB(#{statTime},INTERVAL 1 HOUR)
</if>
<if test="beginTime!=null and endTime!=null">
stat_time >= #{beginTime} and stat_time &lt;= #{endTime}
</if>
GROUP BY app_type ) p
LEFT JOIN ui_code_app_dic c ON p.app_type=c.view_code ORDER BY p.GByte DESC LEFT JOIN ui_code_app_dic c ON p.app_type=c.view_code ORDER BY p.GByte DESC
</select> </select>
</mapper> </mapper>

View File

@@ -17,31 +17,32 @@
</sql> </sql>
<!-- 获取最近时间并且有效 --> <!-- 获取最近时间并且有效 -->
<select id="getMaxStatTime" resultType="com.nis.domain.restful.dashboard.TrafficHttpStatistic"> <select id="getMaxStatTime" resultType="com.nis.domain.restful.dashboard.TrafficHttpStatistic">
SELECT MAX(stat_time) statTime FROM traffic_http_statistic WHERE web_id != 0 SELECT stat_time statTime FROM traffic_http_statistic order by stat_time desc limit 1
</select> </select>
<!-- 根据服务网站将域名分类 网站列表--> <!-- 根据服务网站将域名分类 网站列表-->
<select id="getDomainByWebsiteList" resultType="com.nis.domain.restful.dashboard.TrafficHttpStatistic">
SELECT SUM(c2s_byte_len+s2c_byte_len) count, IFNULL( website_service_id, 268435455 ) websiteServiceId SELECT SUM(c2s_byte_len+s2c_byte_len) count, IFNULL( website_service_id, 268435455 ) websiteServiceId
FROM TRAFFIC_HTTP_STATISTIC t FROM TRAFFIC_HTTP_STATISTIC t
LEFT JOIN ui_website_domain_topic u ON t.web_id = u.id LEFT JOIN ui_website_domain_topic u ON t.web_id = u.id
LEFT JOIN ui_website_domain_topic u ON t.web_id = u.id where t.stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour)
where t.web_id!=0
GROUP BY u.website_service_id ORDER BY count desc limit 0,10 GROUP BY u.website_service_id ORDER BY count desc limit 0,10
GROUP BY u.website_service_id ORDER BY count desc limit 0,10 </select>
</select> <!-- 根据主题将域名分类 主题列表 最近五分钟top10-->
<select id="getDomainByTopicList" resultType="java.util.HashMap">
SELECT SUM(c2s_byte_len+s2c_byte_len) count, IFNULL( topic_id, 268435455 ) topicId SELECT SUM(c2s_byte_len+s2c_byte_len) count, IFNULL( topic_id, 268435455 ) topicId
FROM TRAFFIC_HTTP_STATISTIC t FROM TRAFFIC_HTTP_STATISTIC t
LEFT JOIN ui_website_domain_topic u ON t.web_id = u.id LEFT JOIN ui_website_domain_topic u ON t.web_id = u.id
FROM TRAFFIC_HTTP_STATISTIC t and t.stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour)
LEFT JOIN ui_website_domain_topic u ON t.web_id = u.id
GROUP BY u.topic_id ORDER BY count desc limit 0,10 GROUP BY u.topic_id ORDER BY count desc limit 0,10
GROUP BY u.topic_id ORDER BY count desc limit 0,10 </select>
<!--获取上个时间段该网站站域名流量的数据量 --> <!--获取上个时间段该网站站域名流量的数据量 -->
<select id="preWebsiteListCount" resultType="java.lang.Long"> <select id="preWebsiteListCount" resultType="java.lang.Long">
SELECT SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_http_statistic t SELECT SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_http_statistic t
LEFT JOIN ui_website_domain_topic u ON t.web_id = u.id LEFT JOIN ui_website_domain_topic u ON t.web_id = u.id
WHERE u.website_service_id=#{websiteServiceId} WHERE u.website_service_id=#{websiteServiceId}
LEFT JOIN ui_website_domain_topic u ON t.web_id = u.id and stat_time &lt; DATE_SUB(#{statTime},INTERVAL 1 hour)
and stat_time > DATE_SUB(#{statTime},INTERVAL 2 hour)
</select> </select>
<!--获取网站列表列表 --> <!--获取网站列表列表 -->
@@ -56,7 +57,7 @@
FROM traffic_http_statistic t FROM traffic_http_statistic t
LEFT JOIN ui_website_domain_topic u ON t.web_id=u.id LEFT JOIN ui_website_domain_topic u ON t.web_id=u.id
where u.website_service_id=#{websiteServiceId} where u.website_service_id=#{websiteServiceId}
LEFT JOIN ui_website_domain_topic u ON t.web_id=u.id and t.stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour)
GROUP BY t.web_id ORDER BY count desc limit 0,10 GROUP BY t.web_id ORDER BY count desc limit 0,10
</select> </select>
<!-- 根据主题分组获取子域名 --> <!-- 根据主题分组获取子域名 -->
@@ -65,7 +66,7 @@
FROM traffic_http_statistic t FROM traffic_http_statistic t
LEFT JOIN ui_website_domain_topic u ON t.web_id=u.id LEFT JOIN ui_website_domain_topic u ON t.web_id=u.id
where u.topic_id=#{topicId} where u.topic_id=#{topicId}
LEFT JOIN ui_website_domain_topic u ON t.web_id=u.id and t.stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour)
GROUP BY t.web_id ORDER BY count desc limit 0,10 GROUP BY t.web_id ORDER BY count desc limit 0,10
</select> </select>
@@ -74,13 +75,13 @@
SELECT SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_http_statistic t SELECT SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_http_statistic t
LEFT JOIN ui_website_domain_topic u ON t.web_id=u.id LEFT JOIN ui_website_domain_topic u ON t.web_id=u.id
where u.website_service_id=#{websiteServiceId} where u.website_service_id=#{websiteServiceId}
LEFT JOIN ui_website_domain_topic u ON t.web_id=u.id
<if test="webIdList!=null and webIdList.size()>0"> <if test="webIdList!=null and webIdList.size()>0">
and t.web_id not in and t.web_id not in
<foreach collection="webIdList" item="singleType" index="index" open="(" close=")" separator=","> <foreach collection="webIdList" item="singleType" index="index" open="(" close=")" separator=",">
#{singleType} #{singleType}
</foreach> </foreach>
</if> </if>
</foreach> and stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour)
</select> </select>
</mapper> </mapper>

View File

@@ -24,12 +24,12 @@
</select> --> </select> -->
<!-- IP流量统计五分钟--> <!-- IP流量统计五分钟-->
<select id="getMaxStatTime" resultType="com.nis.domain.restful.dashboard.TrafficIpActiveStatistic"> <select id="getMaxStatTime" resultType="com.nis.domain.restful.dashboard.TrafficIpActiveStatistic">
SELECT MAX(stat_time) statTime FROM traffic_ip_active_statistic SELECT stat_time statTime FROM traffic_ip_active_statistic order by stat_time desc limit 1
</select> </select>
<!-- IP流量统计五分钟--> <!-- IP流量统计五分钟-->
<select id="ipActiveChart" resultType="java.util.LinkedHashMap"> <select id="ipActiveChart" resultType="java.util.LinkedHashMap">
SELECT ip_addr ipAddr, SUM(c2s_byte_len+s2c_byte_len) linkNum FROM traffic_ip_active_statistic SELECT ip_addr ipAddr, SUM(c2s_byte_len+s2c_byte_len) linkNum FROM traffic_ip_active_statistic
WHERE stat_time = #{statTime} WHERE stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour)
GROUP BY ip_addr ORDER BY linkNum DESC limit 0,10 GROUP BY ip_addr ORDER BY linkNum DESC limit 0,10
</select> </select>

View File

@@ -11,18 +11,19 @@
<!-- 获取最大时间 --> <!-- 获取最大时间 -->
<select id="getMaxStatTime" resultMap="BaseResultMap"> <select id="getMaxStatTime" resultMap="BaseResultMap">
SELECT MAX(stat_time) statTime FROM traffic_port_active_statistic SELECT stat_time statTime FROM traffic_port_active_statistic order by stat_time desc limit 1
</select> </select>
<!-- 获取当前时间五分钟数据 --> <!-- 获取当前时间五分钟数据 -->
<select id="getPortActiveList" resultMap="BaseResultMap"> <select id="getPortActiveList" resultMap="BaseResultMap">
SELECT port ,SUM(sum) sum from traffic_port_active_statistic SELECT port ,SUM(sum) sum from traffic_port_active_statistic
WHERE stat_time = #{statTime} WHERE stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour)
GROUP BY port order by sum desc limit 0,10 GROUP BY port order by sum desc limit 0,10
</select> </select>
<!-- 获取上个时间段的数据 --> <!-- 获取上个时间段的数据 -->
<select id="getPortActiveOld" resultMap="BaseResultMap"> <select id="getPortActiveOld" resultMap="BaseResultMap">
SELECT SUM(sum) sum from traffic_port_active_statistic SELECT SUM(sum) sum from traffic_port_active_statistic
WHERE port=#{port} and stat_time = DATE_SUB(#{statTime},INTERVAL 5 MINUTE) WHERE port=#{port} and stat_time &lt; DATE_SUB(#{statTime},INTERVAL 1 hour)
and stat_time > DATE_SUB(#{statTime},INTERVAL 2 hour)
GROUP BY port ORDER BY sum DESC limit 0,10 GROUP BY port ORDER BY sum DESC limit 0,10
</select> </select>
</mapper> </mapper>

View File

@@ -13,5 +13,5 @@ public interface TrafficProtocolStatisticDao {
TrafficProtocolStatistic getMaxStatTime(); TrafficProtocolStatistic getMaxStatTime();
List<Map> protocolChart(@Param("statTime")Date statTime); List<Map> protocolChart(@Param("statTime")Date statTime);
List<Map> getProtocolList(@Param("statTime")Date statTime); List<Map> getProtocolList(@Param("statTime")Date statTime,@Param("beginTime")String beginTime,@Param("endTime")String endTime);
} }

View File

@@ -17,18 +17,25 @@
</sql> </sql>
<!-- 获取最近时间 --> <!-- 获取最近时间 -->
<select id="getMaxStatTime" resultType="com.nis.domain.restful.dashboard.TrafficProtocolStatistic"> <select id="getMaxStatTime" resultType="com.nis.domain.restful.dashboard.TrafficProtocolStatistic">
SELECT MAX(stat_time) statTime FROM traffic_protocol_statistic WHERE proto_type != 0 SELECT stat_time statTime FROM traffic_protocol_statistic WHERE proto_type != 0 order by stat_time desc limit 1
</select> </select>
<select id="protocolChart" resultType="java.util.HashMap"> <select id="protocolChart" resultType="java.util.HashMap">
SELECT proto_type protoType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_protocol_statistic WHERE proto_type != 0 SELECT proto_type protoType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_protocol_statistic WHERE
AND stat_time = #{statTime} GROUP BY proto_type ORDER BY count DESC LIMIT 0, 10 stat_time > DATE_SUB(#{statTime},INTERVAL 1 hour) GROUP BY proto_type ORDER BY count DESC LIMIT 0, 10
</select> </select>
<select id="getProtocolList" resultType="java.util.HashMap"> <select id="getProtocolList" resultType="java.util.HashMap">
SELECT IFNULL(c.protocol_type,p.proto_type) protocolType,p.proto_type protoType,p.link_num linkNum, p.packets packets, p.GByte GByte FROM ( SELECT IFNULL(c.protocol_type,p.proto_type) protocolType,p.proto_type protoType,p.link_num linkNum, p.packets packets, p.GByte GByte FROM (
SELECT proto_type,SUM(link_num) AS link_num,SUM(c2s_pkt_num+s2c_pkt_num) AS packets, SELECT proto_type,SUM(link_num) AS link_num,SUM(c2s_pkt_num+s2c_pkt_num) AS packets,
(SUM(c2s_byte_len+s2c_byte_len)/1024/1024/1024) AS GByte (SUM(c2s_byte_len+s2c_byte_len)/1024/1024/1024) AS GByte
FROM traffic_protocol_statistic WHERE stat_time=#{statTime} GROUP BY proto_type ) p FROM traffic_protocol_statistic WHERE
<if test="beginTime==null or endTime==null">
stat_time > DATE_SUB(#{statTime},INTERVAL 1 HOUR)
</if>
<if test="beginTime!=null and endTime!=null">
stat_time >= #{beginTime} and stat_time &lt;= #{endTime}
</if>
GROUP BY proto_type ) p
LEFT JOIN ui_code_protocol_type_dic c ON p.proto_type=c.view_code ORDER BY p.GByte DESC LEFT JOIN ui_code_protocol_type_dic c ON p.proto_type=c.view_code ORDER BY p.GByte DESC
</select> </select>
</mapper> </mapper>

View File

@@ -18,24 +18,25 @@
</sql> </sql>
<!-- 获取最近时间并且有效 --> <!-- 获取最近时间并且有效 -->
<select id="getMaxStatTime" resultType="com.nis.domain.restful.dashboard.TrafficUaStatistic"> <select id="getMaxStatTime" resultType="com.nis.domain.restful.dashboard.TrafficUaStatistic">
SELECT MAX(stat_time) statTime FROM traffic_ua_statistic WHERE os_type !=0 and bs_type!=0 SELECT stat_time statTime FROM traffic_ua_statistic order by stat_time desc limit 1
</select> </select>
<!--获取操作系统列表TOP10 --> <!--获取操作系统列表TOP10 -->
<select id="systemList" resultMap="BaseResultMap"> <select id="systemList" resultMap="BaseResultMap">
SELECT os_type osType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic SELECT os_type osType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic
WHERE os_type !=0 and stat_time = #{statTime} WHERE stat_time > DATE_SUB(#{statTime}, INTERVAL 1 HOUR)
GROUP BY os_type ORDER BY count DESC limit 0,10 GROUP BY os_type ORDER BY count DESC limit 0,10
</select> </select>
<!--获取上个时间段操作系统的数据量 --> <!--获取上个时间段操作系统的数据量 -->
<select id="preSystemListCount" resultType="java.lang.Long"> <select id="preSystemListCount" resultType="java.lang.Long">
SELECT SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic SELECT SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic
WHERE os_type !=0 and os_type=#{osType} WHERE os_type=#{osType}
and stat_time = DATE_SUB(#{statTime},INTERVAL 5 MINUTE) and stat_time &lt; DATE_SUB(#{statTime},INTERVAL 1 HOUR)
and stat_time > DATE_SUB(#{statTime},INTERVAL 2 HOUR)
</select> </select>
<!-- 根据操作系统获取浏览器分类 --> <!-- 根据操作系统获取浏览器分类 -->
<select id="getBrowserBySystem" resultType="java.util.HashMap"> <select id="getBrowserBySystem" resultType="java.util.HashMap">
SELECT bs_type bsType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic SELECT bs_type bsType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic
WHERE bs_type !=0 and os_type=#{osType} and stat_time = #{statTime} WHERE os_type=#{osType} and stat_time > DATE_SUB(#{statTime},INTERVAL 1 HOUR)
GROUP BY bs_type ORDER BY count DESC limit 0,10 GROUP BY bs_type ORDER BY count DESC limit 0,10
</select> </select>
<!--浏览器TOP10后所有为others --> <!--浏览器TOP10后所有为others -->
@@ -48,25 +49,24 @@
#{singleType} #{singleType}
</foreach> </foreach>
</if> </if>
and stat_time = #{statTime} and stat_time > DATE_SUB(#{statTime},INTERVAL 1 HOUR)
</select> </select>
<!--获取浏览器列表TOP10 --> <!--获取浏览器列表TOP10 -->
<select id="browserList" resultMap="BaseResultMap"> <select id="browserList" resultMap="BaseResultMap">
SELECT bs_type bsType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic SELECT bs_type bsType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic
WHERE bs_type !=0 and stat_time = #{statTime} WHERE stat_time > DATE_SUB(#{statTime},INTERVAL 1 HOUR)
GROUP BY bs_type ORDER BY count DESC limit 0,10 GROUP BY bs_type ORDER BY count DESC limit 0,10
</select> </select>
<!--获取上个时间段浏览器的数据量 --> <!--获取上个时间段浏览器的数据量 -->
<select id="preBrowserListCount" resultType="java.lang.Long"> <select id="preBrowserListCount" resultType="java.lang.Long">
SELECT SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic SELECT SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic
WHERE bs_type !=0 and bs_type=#{bsType} WHERE bs_type=#{bsType}
and stat_time = DATE_SUB(#{statTime},INTERVAL 5 MINUTE) and stat_time &lt; DATE_SUB(#{statTime},INTERVAL 1 HOUR) and stat_time > DATE_SUB(#{statTime},INTERVAL 2 HOUR)
</select> </select>
<!-- 根据浏览器获取操作系统分类 --> <!-- 根据浏览器获取操作系统分类 -->
<select id="getSystemBybrowser" resultType="java.util.HashMap"> <select id="getSystemBybrowser" resultType="java.util.HashMap">
SELECT os_type osType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic SELECT os_type osType, SUM(c2s_byte_len+s2c_byte_len) count FROM traffic_ua_statistic
WHERE os_type !=0 and bs_type=#{bsType} and stat_time = #{statTime} WHERE bs_type=#{bsType} and stat_time > DATE_SUB(#{statTime},INTERVAL 1 HOUR)
GROUP BY os_type ORDER BY count DESC limit 0,10 GROUP BY os_type ORDER BY count DESC limit 0,10
</select> </select>
<!--操作系统TOP10后所有为others --> <!--操作系统TOP10后所有为others -->
@@ -79,6 +79,6 @@
#{singleType} #{singleType}
</foreach> </foreach>
</if> </if>
and stat_time = #{statTime} and stat_time > DATE_SUB(#{statTime},INTERVAL 1 HOUR)
</select> </select>
</mapper> </mapper>

View File

@@ -30,6 +30,8 @@ import com.nis.util.ReadMaatXmlUtil;
import com.nis.util.ServiceAndRDBIndexReal; import com.nis.util.ServiceAndRDBIndexReal;
import redis.clients.jedis.Jedis; import redis.clients.jedis.Jedis;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
import redis.clients.jedis.Transaction; import redis.clients.jedis.Transaction;
import redis.clients.jedis.exceptions.JedisConnectionException; import redis.clients.jedis.exceptions.JedisConnectionException;
@@ -568,8 +570,7 @@ public class ConfigJedisServiceimpl implements ConfigRedisService {
* @param redisTemplate * @param redisTemplate
* @param redisDBIndex * @param redisDBIndex
* @param compileId * @param compileId
* @param isReuse * @param isReuse 是否是分组复用
* 是否是分组复用
*/ */
private void setCommonConfig(MaatXmlConfig maatXmlConfig, Map<String, String> map, int type, Double maatVersion, private void setCommonConfig(MaatXmlConfig maatXmlConfig, Map<String, String> map, int type, Double maatVersion,
int service, Transaction transaction, Integer redisDBIndex, String compileId, boolean isReuse) { int service, Transaction transaction, Integer redisDBIndex, String compileId, boolean isReuse) {
@@ -1028,17 +1029,12 @@ public class ConfigJedisServiceimpl implements ConfigRedisService {
/** /**
* 删除maat类配置 * 删除maat类配置
* *
* @param id * @param id 配置id
* 配置id * @param maatXmlConfig maat.xml中关于当前业务类型的key和value写法的对象
* @param maatXmlConfig * @param maatVersion 版本号
* maat.xml中关于当前业务类型的key和value写法的对象 * @param service 业务类型
* @param maatVersion
* 版本号
* @param service
* 业务类型
* @param redisTemplate * @param redisTemplate
* @param maatRelation * @param maatRelation id对应关系对象
* id对应关系对象
*/ */
private void removeConfig(Long id, MaatXmlConfig maatXmlConfig, Double maatVersion, int service, private void removeConfig(Long id, MaatXmlConfig maatXmlConfig, Double maatVersion, int service,
Transaction transaction, int redisDBIndex, int idRelaRedisDBIndex) { Transaction transaction, int redisDBIndex, int idRelaRedisDBIndex) {
@@ -1700,20 +1696,13 @@ public class ConfigJedisServiceimpl implements ConfigRedisService {
/** /**
* 保存域配置信息,供分组复用单独添加域和普通maat配置添加域 * 保存域配置信息,供分组复用单独添加域和普通maat配置添加域
* *
* @param maatConfig * @param maatConfig 配置信息
* 配置信息 * @param maatXmlConfig 配置的表达式
* @param maatXmlConfig * @param maatVersion 版本号
* 配置的表达式 * @param service 业务类型
* @param maatVersion * @param transaction redis连接
* 版本 * @param redisDBIndex redis编
* @param service * @param isReuseSaveRegion 是否是分组复用单独添加域配置
* 业务类型
* @param transaction
* redis连接
* @param redisDBIndex
* redis编号
* @param isReuseSaveRegion
* 是否是分组复用单独添加域配置
*/ */
private Integer addRegion(MaatConfig maatConfig, MaatXmlConfig maatXmlConfig, Double maatVersion, int service, private Integer addRegion(MaatConfig maatConfig, MaatXmlConfig maatXmlConfig, Double maatVersion, int service,
Transaction transaction, Integer redisDBIndex, boolean isReuseSaveRegion) { Transaction transaction, Integer redisDBIndex, boolean isReuseSaveRegion) {
@@ -1957,6 +1946,7 @@ public class ConfigJedisServiceimpl implements ConfigRedisService {
JedisUtils.returnResource(resource); JedisUtils.returnResource(resource);
} }
} }
private void updateMaatInfo(List<MaatXmlExpr> list, String maatKey, Transaction transaction, Double maatVersion, private void updateMaatInfo(List<MaatXmlExpr> list, String maatKey, Transaction transaction, Double maatVersion,
int redisDBIndex, boolean idDel) { int redisDBIndex, boolean idDel) {
if (list != null && list.size() > 0) { if (list != null && list.size() > 0) {
@@ -1991,4 +1981,78 @@ public class ConfigJedisServiceimpl implements ConfigRedisService {
} }
} }
public Map<Integer, Map<String, String>> getAllKVByCompileId(Long id) {
Jedis resource = JedisUtils.getResource(1);
try {
Map<Integer, Map<String, String>> map = new HashMap<>();
String pattern = "*," + id;
for (int index = 2; index < 6; index++) {
resource.select(index);
List<String> keyByScan = getKeyByScan(pattern, resource);
for (String redisKey : keyByScan) {
if (resource.exists(redisKey)) {
String statusMebmer = null;
if (redisKey.startsWith("EFFECTIVE_RULE:")) {
statusMebmer = redisKey.replace("EFFECTIVE_RULE:", "ADD,");
} else if (redisKey.startsWith("OBSOLETE_RULE:")) {
statusMebmer = redisKey.replace("OBSOLETE_RULE:", "DEL,");
}
if (statusMebmer != null) {
Double zscore = resource.zscore("MAAT_UPDATE_STATUS", statusMebmer);
if (zscore != null) {
Set<String> zrangeByScore = resource.zrangeByScore("MAAT_UPDATE_STATUS", zscore,
zscore);
for (String statusKey : zrangeByScore) {
String replaceKey = null;
if (statusKey.startsWith("ADD,")) {
replaceKey = statusKey.replace("ADD,", "EFFECTIVE_RULE:");
} else if (statusKey.startsWith("DEL,")) {
replaceKey = statusKey.replace("DEL,", "OBSOLETE_RULE:");
}
if (replaceKey != null) {
String val = resource.get(replaceKey);
if (val != null) {
if (map.containsKey(index)) {
Map<String, String> map2 = map.get(index);
map2.put(replaceKey, val);
} else {
Map<String, String> map2 = new HashMap<>();
map2.put(replaceKey, val);
map.put(index, map2);
}
}
}
}
}
}
}
}
}
return map;
} finally {
resource.close();
}
}
public List<String> getKeyByScan(String pattern, Jedis resource) {
List<String> list = new ArrayList<>();
int count = 1000;
String cursor = ScanParams.SCAN_POINTER_START;
ScanParams scanParams = new ScanParams();
scanParams.count(count);
scanParams.match(pattern);
do {
ScanResult<String> scanResult = resource.scan(cursor, scanParams);
list.addAll(scanResult.getResult());
cursor = scanResult.getStringCursor();
} while ("!0".equals(cursor));
return list;
}
} }

View File

@@ -7,16 +7,24 @@ import com.nis.domain.restful.MaatConfig;
/** /**
* *
* <p>Title: ConfigRedisService</p> * <p>
* <p>Description: 保存配置到redis中</p> * Title: ConfigRedisService
* <p>Company: IIE</p> * </p>
* <p>
* Description: 保存配置到redis中
* </p>
* <p>
* Company: IIE
* </p>
*
* @author rkg * @author rkg
* @date 2018年5月17日 * @date 2018年5月17日
* *
*/ */
public interface ConfigRedisService { public interface ConfigRedisService {
/** /**
* 下发非maat配置接口,key是redisDBIndex,value是配置集合 * 下发非maat配置接口,key是redisDBIndex,value是配置集合
*
* @param configMap * @param configMap
* @return 成功返回true,失败返回false或抛出异常 * @return 成功返回true,失败返回false或抛出异常
*/ */
@@ -24,6 +32,7 @@ public interface ConfigRedisService {
/** /**
* 保存 maat配置接口,key是redisDBIndex,value是配置集合 * 保存 maat配置接口,key是redisDBIndex,value是配置集合
*
* @param configMap * @param configMap
* @return 成功返回true,失败返回false或抛出异常 * @return 成功返回true,失败返回false或抛出异常
*/ */
@@ -31,27 +40,32 @@ public interface ConfigRedisService {
/** /**
* 分组复用域配置新增接口,value是配置集合 * 分组复用域配置新增接口,value是配置集合
*
* @param configList * @param configList
* @return 成功返回true,失败返回false或抛出异常 * @return 成功返回true,失败返回false或抛出异常
*/ */
public boolean saveGroupReuseConfig( List<MaatConfig> configList); public boolean saveGroupReuseConfig(List<MaatConfig> configList);
/** /**
* 获取指定key的自增长值 * 获取指定key的自增长值
*
* @param key 需要自增的key * @param key 需要自增的key
* @return * @return
*/ */
public Long getIncrId(String key); public Long getIncrId(String key);
/** /**
* 批量获取接口,利用java进行遍历,比从redis中自增要快很多 * 批量获取接口,利用java进行遍历,比从redis中自增要快很多
*
* @param key * @param key
* @param num * @param num
* @return * @return
*/ */
public Long getIncrById(String key,int num); public Long getIncrById(String key, int num);
/** /**
* 删除非maat类配置,第一个key是redisDBIndex,第二个key是业务类型,value是配置id集合 * 删除非maat类配置,第一个key是redisDBIndex,第二个key是业务类型,value是配置id集合
*
* @param idMap * @param idMap
* @param isInvalid 是否将配置置为无效 * @param isInvalid 是否将配置置为无效
* @return 成功返回true,失败返回false或抛出异常 * @return 成功返回true,失败返回false或抛出异常
@@ -60,6 +74,7 @@ public interface ConfigRedisService {
/** /**
* 删除maat类配置,key是业务类型,value是配置id集合 * 删除maat类配置,key是业务类型,value是配置id集合
*
* @param serviceConfigMap * @param serviceConfigMap
* @return 成功返回true,失败返回false或抛出异常 * @return 成功返回true,失败返回false或抛出异常
*/ */
@@ -67,9 +82,17 @@ public interface ConfigRedisService {
/** /**
* 删除分组复用的域配置 * 删除分组复用的域配置
*
* @param reuseMap key是groupId,value是"tableName,regionId" * @param reuseMap key是groupId,value是"tableName,regionId"
* @return * @return
*/ */
public boolean delGroupReuseConfig( Map<Long, List<String>> reuseMap); public boolean delGroupReuseConfig(Map<Long, List<String>> reuseMap);
/**
* 根据配置id获取对应的编译,组,域等信息
*
* @param id
* @return
*/
public Map<Integer, Map<String, String>> getAllKVByCompileId(Long id);
} }

View File

@@ -306,12 +306,12 @@ public class DashboardService extends BaseService{
* 协议统计报表 * 协议统计报表
* @return * @return
*/ */
public List<Map> getProtocolList(){ public List<Map> getProtocolList(String startTime,String endTime){
TrafficProtocolStatistic maxStatTime = trafficProtocolStatisticDao.getMaxStatTime(); TrafficProtocolStatistic maxStatTime = trafficProtocolStatisticDao.getMaxStatTime();
List<Map> list=new ArrayList<Map>(); List<Map> list=new ArrayList<Map>();
if(maxStatTime!=null&&maxStatTime.getStatTime()!=null) { if(maxStatTime!=null&&maxStatTime.getStatTime()!=null) {
Date statTime = maxStatTime.getStatTime(); Date statTime = maxStatTime.getStatTime();
list = trafficProtocolStatisticDao.getProtocolList(statTime); list = trafficProtocolStatisticDao.getProtocolList(statTime,startTime,endTime);
} }
return list; return list;
} }
@@ -339,12 +339,12 @@ public class DashboardService extends BaseService{
* App统计报表 * App统计报表
* @return * @return
*/ */
public List<Map> getAppList(){ public List<Map> getAppList(String startTime,String endTime){
TrafficAppStatistic maxStatTime = trafficAppStatisticDao.getMaxStatTime(); TrafficAppStatistic maxStatTime = trafficAppStatisticDao.getMaxStatTime();
List<Map> list=new ArrayList<Map>(); List<Map> list=new ArrayList<Map>();
if(maxStatTime!=null&&maxStatTime.getStatTime()!=null) { if(maxStatTime!=null&&maxStatTime.getStatTime()!=null) {
Date statTime = maxStatTime.getStatTime(); Date statTime = maxStatTime.getStatTime();
list = trafficAppStatisticDao.getAppList(statTime); list = trafficAppStatisticDao.getAppList(statTime,startTime,endTime);
} }
return list; return list;
} }

View File

@@ -11,15 +11,17 @@ import java.util.UUID;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.PropertySource;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import com.nis.restful.RestBusinessCode;
import com.nis.restful.ServiceRuntimeException;
import com.nis.util.Configurations; import com.nis.util.Configurations;
import com.nis.util.Constants; import com.nis.util.Constants;
import com.nis.util.ExceptionUtil; import com.nis.util.ExceptionUtil;
import com.nis.util.JedisUtils; import com.nis.util.JedisUtils;
import com.nis.web.service.SpringContextHolder;
import com.nis.web.service.restful.ConfigJedisServiceimpl; import com.nis.web.service.restful.ConfigJedisServiceimpl;
import redis.clients.jedis.Jedis; import redis.clients.jedis.Jedis;
@@ -28,6 +30,7 @@ import redis.clients.jedis.JedisPool;
import redis.clients.jedis.ScanParams; import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult; import redis.clients.jedis.ScanResult;
import redis.clients.jedis.Tuple; import redis.clients.jedis.Tuple;
import redis.clients.jedis.exceptions.JedisException;
import redis.clients.util.JedisClusterCRC16; import redis.clients.util.JedisClusterCRC16;
@Component @Component
@@ -36,14 +39,46 @@ public class SyncRedisToCluster {
private static Logger logger = LoggerFactory.getLogger(ConfigJedisServiceimpl.class); private static Logger logger = LoggerFactory.getLogger(ConfigJedisServiceimpl.class);
private static int redisStatisticsRealDBIndex = Configurations.getIntProperty("redisStatisticsRealDBIndex", 14); private static int redisStatisticsRealDBIndex = Configurations.getIntProperty("redisStatisticsRealDBIndex", 14);
@Autowired /**
private JedisCluster jedisCluster; * 每次使用获取新连接,避免集群某些机器宕机后影响连接使用,注意需要在applicationContext-redis.xml中修改jedisCluster的scope为
* prototype
*
* @return
* @throws JedisException
*/
public JedisCluster getResource() throws JedisException {
JedisCluster jedisCluster = SpringContextHolder.getBean(JedisCluster.class);
if (jedisCluster == null) {
throw new ServiceRuntimeException("无法获取redis-cluster连接,请联系管理员检查程序",
RestBusinessCode.CannotConnectionRedis.getValue());
}
return jedisCluster;
}
/**
* 关闭集群连接
*
* @param jedisCluster
*/
private void closeConn(JedisCluster jedisCluster) {
if (jedisCluster != null) {
try {
jedisCluster.close();
} catch (Exception e) {
throw new ServiceRuntimeException("释放redis-cluster连接失败",
RestBusinessCode.CannotConnectionRedis.getValue());
}
}
}
@Scheduled(cron = "${syncRedisToClusterCron}") @Scheduled(cron = "${syncRedisToClusterCron}")
public void syncRedisToCluster() { public void syncRedisToCluster() {
JedisCluster jedisCluster = getResource();
System.out.println(jedisCluster);
String requestId = UUID.randomUUID().toString(); String requestId = UUID.randomUUID().toString();
try { try {
if (lock(requestId)) {// 避免集群环境下同一秒钟所有的机器都执行这个定时任务 if (lock(jedisCluster, requestId)) {// 避免集群环境下同一秒钟所有的机器都执行这个定时任务
// keys("EFFECTIVE_RULE*"); // keys("EFFECTIVE_RULE*");
// keys("OBSOLETE_RULE*"); // keys("OBSOLETE_RULE*");
String clusterMaatVersionStr = jedisCluster.get("MAAT_VERSION"); String clusterMaatVersionStr = jedisCluster.get("MAAT_VERSION");
@@ -54,11 +89,11 @@ public class SyncRedisToCluster {
Integer redisMaatVersion = Integer.valueOf(redisMaatVersionStr); Integer redisMaatVersion = Integer.valueOf(redisMaatVersionStr);
if (redisMaatVersion < clusterMaatVersion) {// 如果主从库比集群库的版本号小则下发全量 if (redisMaatVersion < clusterMaatVersion) {// 如果主从库比集群库的版本号小则下发全量
logger.info("redis集群中的MAAT_VERSION为大于配置库中的MAAT_VERSION,开始执行全量同步"); logger.info("redis集群中的MAAT_VERSION为大于配置库中的MAAT_VERSION,开始执行全量同步");
syncAllData(redisMaatVersionStr); syncAllData(jedisCluster, redisMaatVersionStr);
} else if (redisMaatVersion > clusterMaatVersion) {// 获取增量的数据 } else if (redisMaatVersion > clusterMaatVersion) {// 获取增量的数据
logger.info("redis集群中的MAAT_VERSION为小于配置库中的MAAT_VERSION,开始执行增量同步,score是{}-{}", logger.info("redis集群中的MAAT_VERSION为小于配置库中的MAAT_VERSION,开始执行增量同步,score是{}-{}",
clusterMaatVersion, +redisMaatVersion); clusterMaatVersion, +redisMaatVersion);
syncData(clusterMaatVersion.doubleValue(), redisMaatVersion.doubleValue(), syncData(jedisCluster, clusterMaatVersion.doubleValue(), redisMaatVersion.doubleValue(),
redisMaatVersionStr); redisMaatVersionStr);
} else { } else {
logger.info("redis集群中的MAAT_VERSION与配置库中的MAAT_VERSION相等,暂不执行配置同步操作"); logger.info("redis集群中的MAAT_VERSION与配置库中的MAAT_VERSION相等,暂不执行配置同步操作");
@@ -66,13 +101,13 @@ public class SyncRedisToCluster {
} else { } else {
logger.info("redis配置库中MAAT_VERSION为null,但是redis集群中的MAAT_VERSION为{},集群与配置库的数据不同步,开始删除集群中的配置", logger.info("redis配置库中MAAT_VERSION为null,但是redis集群中的MAAT_VERSION为{},集群与配置库的数据不同步,开始删除集群中的配置",
clusterMaatVersionStr); clusterMaatVersionStr);
delClusterData(); delClusterData(jedisCluster);
} }
} else { } else {
if (redisMaatVersionStr != null && !redisMaatVersionStr.trim().equals("")) { if (redisMaatVersionStr != null && !redisMaatVersionStr.trim().equals("")) {
logger.info("redis配置库中的MAAT_VERSION为{},redis集群中的MAAT_VERSION为null,开始执行全量同步", logger.info("redis配置库中的MAAT_VERSION为{},redis集群中的MAAT_VERSION为null,开始执行全量同步",
redisMaatVersionStr); redisMaatVersionStr);
syncAllData(redisMaatVersionStr); syncAllData(jedisCluster, redisMaatVersionStr);
} else { } else {
logger.info("redis配置库中和redis集群中的MAAT_VERSION都为null,暂时不执行全量同步"); logger.info("redis配置库中和redis集群中的MAAT_VERSION都为null,暂时不执行全量同步");
} }
@@ -83,12 +118,13 @@ public class SyncRedisToCluster {
} catch (Exception e) { } catch (Exception e) {
logger.error("同步配置库配置到3A-redisCluster失败,失败原因:{}", ExceptionUtil.getExceptionMsg(e)); logger.error("同步配置库配置到3A-redisCluster失败,失败原因:{}", ExceptionUtil.getExceptionMsg(e));
} finally { } finally {
unlock(requestId); unlock(jedisCluster, requestId);
closeConn(jedisCluster);
} }
} }
// 删除之前的key // 删除之前的key
public void deleteRedisKeyStartWith(String redisKeyStartWith) { public void deleteRedisKeyStartWith(JedisCluster jedisCluster, String redisKeyStartWith) {
try { try {
Map<String, JedisPool> clusterNodes = jedisCluster.getClusterNodes(); Map<String, JedisPool> clusterNodes = jedisCluster.getClusterNodes();
for (Map.Entry<String, JedisPool> entry : clusterNodes.entrySet()) { for (Map.Entry<String, JedisPool> entry : clusterNodes.entrySet()) {
@@ -126,7 +162,7 @@ public class SyncRedisToCluster {
} }
} }
public TreeSet<String> keys(String pattern) { public TreeSet<String> keys(JedisCluster jedisCluster, String pattern) {
TreeSet<String> keys = new TreeSet<>(); TreeSet<String> keys = new TreeSet<>();
// 获取所有的节点 // 获取所有的节点
Map<String, JedisPool> clusterNodes = jedisCluster.getClusterNodes(); Map<String, JedisPool> clusterNodes = jedisCluster.getClusterNodes();
@@ -150,7 +186,7 @@ public class SyncRedisToCluster {
return keys; return keys;
} }
public void delClusterData() { public void delClusterData(JedisCluster jedisCluster) {
// 清空集群库配置数据 // 清空集群库配置数据
if (jedisCluster.exists("MAAT_UPDATE_STATUS")) { if (jedisCluster.exists("MAAT_UPDATE_STATUS")) {
jedisCluster.del("MAAT_UPDATE_STATUS"); jedisCluster.del("MAAT_UPDATE_STATUS");
@@ -160,18 +196,18 @@ public class SyncRedisToCluster {
jedisCluster.del("MAAT_VERSION"); jedisCluster.del("MAAT_VERSION");
logger.info("删除MAAT_VERSION成功"); logger.info("删除MAAT_VERSION成功");
} }
deleteRedisKeyStartWith("EFFECTIVE_RULE*"); deleteRedisKeyStartWith(jedisCluster, "EFFECTIVE_RULE*");
logger.info("删除EFFECTIVE_RULE*成功"); logger.info("删除EFFECTIVE_RULE*成功");
deleteRedisKeyStartWith("OBSOLETE_RULE*"); deleteRedisKeyStartWith(jedisCluster, "OBSOLETE_RULE*");
logger.info("删除OBSOLETE_RULE*成功"); logger.info("删除OBSOLETE_RULE*成功");
} }
public void syncAllData(String version) { public void syncAllData(JedisCluster jedisCluster, String version) {
delClusterData(); delClusterData(jedisCluster);
syncData(null, null, version); syncData(jedisCluster, null, null, version);
} }
public void syncData(Double min, Double max, String verionStr) { public void syncData(JedisCluster jedisCluster, Double min, Double max, String verionStr) {
Jedis resource = JedisUtils.getResource(redisStatisticsRealDBIndex); Jedis resource = JedisUtils.getResource(redisStatisticsRealDBIndex);
Set<Tuple> zrangeByScoreWithScores = null; Set<Tuple> zrangeByScoreWithScores = null;
if (min == null && max == null) { if (min == null && max == null) {
@@ -190,20 +226,20 @@ public class SyncRedisToCluster {
} }
min = Double.parseDouble(start + ""); min = Double.parseDouble(start + "");
max = Double.parseDouble(end + ""); max = Double.parseDouble(end + "");
//分批获取,避免一次性获取太多,造成redis阻塞 // 分批获取,避免一次性获取太多,造成redis阻塞
zrangeByScoreWithScores = resource.zrangeByScoreWithScores("MAAT_UPDATE_STATUS", min, max);// 获取所有的maat_update_status zrangeByScoreWithScores = resource.zrangeByScoreWithScores("MAAT_UPDATE_STATUS", min, max);// 获取所有的maat_update_status
syncData(zrangeByScoreWithScores); syncData(jedisCluster, zrangeByScoreWithScores);
} }
} else { } else {
zrangeByScoreWithScores = resource.zrangeByScoreWithScores("MAAT_UPDATE_STATUS", min, max);// 获取增量的数据 zrangeByScoreWithScores = resource.zrangeByScoreWithScores("MAAT_UPDATE_STATUS", min, max);// 获取增量的数据
syncData(zrangeByScoreWithScores); syncData(jedisCluster, zrangeByScoreWithScores);
} }
jedisCluster.set("MAAT_VERSION", verionStr); jedisCluster.set("MAAT_VERSION", verionStr);
logger.info("更新了redis集群中的MAAT_VERSION,更新后值是{}", verionStr); logger.info("更新了redis集群中的MAAT_VERSION,更新后值是{}", verionStr);
logger.info("向redis集群同步数据成功"); logger.info("向redis集群同步数据成功");
} }
private void syncData(Set<Tuple> zrangeByScoreWithScores) { private void syncData(JedisCluster jedisCluster, Set<Tuple> zrangeByScoreWithScores) {
for (Tuple tuple : zrangeByScoreWithScores) { for (Tuple tuple : zrangeByScoreWithScores) {
String key = tuple.getElement(); String key = tuple.getElement();
String zset = key; String zset = key;
@@ -239,7 +275,7 @@ public class SyncRedisToCluster {
* @param requestId 请求标识(UUID.randomUUID().toString()),正常情况下是谁加的锁,谁去解锁不能a加的锁,b去解锁 * @param requestId 请求标识(UUID.randomUUID().toString()),正常情况下是谁加的锁,谁去解锁不能a加的锁,b去解锁
* @return 是否获取成功 * @return 是否获取成功
*/ */
public Boolean lock(String requestId) { public Boolean lock(JedisCluster jedisCluster, String requestId) {
String key = "configSyncDistributedLock"; String key = "configSyncDistributedLock";
String var1 = jedisCluster.set(key, requestId, SET_IF_NOT_EXIST, SET_WITH_EXPIRE_TIME, Constants.REDISLOCKTIME); String var1 = jedisCluster.set(key, requestId, SET_IF_NOT_EXIST, SET_WITH_EXPIRE_TIME, Constants.REDISLOCKTIME);
if (LOCK_SUCCESS.equals(var1)) { if (LOCK_SUCCESS.equals(var1)) {
@@ -253,7 +289,7 @@ public class SyncRedisToCluster {
* *
* @param requestId * @param requestId
*/ */
protected boolean unlock(String requestId) { protected boolean unlock(JedisCluster jedisCluster, String requestId) {
String key = "configSyncDistributedLock"; String key = "configSyncDistributedLock";
// 这个字符串是个lua脚本代表的意思是如果根据key拿到的value跟传入的value相同就执行del否则就返回0【保证安全性】 // 这个字符串是个lua脚本代表的意思是如果根据key拿到的value跟传入的value相同就执行del否则就返回0【保证安全性】
String luaScript = "if redis.call(\"get\",KEYS[1]) == ARGV[1] then return redis.call(\"del\",KEYS[1]) else return 0 end"; String luaScript = "if redis.call(\"get\",KEYS[1]) == ARGV[1] then return redis.call(\"del\",KEYS[1]) else return 0 end";

View File

@@ -233,8 +233,9 @@
<t:field fieldType="String" srcName="service" dstName="service" isRequired="true"/> <t:field fieldType="String" srcName="service" dstName="service" isRequired="true"/>
<t:field fieldType="Number" srcName="action" dstName="action" isRequired="true"/> <t:field fieldType="Number" srcName="action" dstName="action" isRequired="true"/>
<t:field fieldType="Number" srcName="configId" dstName="cfg_id" isRequired="true" isCfgId="true"/> <t:field fieldType="Number" srcName="configId" dstName="cfg_id" isRequired="true" isCfgId="true"/>
<t:field fieldType="Number" srcName="originalAddrType" dstName="original_addr_type" isRequired="true" regexp="^([4|6]{1})$"/>
<t:field fieldType="String" srcName="originalDestIp" dstName="original_dest_ip" isRequired="true"/> <t:field fieldType="String" srcName="originalDestIp" dstName="original_dest_ip" isRequired="true"/>
<t:field fieldType="String" srcName="origzinalDestPort" dstName="original_dest_port" isRequired="true"/> <t:field fieldType="String" srcName="originalDestPort" dstName="original_dest_port" isRequired="true"/>
<t:field fieldType="String" srcName="originalProtocol" dstName="original_protocol" isRequired="true"/> <t:field fieldType="String" srcName="originalProtocol" dstName="original_protocol" isRequired="true"/>
<t:field fieldType="String" srcName="translateDestIp" dstName="translate_dest_ip" isRequired="true"/> <t:field fieldType="String" srcName="translateDestIp" dstName="translate_dest_ip" isRequired="true"/>
<t:field fieldType="String" srcName="translateDestPort" dstName="translate_dest_port" isRequired="true"/> <t:field fieldType="String" srcName="translateDestPort" dstName="translate_dest_port" isRequired="true"/>

View File

@@ -184,7 +184,7 @@
<p:maatType service="836"> <p:maatType service="836">
<p:expressions> <p:expressions>
<p:keyExpression>EFFECTIVE_RULE;:;{un_maat_table_name};,;[cfg_id]</p:keyExpression> <p:keyExpression>EFFECTIVE_RULE;:;{un_maat_table_name};,;[cfg_id]</p:keyExpression>
<p:valueExpression>[cfg_id];\t;[original_dest_ip];\t;[original_dest_port];\t;[original_protocol];\t;[translate_dest_ip];\t;[translate_dest_port];\t;[do_log];\t;[action];\t;[service];\t;[is_valid];\t;[effective_range];\t;[op_time];&amp;nbsp;0;\n</p:valueExpression> <p:valueExpression>[cfg_id];\t;[original_addr_type];\t;[original_dest_ip];\t;[original_dest_port];\t;[original_protocol];\t;[translated_dest_ip];\t;[translated_dest_port];\t;[do_log];\t;[action];\t;[service];\t;[is_valid];\t;[effective_range];\t;[op_time];&amp;nbsp;0;\n</p:valueExpression>
</p:expressions> </p:expressions>
<p:sequences> <p:sequences>
<p:operation>1</p:operation> <p:operation>1</p:operation>