gfwleak/k18-ntcs-web-argus-service
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1:为日志查询添加请求内容的条件

Browse Source 
2:将请求日志表分为配置请求记录表和日志请求记录表
3:更改redis数据同步到集群的策略
This commit is contained in:
renkaige
2018-10-19 17:17:04 +08:00
parent ddf64439ea
commit f6c056231c
8 changed files with 313 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/java/com/nis/domain/SwaggerLog.java
View File

@@ -17,7 +17,7 @@ public class SwaggerLog extends BaseEntity<SwaggerLog> {
private String serverIp;
private String traceCode;
private String requestUri;
private String tableName;
private String exception;// 界面的异常信息框
private Date beginDate; // 开始日期
@@ -151,4 +151,12 @@ public class SwaggerLog extends BaseEntity<SwaggerLog> {
this.requestUri = requestUri;
}
public String getTableName() {
return tableName;
}
public void setTableName(String tableName) {
this.tableName = tableName;
}
}

13
src/main/java/com/nis/web/controller/sys/SysLogController.java
View File

@@ -3,6 +3,7 @@ package com.nis.web.controller.sys;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.shiro.authz.annotation.RequiresPermissions;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
@@ -27,8 +28,16 @@ public class SysLogController extends BaseController {
@RequiresPermissions("sys:swaggerLogList:view")
@RequestMapping(value = "swaggerLogList")
public String swaggerLogList(SwaggerLog swaggerLog, HttpServletRequest request, HttpServletResponse response, Model model) {
Page<SwaggerLog> page = swaggerLogService.findSwaggerLogPage(new Page<SwaggerLog>(request, response), swaggerLog);
public String swaggerLogList(SwaggerLog swaggerLog, HttpServletRequest request, HttpServletResponse response,
Model model) {
if (swaggerLog.getOpAction() != null && swaggerLog.getOpAction() == 4) {
swaggerLog.setTableName("services_log_request_log");
} else {
swaggerLog.setTableName("services_config_request_log");
}
swaggerLog.setRequestContent(StringEscapeUtils.unescapeHtml4(swaggerLog.getRequestContent()));
Page<SwaggerLog> page = swaggerLogService.findSwaggerLogPage(new Page<SwaggerLog>(request, response),
swaggerLog);
model.addAttribute("page", page);
return "/sys/swaggerLogList";
}

2
src/main/java/com/nis/web/dao/ServicesRequestLogDao.java
View File

@@ -16,6 +16,8 @@ import com.nis.domain.restful.ServicesRequestLogBean;
public interface ServicesRequestLogDao {
int insert(ServicesRequestLog record);
int insertConfigLog(ServicesRequestLog record);
int insertLogLog(ServicesRequestLog record);
public void deleteById(Long id);

181
src/main/java/com/nis/web/dao/ServicesRequestLogDao.xml
View File

@@ -176,4 +176,185 @@
</if>
</trim>
</insert>
<insert id="insertConfigLog" parameterType="com.nis.domain.ServicesRequestLog">
insert into SERVICES_CONFIG_REQUEST_LOG
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="operator != null">
OPERATOR,
</if>
<if test="version != null">
VERSION,
</if>
<if test="opAction != null">
OPACTION,
</if>
<if test="opTime != null">
OPTIME,
</if>
<if test="requestContent != null">
REQUEST_CONTENT,
</if>
<if test="requestTime != null">
REQUEST_TIME,
</if>
<if test="consumerTime != null">
CONSUMER_TIME,
</if>
<if test="requestIp != null">
REQUEST_IP,
</if>
<if test="businessCode != null">
BUSINESS_CODE,
</if>
<if test="exceptionInfo != null">
EXCEPTION_INFO,
</if>
<if test="serverIp != null">
SERVER_IP,
</if>
<if test="traceCode != null">
TRACE_CODE,
</if>
<if test="requestURI != null">
REQUEST_URI,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="operator != null">
#{operator,jdbcType=VARCHAR},
</if>
<if test="version != null">
#{version,jdbcType=VARCHAR},
</if>
<if test="opAction != null">
#{opAction,jdbcType=INTEGER},
</if>
<if test="opTime != null">
#{opTime,jdbcType=TIMESTAMP},
</if>
<if test="requestContent != null">
#{requestContent,jdbcType=CLOB},
</if>
<if test="requestTime != null">
#{requestTime,jdbcType=TIMESTAMP},
</if>
<if test="consumerTime != null">
#{consumerTime,jdbcType=BIGINT},
</if>
<if test="requestIp != null">
#{requestIp,jdbcType=VARCHAR},
</if>
<if test="businessCode != null">
#{businessCode,jdbcType=INTEGER},
</if>
<if test="exceptionInfo != null">
#{exceptionInfo,jdbcType=VARCHAR},
</if>
<if test="serverIp != null">
#{serverIp,jdbcType=VARCHAR},
</if>
<if test="traceCode != null">
#{traceCode,jdbcType=VARCHAR},
</if>
<if test="requestURI != null">
#{requestURI,jdbcType=VARCHAR},
</if>
</trim>
</insert>
<insert id="insertLogLog" parameterType="com.nis.domain.ServicesRequestLog">
insert into SERVICES_LOG_REQUEST_LOG
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="operator != null">
OPERATOR,
</if>
<if test="version != null">
VERSION,
</if>
<if test="opAction != null">
OPACTION,
</if>
<if test="opTime != null">
OPTIME,
</if>
<if test="requestContent != null">
REQUEST_CONTENT,
</if>
<if test="requestTime != null">
REQUEST_TIME,
</if>
<if test="consumerTime != null">
CONSUMER_TIME,
</if>
<if test="requestIp != null">
REQUEST_IP,
</if>
<if test="businessCode != null">
BUSINESS_CODE,
</if>
<if test="exceptionInfo != null">
EXCEPTION_INFO,
</if>
<if test="serverIp != null">
SERVER_IP,
</if>
<if test="traceCode != null">
TRACE_CODE,
</if>
<if test="requestURI != null">
REQUEST_URI,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="operator != null">
#{operator,jdbcType=VARCHAR},
</if>
<if test="version != null">
#{version,jdbcType=VARCHAR},
</if>
<if test="opAction != null">
#{opAction,jdbcType=INTEGER},
</if>
<if test="opTime != null">
#{opTime,jdbcType=TIMESTAMP},
</if>
<if test="requestContent != null">
#{requestContent,jdbcType=CLOB},
</if>
<if test="requestTime != null">
#{requestTime,jdbcType=TIMESTAMP},
</if>
<if test="consumerTime != null">
#{consumerTime,jdbcType=BIGINT},
</if>
<if test="requestIp != null">
#{requestIp,jdbcType=VARCHAR},
</if>
<if test="businessCode != null">
#{businessCode,jdbcType=INTEGER},
</if>
<if test="exceptionInfo != null">
#{exceptionInfo,jdbcType=VARCHAR},
</if>
<if test="serverIp != null">
#{serverIp,jdbcType=VARCHAR},
</if>
<if test="traceCode != null">
#{traceCode,jdbcType=VARCHAR},
</if>
<if test="requestURI != null">
#{requestURI,jdbcType=VARCHAR},
</if>
</trim>
</insert>
</mapper>

2
src/main/java/com/nis/web/dao/SwaggerLogDao.xml
View File

@@ -5,7 +5,7 @@
<select id="findList" resultType="swaggerLog">
SELECT
a.*
FROM services_request_log a
FROM ${tableName} a
WHERE a.request_time BETWEEN
#{beginDate} AND #{endDate}

77
src/main/java/com/nis/web/service/ServicesRequestLogService.java
View File

@@ -51,8 +51,13 @@ public class ServicesRequestLogService {
@Autowired
public ServicesRequestLogDao servicesRequestLogDao;
private void saveLog(ServicesRequestLog log) {
servicesRequestLogDao.insert(log);
private void saveLog(ServicesRequestLog log, boolean isPzLog) {
if (isPzLog) {
servicesRequestLogDao.insertConfigLog(log);
} else {
servicesRequestLogDao.insertLogLog(log);
}
}
/**
@@ -66,8 +71,7 @@ public class ServicesRequestLogService {
* @param queryString
* request中的参数
* @param contextPath
* request中的参数
* 响应
* request中的参数 响应
* @param operator
* 操作人
* @param version
@@ -82,13 +86,13 @@ public class ServicesRequestLogService {
* 请求到达服务器时间
* @param consumerTime
* 耗时 void
* @throws UnknownHostException
* @throws UnknownHostException
* @exception @since
* 1.0.0
*/
public void saveRequestLog(String requestAddr, String requestURI, String queryString, String contextPath,
String operator, String version, int opAction, Date opTime, Object content, Date requestTime,
long consumerTime, int businessCode, String exceptionInfo,String traceCode) {
long consumerTime, int businessCode, String exceptionInfo, String traceCode) {
logger.info("开始记录日志---");
logger.info("请求IP---" + requestAddr);
logger.info("请求路径---" + requestURI);
@@ -107,36 +111,37 @@ public class ServicesRequestLogService {
log.setTraceCode(traceCode);
log.setRequestURI(requestURI);
try {
if(Constants.SERVCER_HOST!=null){
if (Constants.SERVCER_HOST != null) {
log.setServerIp(Constants.SERVCER_HOST);
}else if(isWindows()){
Constants.SERVCER_HOST=InetAddress.getLocalHost().getHostAddress();
} else if (isWindows()) {
Constants.SERVCER_HOST = InetAddress.getLocalHost().getHostAddress();
log.setServerIp(Constants.SERVCER_HOST);
}else{
InetAddress ip=null;
boolean bFindIP=false;
Enumeration<NetworkInterface> netInterfaces=(Enumeration<NetworkInterface>)
NetworkInterface.getNetworkInterfaces();
while(netInterfaces.hasMoreElements()){
if(bFindIP){
} else {
InetAddress ip = null;
boolean bFindIP = false;
Enumeration<NetworkInterface> netInterfaces = (Enumeration<NetworkInterface>) NetworkInterface
.getNetworkInterfaces();
while (netInterfaces.hasMoreElements()) {
if (bFindIP) {
break;
}
NetworkInterface ni=(NetworkInterface)netInterfaces.nextElement();
Enumeration<InetAddress> ips=ni.getInetAddresses();
while(ips.hasMoreElements()){
ip=(InetAddress)ips.nextElement();
if(ip.isSiteLocalAddress()&&!ip.isLoopbackAddress()&&ip.getHostAddress().indexOf(":")==-1){
bFindIP=true;
NetworkInterface ni = (NetworkInterface) netInterfaces.nextElement();
Enumeration<InetAddress> ips = ni.getInetAddresses();
while (ips.hasMoreElements()) {
ip = (InetAddress) ips.nextElement();
if (ip.isSiteLocalAddress() && !ip.isLoopbackAddress()
&& ip.getHostAddress().indexOf(":") == -1) {
bFindIP = true;
break;
}
}
}
if(null !=ip){
Constants.SERVCER_HOST=ip.getHostAddress();
if (null != ip) {
Constants.SERVCER_HOST = ip.getHostAddress();
log.setServerIp(ip.getHostAddress());
}
}
} catch (UnknownHostException e) {
logger.error("无法获取当前服务器ip");
e.printStackTrace();
@@ -173,19 +178,25 @@ public class ServicesRequestLogService {
}
}
this.saveLog(log);
if (log.getRequestURI() == null || log.getRequestURI().toLowerCase().contains("service/log")) {
this.saveLog(log, false);
} else {
this.saveLog(log, true);
}
logger.info("记录日志完成---");
}
public boolean isWindows(){
boolean isWindows=false;
if(System.getProperty("os.name").toLowerCase().indexOf("windows")>-1){
isWindows=true;
public boolean isWindows() {
boolean isWindows = false;
if (System.getProperty("os.name").toLowerCase().indexOf("windows") > -1) {
isWindows = true;
}
return isWindows;
}
/**
* 直接从数据源获取connection保存日志的方法，用saveLog()方法就行 save(这里用一句话描述这个方法的作用)
* (这里描述这个方法适用条件 – 可选)
* 直接从数据源获取connection保存日志的方法，用saveLog()方法就行 save(这里用一句话描述这个方法的作用) (这里描述这个方法适用条件
* – 可选)
*
* @param data
* @throws IllegalArgumentException
@@ -302,7 +313,7 @@ public class ServicesRequestLogService {
public void deleteById(String id) {
String[] split = id.split(",");
for (String str : split) {
if(!str.equals("")){
if (!str.equals("")) {
servicesRequestLogDao.deleteById(Long.parseLong(str));
}
}

85
src/main/java/com/nis/web/task/SyncRedisToCluster.java
View File

@@ -37,39 +37,47 @@ public class SyncRedisToCluster {
// private JedisSentinelPool jedisSentinelPool;
// @Scheduled(cron = "0/3 * * * * ?")
// @Scheduled(cron = "${syncRedisToClusterCron}")
// @Scheduled(cron = "${syncRedisToClusterCron}")
public void syncRedisToCluster() {
try {
// keys("EFFECTIVE_RULE*");
// keys("OBSOLETE_RULE*");
String clusterMaatVersionStr = jedisCluster.get("MAAT_VERSION");
String redisMaatVersionStr = JedisUtils.get("MAAT_VERSION", redisStatisticsRealDBIndex);
if (clusterMaatVersionStr != null && !clusterMaatVersionStr.trim().equals("")) {
if (redisMaatVersionStr != null && !redisMaatVersionStr.trim().equals("")) {
Integer clusterMaatVersion = Integer.valueOf(clusterMaatVersionStr);
Integer redisMaatVersion = Integer.valueOf(redisMaatVersionStr);
if (redisMaatVersion < clusterMaatVersion) {// 如果主从库比集群库的版本号小则下发全量
logger.info("redis集群中的MAAT_VERSION为大于配置库中的MAAT_VERSION,开始执行全量同步");
syncAllData(redisMaatVersionStr);
} else if (redisMaatVersion > clusterMaatVersion) {// 获取增量的数据
logger.info("redis集群中的MAAT_VERSION为小于配置库中的MAAT_VERSION,开始执行增量同步,score是{}-{}",
clusterMaatVersion, +redisMaatVersion);
syncData(clusterMaatVersion.doubleValue(), redisMaatVersion.doubleValue(), redisMaatVersionStr);
String rpopRedisList = rpopRedisList("SyncRedisToCluster");
if (rpopRedisList != null) {//避免集群环境下同一秒钟所有的机器都执行这个定时任务
// keys("EFFECTIVE_RULE*");
// keys("OBSOLETE_RULE*");
String clusterMaatVersionStr = jedisCluster.get("MAAT_VERSION");
String redisMaatVersionStr = JedisUtils.get("MAAT_VERSION", redisStatisticsRealDBIndex);
if (clusterMaatVersionStr != null && !clusterMaatVersionStr.trim().equals("")) {
if (redisMaatVersionStr != null && !redisMaatVersionStr.trim().equals("")) {
Integer clusterMaatVersion = Integer.valueOf(clusterMaatVersionStr);
Integer redisMaatVersion = Integer.valueOf(redisMaatVersionStr);
if (redisMaatVersion < clusterMaatVersion) {// 如果主从库比集群库的版本号小则下发全量
logger.info("redis集群中的MAAT_VERSION为大于配置库中的MAAT_VERSION,开始执行全量同步");
syncAllData(redisMaatVersionStr);
} else if (redisMaatVersion > clusterMaatVersion) {// 获取增量的数据
logger.info("redis集群中的MAAT_VERSION为小于配置库中的MAAT_VERSION,开始执行增量同步,score是{}-{}",
clusterMaatVersion, +redisMaatVersion);
syncData(clusterMaatVersion.doubleValue(), redisMaatVersion.doubleValue(),
redisMaatVersionStr);
} else {
logger.info("redis集群中的MAAT_VERSION与配置库中的MAAT_VERSION相等,暂不执行配置同步操作");
}
} else {
logger.info("redis集群中的MAAT_VERSION与配置库中的MAAT_VERSION相等,暂不执行配置同步操作");
logger.info("redis配置库中MAAT_VERSION为null,但是redis集群中的MAAT_VERSION为{},集群与配置库的数据不同步,开始删除集群中的配置",
clusterMaatVersionStr);
delClusterData();
}
} else {
logger.info("redis配置库中MAAT_VERSION为null,但是redis集群中的MAAT_VERSION为{},集群与配置库的数据不同步,开始删除集群中的配置",
clusterMaatVersionStr);
delClusterData();
if (redisMaatVersionStr != null && !redisMaatVersionStr.trim().equals("")) {
logger.info("redis配置库中的MAAT_VERSION为{},redis集群中的MAAT_VERSION为null,开始执行全量同步",
redisMaatVersionStr);
syncAllData(redisMaatVersionStr);
} else {
logger.info("redis配置库中和redis集群中的MAAT_VERSION都为null,暂时不执行全量同步");
}
}
lpushRedisList("SyncRedisToCluster", "1");
} else {
if (redisMaatVersionStr != null && !redisMaatVersionStr.trim().equals("")) {
logger.info("redis配置库中的MAAT_VERSION为{},redis集群中的MAAT_VERSION为null,开始执行全量同步", redisMaatVersionStr);
syncAllData(redisMaatVersionStr);
} else {
logger.info("redis配置库中和redis集群中的MAAT_VERSION都为null,暂时不执行全量同步");
}
logger.info("没有从rediscluster中获取到SyncRedisToCluster的值,暂时不执行数据同步!");
}
} catch (Exception e) {
logger.error("同步配置库配置到3A-redisCluster失败,失败原因:{}", ExceptionUtil.getExceptionMsg(e));
@@ -190,4 +198,29 @@ public class SyncRedisToCluster {
logger.info("向redis集群同步数据成功");
}
/**
* 存储Redis队列，顺序存储
*
* @param key
* redis键名
* @param value
* 键值
*/
public void lpushRedisList(String key, String value) {
// LPUSH SyncRedisToCluster "1"
jedisCluster.lpush(key, value);
logger.info("向redis集群的{}中lpush了一个值{}", key, value);
}
/**
* 移除并获取列表最后一个元素
*
* @param key
* @return
*/
public String rpopRedisList(String key) {
// RPOP SyncRedisToCluster
return jedisCluster.rpop(key);
}
}