gfwleak/k18-ntcs-web-argus-service
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1:删除无用的数据库连接

Browse Source 
2:修改pom中使用的hive版本为2.1.1
3:添加IP地址日志查询服务
This commit is contained in:
RenKaiGe-Office
2018-07-02 16:10:48 +08:00
parent 4f4ad8464a
commit 0530479502
22 changed files with 731 additions and 901 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pom.xml
View File

@@ -570,10 +570,11 @@
<version>2.4</version> <version>2.4</version>
<classifier>jdk15</classifier> <classifier>jdk15</classifier>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hive</groupId> <groupId>org.apache.hive</groupId>
<artifactId>hive-jdbc</artifactId> <artifactId>hive-jdbc</artifactId>
<version>2.1.0</version> <version>2.1.1</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<groupId>org.apache.logging.log4j</groupId> <groupId>org.apache.logging.log4j</groupId>

5
src/main/java/com/nis/datasource/CustomerContextHolder.java
View File

@@ -4,11 +4,6 @@ public class CustomerContextHolder {
public static final String DATA_SOURCE_A = "dataSourceA"; public static final String DATA_SOURCE_A = "dataSourceA";
public static final String DATA_SOURCE_B = "dataSourceB"; public static final String DATA_SOURCE_B = "dataSourceB";
public static final String DATA_SOURCE_C = "dataSourceC"; public static final String DATA_SOURCE_C = "dataSourceC";
public static final String DATA_SOURCE_D = "dataSourceD";
public static final String DATA_SOURCE_E = "dataSourceE";
public static final String DATA_SOURCE_F = "dataSourceF";
public static final String DATA_SOURCE_G = "dataSourceG";
public static final String DATA_SOURCE_H = "dataSourceH";
//线程本地环境 //线程本地环境
private static final ThreadLocal<String> contextHolder = new ThreadLocal<String>(); private static final ThreadLocal<String> contextHolder = new ThreadLocal<String>();

308
src/main/java/com/nis/domain/LogEntity.java
View File

@@ -9,8 +9,6 @@
package com.nis.domain; package com.nis.domain;
import java.io.Serializable; import java.io.Serializable;
import java.math.BigDecimal;
import java.text.SimpleDateFormat;
import java.util.Date; import java.util.Date;
import java.util.Map; import java.util.Map;
@@ -77,57 +75,16 @@ public abstract class LogEntity<T> implements Serializable {
@ApiModelProperty(value = "用户自定义域", required = true) @ApiModelProperty(value = "用户自定义域", required = true)
private String userRegion; private String userRegion;
protected Long foundTimeCluster;
protected Long recvTimeCluster;
protected String searchCfgId; protected String searchCfgId;
protected String searchTransProto;
protected String searchDIp;
protected String searchSIp;
protected String searchDirection;
protected String searchService;
protected String searchEntranceId;
protected String searchCapIp;
protected String searchFoundStartTime; protected String searchFoundStartTime;
protected String searchFoundEndTime; protected String searchFoundEndTime;
protected Long searchFoundStartTimeCluster;
protected Long searchFoundEndTimeCluster;
protected String searchProtocol;
protected String searchServerIp;
protected String searchClientIp;
protected Integer searchDirection;
protected String searchServiceType;
protected String searchEntranceId;
protected String searchCljIp;
@JsonIgnore
public Long getFoundTimeCluster() {
return foundTimeCluster;
}
@JsonIgnore
public Long getRecvTimeCluster() {
return recvTimeCluster;
}
public void setFoundTimeCluster(Long foundTimeCluster) {
this.foundTimeCluster = foundTimeCluster;
}
public void setRecvTimeCluster(Long recvTimeCluster) {
this.recvTimeCluster = recvTimeCluster;
}
@JsonIgnore
public Long getSearchFoundStartTimeCluster() {
return searchFoundStartTimeCluster;
}
public void setSearchFoundStartTimeCluster(Long searchFoundStartTimeCluster) {
this.searchFoundStartTimeCluster = searchFoundStartTimeCluster;
}
@JsonIgnore
public Long getSearchFoundEndTimeCluster() {
return searchFoundEndTimeCluster;
}
public void setSearchFoundEndTimeCluster(Long searchFoundEndTimeCluster) {
this.searchFoundEndTimeCluster = searchFoundEndTimeCluster;
}
/** /**
* 当前实体分页对象 * 当前实体分页对象
@@ -151,6 +108,7 @@ public abstract class LogEntity<T> implements Serializable {
/** /**
* @return id * @return id
*/ */
@JsonIgnore
public Long getId() { public Long getId() {
return id; return id;
} }
@@ -183,9 +141,6 @@ public abstract class LogEntity<T> implements Serializable {
*/ */
@JsonSerialize(using = JsonDateSerializer.class) @JsonSerialize(using = JsonDateSerializer.class)
public Date getFoundTime() { public Date getFoundTime() {
if (foundTime == null && this.foundTimeCluster != null) {
foundTime = new Date(this.foundTimeCluster * 1000);
}
return foundTime; return foundTime;
} }
@@ -202,9 +157,6 @@ public abstract class LogEntity<T> implements Serializable {
*/ */
@JsonSerialize(using = JsonDateSerializer.class) @JsonSerialize(using = JsonDateSerializer.class)
public Date getRecvTime() { public Date getRecvTime() {
if (recvTime == null && this.recvTimeCluster != null) {
recvTime = new Date(this.recvTimeCluster * 1000);
}
return recvTime; return recvTime;
} }
@@ -353,151 +305,6 @@ public abstract class LogEntity<T> implements Serializable {
return null == this.getId() ? false : this.getId().equals(that.getId()); return null == this.getId() ? false : this.getId().equals(that.getId());
} }
/**
* @return searchFoundStartTime
*/
@JsonIgnore
public String getSearchFoundStartTime() {
return searchFoundStartTime;
}
/**
* @param searchFoundStartTime
* 要设置的 searchFoundStartTime
*/
public void setSearchFoundStartTime(String searchFoundStartTime) {
this.searchFoundStartTime = searchFoundStartTime;
}
/**
* @return searchFoundEndTime
*/
@JsonIgnore
public String getSearchFoundEndTime() {
return searchFoundEndTime;
}
/**
* @param searchFoundEndTime
* 要设置的 searchFoundEndTime
*/
public void setSearchFoundEndTime(String searchFoundEndTime) {
this.searchFoundEndTime = searchFoundEndTime;
}
/**
* @return searchCfgId
*/
@JsonIgnore
public String getSearchCfgId() {
return searchCfgId;
}
/**
* @param searchCfgId
* 要设置的 searchCfgId
*/
public void setSearchCfgId(String searchCfgId) {
this.searchCfgId = searchCfgId;
}
/**
* @return searchProtocol
*/
@JsonIgnore
public String getSearchProtocol() {
return searchProtocol;
}
/**
* @param searchProtocol
* 要设置的 searchProtocol
*/
public void setSearchProtocol(String searchProtocol) {
this.searchProtocol = searchProtocol;
}
/**
* @return searchServerIp
*/
@JsonIgnore
public String getSearchServerIp() {
return searchServerIp;
}
/**
* @param searchServerIp
* 要设置的 searchServerIp
*/
public void setSearchServerIp(String searchServerIp) {
this.searchServerIp = searchServerIp;
}
/**
* @return searchClientIp
*/
@JsonIgnore
public String getSearchClientIp() {
return searchClientIp;
}
/**
* @param searchClientIp
* 要设置的 searchClientIp
*/
public void setSearchClientIp(String searchClientIp) {
this.searchClientIp = searchClientIp;
}
/**
* @return searchEntranceId
*/
@JsonIgnore
public String getSearchEntranceId() {
return searchEntranceId;
}
/**
* @param searchEntranceId
* 要设置的 searchEntranceId
*/
public void setSearchEntranceId(String searchEntranceId) {
this.searchEntranceId = searchEntranceId;
}
/**
* @return searchCljIp
*/
@JsonIgnore
public String getSearchCljIp() {
return searchCljIp;
}
/**
* @param searchCljIp
* 要设置的 searchCljIp
*/
public void setSearchCljIp(String searchCljIp) {
this.searchCljIp = searchCljIp;
}
/**
* @return searchServiceType
*/
@JsonIgnore
public String getSearchServiceType() {
return searchServiceType;
}
/**
* @param searchServiceType
* 要设置的 searchServiceType
*/
public void setSearchServiceType(String searchServiceType) {
this.searchServiceType = searchServiceType;
}
public Integer getAddrType() { public Integer getAddrType() {
return addrType; return addrType;
} }
@@ -522,15 +329,6 @@ public abstract class LogEntity<T> implements Serializable {
this.direction = direction; this.direction = direction;
} }
@JsonIgnore
public Integer getSearchDirection() {
return searchDirection;
}
public void setSearchDirection(Integer searchDirection) {
this.searchDirection = searchDirection;
}
public String getUserRegion() { public String getUserRegion() {
return userRegion; return userRegion;
} }
@@ -544,4 +342,94 @@ public abstract class LogEntity<T> implements Serializable {
return ReflectionToStringBuilder.toString(this); return ReflectionToStringBuilder.toString(this);
} }
@JsonIgnore
public String getSearchCfgId() {
return searchCfgId;
}
public void setSearchCfgId(String searchCfgId) {
this.searchCfgId = searchCfgId;
}
@JsonIgnore
public String getSearchTransProto() {
return searchTransProto;
}
public void setSearchTransProto(String searchTransProto) {
this.searchTransProto = searchTransProto;
}
@JsonIgnore
public String getSearchDIp() {
return searchDIp;
}
public void setSearchDIp(String searchDIp) {
this.searchDIp = searchDIp;
}
@JsonIgnore
public String getSearchSIp() {
return searchSIp;
}
public void setSearchSIp(String searchSIp) {
this.searchSIp = searchSIp;
}
@JsonIgnore
public String getSearchDirection() {
return searchDirection;
}
public void setSearchDirection(String searchDirection) {
this.searchDirection = searchDirection;
}
@JsonIgnore
public String getSearchService() {
return searchService;
}
public void setSearchService(String searchService) {
this.searchService = searchService;
}
@JsonIgnore
public String getSearchEntranceId() {
return searchEntranceId;
}
public void setSearchEntranceId(String searchEntranceId) {
this.searchEntranceId = searchEntranceId;
}
@JsonIgnore
public String getSearchCapIp() {
return searchCapIp;
}
public void setSearchCapIp(String searchCapIp) {
this.searchCapIp = searchCapIp;
}
@JsonIgnore
public String getSearchFoundStartTime() {
return searchFoundStartTime;
}
public void setSearchFoundStartTime(String searchFoundStartTime) {
this.searchFoundStartTime = searchFoundStartTime;
}
@JsonIgnore
public String getSearchFoundEndTime() {
return searchFoundEndTime;
}
public void setSearchFoundEndTime(String searchFoundEndTime) {
this.searchFoundEndTime = searchFoundEndTime;
}
} }

23
src/main/java/com/nis/interceptor/DataSourceDInterceptor.java → src/main/java/com/nis/interceptor/DataSourceAInterceptor.java
View File

@@ -9,15 +9,15 @@ import org.springframework.web.servlet.ModelAndView;
import com.nis.datasource.CustomerContextHolder; import com.nis.datasource.CustomerContextHolder;
public class DataSourceDInterceptor implements HandlerInterceptor { public class DataSourceAInterceptor implements HandlerInterceptor {
Logger logger = Logger.getLogger(DataSourceDInterceptor.class); Logger logger = Logger.getLogger(DataSourceAInterceptor.class);
@Override @Override
public boolean preHandle(HttpServletRequest request, public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler)
HttpServletResponse response, Object handler) throws Exception { throws Exception {
logger.info("开启测试配置库---"); logger.info("开启数据源日志操作库---" + System.currentTimeMillis());
CustomerContextHolder.setCustomerType(CustomerContextHolder.DATA_SOURCE_D);//开启数据源C CustomerContextHolder.setCustomerType(CustomerContextHolder.DATA_SOURCE_A);// 开启数据源A
logger.info("日志数据源开启成功---" + System.currentTimeMillis());
return true; return true;
} }
@@ -28,16 +28,11 @@ public class DataSourceDInterceptor implements HandlerInterceptor {
} }
@Override @Override
public void afterCompletion(HttpServletRequest request, public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex)
HttpServletResponse response, Object handler, Exception ex)
throws Exception { throws Exception {
CustomerContextHolder.clearCustomerType(); CustomerContextHolder.clearCustomerType();
logger.info("释放测试配置库---"); logger.info("释放数据源日志操作库---");
} }
} }

2
src/main/java/com/nis/interceptor/DataSourceBInterceptor.java
View File

@@ -16,7 +16,7 @@ public class DataSourceBInterceptor implements HandlerInterceptor {
HttpServletResponse response, Object handler) throws Exception { HttpServletResponse response, Object handler) throws Exception {
logger.info("开启数据源配置操作库---"); logger.info("开启数据源配置操作库---");
CustomerContextHolder.setCustomerType(CustomerContextHolder.DATA_SOURCE_B);//开启数据源B CustomerContextHolder.setCustomerType(CustomerContextHolder.DATA_SOURCE_B);//开启数据源B
logger.info("mysql日志数据源开启成功---"+System.currentTimeMillis());
return true; return true;
} }

60
src/main/java/com/nis/interceptor/DataSourceCInterceptor.java
View File

@@ -1,60 +0,0 @@
package com.nis.interceptor;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.log4j.Logger;
import org.springframework.web.servlet.HandlerInterceptor;
import org.springframework.web.servlet.ModelAndView;
import com.nis.datasource.CustomerContextHolder;
import com.nis.restful.RestBusinessCode;
import com.nis.restful.RestServiceException;
import com.nis.util.Constants;
public class DataSourceCInterceptor implements HandlerInterceptor {
Logger logger = Logger.getLogger(DataSourceCInterceptor.class);
String searchActiveSys="";
@Override
public boolean preHandle(HttpServletRequest request,
HttpServletResponse response, Object handler) throws Exception {
// searchActiveSys=request.getParameter("searchActiveSys");
// if(searchActiveSys == null
// || !(Constants.ACTIVESYS_A.equals(searchActiveSys)
// || Constants.ACTIVESYS_C.equals(searchActiveSys))
// ) searchActiveSys=Constants.ACTIVESYS_B;
// if(Constants.ACTIVESYS_A.equals(searchActiveSys)){
// logger.info("开启数据源日志A操作库---"+System.currentTimeMillis());
// CustomerContextHolder.setCustomerType(CustomerContextHolder.DATA_SOURCE_F);//开启数据源F
// }else if(Constants.ACTIVESYS_C.equals(searchActiveSys)){
// logger.info("开启数据源日志C操作库---"+System.currentTimeMillis());
// CustomerContextHolder.setCustomerType(CustomerContextHolder.DATA_SOURCE_G);//开启数据源F
// }else{
logger.info("开启数据源日志操作库---"+System.currentTimeMillis());
CustomerContextHolder.setCustomerType(CustomerContextHolder.DATA_SOURCE_C);//开启数据源C
// }
logger.info("日志数据源开启成功---"+System.currentTimeMillis());
return true;
}
@Override
public void postHandle(HttpServletRequest arg0, HttpServletResponse arg1, Object arg2, ModelAndView arg3)
throws Exception {
logger.info("postHandle---");
}
@Override
public void afterCompletion(HttpServletRequest request,
HttpServletResponse response, Object handler, Exception ex)
throws Exception {
CustomerContextHolder.clearCustomerType();
logger.info("释放数据源日志操作库---");
}
}

38
src/main/java/com/nis/interceptor/DataSourceEInterceptor.java
View File

@@ -1,38 +0,0 @@
package com.nis.interceptor;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.log4j.Logger;
import org.springframework.web.servlet.HandlerInterceptor;
import org.springframework.web.servlet.ModelAndView;
import com.nis.datasource.CustomerContextHolder;
public class DataSourceEInterceptor implements HandlerInterceptor {
Logger logger = Logger.getLogger(DataSourceEInterceptor.class);
@Override
public void afterCompletion(HttpServletRequest arg0, HttpServletResponse arg1, Object arg2, Exception arg3)
throws Exception {
CustomerContextHolder.clearCustomerType();
logger.info("释放数据静控操作库---");
}
@Override
public void postHandle(HttpServletRequest arg0, HttpServletResponse arg1, Object arg2, ModelAndView arg3)
throws Exception {
logger.info("postHandle---");
}
@Override
public boolean preHandle(HttpServletRequest arg0, HttpServletResponse arg1, Object arg2) throws Exception {
logger.info("开启数据源静控操作库---");
CustomerContextHolder.setCustomerType(CustomerContextHolder.DATA_SOURCE_E);//开启数据源C
return true;
}
}

10
src/main/java/com/nis/persistence/interceptor/SQLHelper.java
View File

@@ -125,17 +125,9 @@ public class SQLHelper {
} else if (dataSource.equals(CustomerContextHolder.DATA_SOURCE_A)) { } else if (dataSource.equals(CustomerContextHolder.DATA_SOURCE_A)) {
return "mysql"; return "mysql";
} else if (dataSource.equals(CustomerContextHolder.DATA_SOURCE_B)) { } else if (dataSource.equals(CustomerContextHolder.DATA_SOURCE_B)) {
return "oracle"; return "mysql";
} else if (dataSource.equals(CustomerContextHolder.DATA_SOURCE_C)) { } else if (dataSource.equals(CustomerContextHolder.DATA_SOURCE_C)) {
return "mysql"; return "mysql";
} else if (dataSource.equals(CustomerContextHolder.DATA_SOURCE_D)) {
return "oracle";
} else if (dataSource.equals(CustomerContextHolder.DATA_SOURCE_F)) {
return "oracle";
} else if (dataSource.equals(CustomerContextHolder.DATA_SOURCE_G)) {
return "oracle";
} else if (dataSource.equals(CustomerContextHolder.DATA_SOURCE_H)) {//神通数据库
return "cluster";
}else { }else {
return "mysql"; return "mysql";
} }

8
src/main/java/com/nis/util/Constants.java
View File

@@ -181,13 +181,9 @@ public final class Constants {
public static final String SEARCH_ES_HOSTANDPORT_C = Configurations.getStringProperty("search.eshostandport_C", public static final String SEARCH_ES_HOSTANDPORT_C = Configurations.getStringProperty("search.eshostandport_C",
null); null);
/** /**
* 数据中心A版数据库名称,程序中每次查询时使用的数据库名称 use HIVEADBNAME * 数据中心数据库名称,程序中每次查询时使用的数据库名称 use HIVEDBNAME
*/ */
public static final String HIVEADBNAME = Configurations.getStringProperty("jdbc.hive.AName", "xa_dfbhit_hive"); public static final String HIVEDBNAME = Configurations.getStringProperty("jdbc.hive.Name", "xa_dfbhit_hive");
/**
* 数据中心B版数据库名称,程序中每次查询时使用的数据库名称 use HIVEBDBNAME
*/
public static final String HIVEBDBNAME = Configurations.getStringProperty("jdbc.hive.BName", "xa_z2_mesalog_hive");
public static final String DIGEST_GEN_TOOL_PATH = Configurations.getStringProperty("digest.gen.tool.path", "maat-redis/digest_gen"); public static final String DIGEST_GEN_TOOL_PATH = Configurations.getStringProperty("digest.gen.tool.path", "maat-redis/digest_gen");

85
src/main/java/com/nis/util/HiveDataSource.java
View File

@@ -3,55 +3,56 @@ package com.nis.util;
import java.sql.Connection; import java.sql.Connection;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.Statement; import java.sql.Statement;
import org.apache.log4j.Logger;
import org.springframework.context.ApplicationContext; import org.slf4j.Logger;
import org.springframework.context.support.ClassPathXmlApplicationContext; import org.slf4j.LoggerFactory;
import com.jolbox.bonecp.BoneCPDataSource; import com.jolbox.bonecp.BoneCPDataSource;
import com.nis.web.service.SpringContextHolder;
public class HiveDataSource { public class HiveDataSource {
private final static Logger logger = Logger.getLogger(HiveDataSource.class); private final static Logger logger = LoggerFactory.getLogger(HiveDataSource.class);
static BoneCPDataSource datasource = null;
static Connection conn = null; static Connection conn = null;
static ResultSet rsA = null; static ResultSet rs = null;
static Statement stA = null; static Statement st = null;
static ResultSet rsB = null; public static ResultSet query(String sql) throws Exception {
static Statement stB = null; if (datasource == null) {
datasource = (BoneCPDataSource) SpringContextHolder.getBean("HiveDataSource");
public static ResultSet query(String sql, String searchActiveSys) throws Exception { }
if (null != searchActiveSys && searchActiveSys.equals("4")) {// A版数据库
logger.info("开始连接数据中心A版日志库--------------------------");
if (conn == null || conn.isClosed()) {
ApplicationContext ct = new ClassPathXmlApplicationContext("applicationContext-mybatis.xml");
BoneCPDataSource datasource = (BoneCPDataSource) ct.getBean("HiveADataSource");
conn = datasource.getConnection(); conn = datasource.getConnection();
} logger.info("连接数据中心日志库成功--------------------------");
logger.info("连接数据中心A版日志库成功--------------------------"); st = conn.createStatement();
stA = conn.createStatement(); // logger.info("开始选择{}数据库--------------------------", Constants.HIVEDBNAME);
String hiveAName = "use " + Constants.HIVEADBNAME; // String hiveAName = "use " + Constants.HIVEDBNAME;
// stA.execute("use xa_dfbhit_p_hive"); // st.execute(hiveAName);
stA.execute(hiveAName); // logger.info("选择数据库{}成功,开始执行查询", Constants.HIVEDBNAME);
logger.info("开始执行查询数据中心A版日志库操作--------------------------" + sql); // logger.info("选择数据库{}成功,开始执行查询", Constants.HIVEDBNAME);
rsA = stA.executeQuery(sql); rs = st.executeQuery(sql);
logger.info("执行查询数据中心A版日志库成功--------------------------"); logger.info("执行查询语句成功sql={}",sql);
return rsA; return rs;
} else {// 目前默认B版数据库,后期增加C版数据库
logger.info("开始连接数据中心B版日志库--------------------------");
if (conn == null || conn.isClosed()) {
ApplicationContext ct = new ClassPathXmlApplicationContext("applicationContext-mybatis.xml");
BoneCPDataSource datasource = (BoneCPDataSource) ct.getBean("HiveBDataSource");
conn = datasource.getConnection();
}
logger.info("连接数据中心B版日志库成功--------------------------");
stB = conn.createStatement();
String hiveBName = "use " + Constants.HIVEBDBNAME;
// stB.execute("use xa_z2_mesalog_hive");
stB.execute(hiveBName);
logger.info("开始执行查询数据中心B版日志库操作--------------------------" + sql);
rsB = stB.executeQuery(sql);
logger.info("执行查询数据中心B版日志库成功--------------------------");
return rsB;
}
} }
public static void closeConn() {
try {
if (rs != null) {
rs.close();
rs = null;
}
if (st != null) {
st.close();
st = null;
}
if (conn != null) {
conn.close();
conn = null;
}
logger.info("关闭数据中心连接成功");
} catch (Exception e) {
e.printStackTrace();
logger.error("关闭数据中心连接失败,失败原因" + e);
}
}
} }

2
src/main/java/com/nis/util/HiveJDBC.java
View File

@@ -169,7 +169,7 @@ public class HiveJDBC {
listObject.add(map2Obj(map, entityClass)); listObject.add(map2Obj(map, entityClass));
} }
logger.info("开始关闭数据中心连接"); logger.info("开始关闭数据中心连接");
HiveJDBC.closeConn(); HiveDataSource.closeConn();
if (null == listString || listString.size() == 0 || null == listObject || listObject.size() == 0) { if (null == listString || listString.size() == 0 || null == listObject || listObject.size() == 0) {
return null; return null;
} else { } else {

260
src/main/java/com/nis/web/controller/restful/LogController.java
View File

@@ -1,5 +1,10 @@
package com.nis.web.controller.restful; package com.nis.web.controller.restful;
import java.sql.ResultSet;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
@@ -11,6 +16,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import com.nis.domain.LogEntity;
import com.nis.domain.Page; import com.nis.domain.Page;
import com.nis.domain.restful.NtcDnsLog; import com.nis.domain.restful.NtcDnsLog;
import com.nis.domain.restful.NtcFtpLog; import com.nis.domain.restful.NtcFtpLog;
@@ -24,8 +30,15 @@ import com.nis.domain.restful.NtcPptpLog;
import com.nis.domain.restful.NtcPzReport; import com.nis.domain.restful.NtcPzReport;
import com.nis.domain.restful.NtcSshLog; import com.nis.domain.restful.NtcSshLog;
import com.nis.domain.restful.NtcSslLog; import com.nis.domain.restful.NtcSslLog;
import com.nis.restful.RestServiceException;
import com.nis.util.Configurations;
import com.nis.util.Constants; import com.nis.util.Constants;
import com.nis.util.DateUtils;
import com.nis.util.HiveJDBC;
import com.nis.util.JsonMapper;
import com.nis.web.controller.BaseRestController; import com.nis.web.controller.BaseRestController;
import com.nis.web.service.BaseLogService;
import com.nis.web.service.HiveSqlService;
import com.nis.web.service.SaveRequestLogThread; import com.nis.web.service.SaveRequestLogThread;
import com.nis.web.service.ServicesRequestLogService; import com.nis.web.service.ServicesRequestLogService;
import com.nis.web.service.restful.LogTestService; import com.nis.web.service.restful.LogTestService;
@@ -33,7 +46,13 @@ import com.wordnik.swagger.annotations.Api;
import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiOperation;
/** /**
* Created by darnell on 2018/6/10. *
* <p>Title: LogController</p>
* <p>Description: 日志查询controller</p>
* <p>Company: IIE</p>
* @author rkg
* @date 2018年7月2日
*
*/ */
@RestController @RestController
@RequestMapping("${servicePath}/log/v1") @RequestMapping("${servicePath}/log/v1")
@@ -41,155 +60,196 @@ import com.wordnik.swagger.annotations.ApiOperation;
public class LogController extends BaseRestController { public class LogController extends BaseRestController {
@Autowired @Autowired
public LogTestService testService; public LogTestService testService;
@Autowired @Autowired
protected ServicesRequestLogService servicesRequestLogService; protected ServicesRequestLogService servicesRequestLogService;
@RequestMapping(value = "/ntcIpLogs", method = RequestMethod.GET) @RequestMapping(value = "/ntcIpLogs", method = RequestMethod.GET)
@ApiOperation(value = "IP地址日志查询", httpMethod = "GET", notes = "对应配置为IP地址管理，存储动作为阻断与监测的命中日志。对日志功能IP地址提供数据基础查询服务") @ApiOperation(value = "IP地址日志查询", httpMethod = "GET", notes = "对应配置为IP地址管理，存储动作为阻断与监测的命中日志。对日志功能IP地址提供数据基础查询服务")
public Map<String, ?> ntcIpLogs(Page page, NtcIpLog ntcIpLog, Model model, HttpServletRequest request, HttpServletResponse response) { public Map<String, ?> ntcIpLogs(Page page, NtcIpLog ntcIpLog, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET,
request, null);
Page<NtcIpLog> ntcIpLogPage = new Page<>(); Page<NtcIpLog> ntcIpLogPage = null;
try { try {
ntcIpLogPage = testService.findNtcIpLogPage( testService.queryConditionCheck(auditLogThread, start, ntcIpLog, NtcIpLog.class, page);
new Page<NtcIpLog>(request, response, NtcIpLog.class), ntcIpLog); ntcIpLogPage = new Page<NtcIpLog>();
String orderBy = "";
if (null != page.getOrderBy() && !page.getOrderBy().equals("")) {
orderBy = page.getOrderBySql(NtcIpLog.class.getSimpleName(), page.getOrderBy());
} else {
orderBy = "found_Time";
}
ResultSet rs = HiveSqlService.getResultSet(page, ntcIpLog,
Configurations.getStringProperty(NtcIpLog.class.getSimpleName() + "HiveTable", "ntc_ip_log"),
getCol2Col(), orderBy, null);
Map<String, List> tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcIpLog.class, "foundTime",
"recvTime");
if (tableMapping == null) {
ntcIpLogPage.setList(new ArrayList());
} else {
List list = new ArrayList();
if (tableMapping.get("obj").size() > page.getPageSize()) {
list = tableMapping.get("obj").subList(0, page.getPageSize());
} else {
list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size());
}
if (list.size() > 0) {
String jsonString = JsonMapper.toJsonString(list);
List<NtcIpLog> List = (java.util.List<NtcIpLog>) JsonMapper.fromJsonList(jsonString,
NtcIpLog.class);
ntcIpLogPage.setList(List);
} else {
ntcIpLogPage.setList(new ArrayList());
}
}
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
auditLogThread.setExceptionInfo(e.getMessage() + " " + e.getCause());
logger.error(e);
if (!(e instanceof RestServiceException)) {
e = new RestServiceException(auditLogThread, System.currentTimeMillis() - start, "HTTP协议请求日志检索失败");
} }
throw ((RestServiceException) e);
}
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "IP地址日志检索成功",
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "IP地址日志检索成功", ntcIpLogPage, 0); ntcIpLogPage, 0);
} }
@RequestMapping(value = "/ntcHttpLogs", method = RequestMethod.GET) @RequestMapping(value = "/ntcHttpLogs", method = RequestMethod.GET)
@ApiOperation(value = "HTTP日志查询", httpMethod = "GET", notes = "对应配置为“网站管理-HTTP”，存储动作为阻断与监测的命中日志。对日志功能“网站管理-HTTP”提供数据基础查询服务") @ApiOperation(value = "HTTP日志查询", httpMethod = "GET", notes = "对应配置为“网站管理-HTTP”，存储动作为阻断与监测的命中日志。对日志功能“网站管理-HTTP”提供数据基础查询服务")
public Map<String, ?> ntcHttpLogs(Page page, NtcHttpLog ntcHttpLog, Model model, HttpServletRequest request, HttpServletResponse response) { public Map<String, ?> ntcHttpLogs(Page page, NtcHttpLog ntcHttpLog, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET,
request, null);
Page<NtcHttpLog> ntcHttpLogPage = new Page<>(); Page<NtcHttpLog> ntcHttpLogPage = new Page<>();
try { try {
ntcHttpLogPage = testService.findNtcHttpLogPage( ntcHttpLogPage = testService.findNtcHttpLogPage(new Page<NtcHttpLog>(request, response, NtcHttpLog.class),
new Page<NtcHttpLog>(request, response, NtcHttpLog.class), ntcHttpLog); ntcHttpLog);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "Http日志检索成功",
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "Http日志检索成功", ntcHttpLogPage, 0); ntcHttpLogPage, 0);
} }
@RequestMapping(value = "/ntcDnsLogs", method = RequestMethod.GET) @RequestMapping(value = "/ntcDnsLogs", method = RequestMethod.GET)
@ApiOperation(value = "DNS日志查询", httpMethod = "GET", notes = "对应配置为“网站管理-DNS”，存储动作为阻断与监测的命中日志。对日志功能“网站管理-DNS”提供数据基础查询服务。") @ApiOperation(value = "DNS日志查询", httpMethod = "GET", notes = "对应配置为“网站管理-DNS”，存储动作为阻断与监测的命中日志。对日志功能“网站管理-DNS”提供数据基础查询服务。")
public Map<String, ?> ntcDnsLogs(Page page, NtcDnsLog ntcDnsLog, Model model, HttpServletRequest request, HttpServletResponse response) { public Map<String, ?> ntcDnsLogs(Page page, NtcDnsLog ntcDnsLog, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET,
request, null);
Page<NtcDnsLog> ntcDnsLogPage = new Page<>(); Page<NtcDnsLog> ntcDnsLogPage = new Page<>();
try { try {
ntcDnsLogPage = testService.findNtcDnsLogPage( ntcDnsLogPage = testService.findNtcDnsLogPage(new Page<NtcDnsLog>(request, response, NtcDnsLog.class),
new Page<NtcDnsLog>(request, response, NtcDnsLog.class), ntcDnsLog); ntcDnsLog);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "Dns日志检索成功",
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "Dns日志检索成功", ntcDnsLogPage, 0); ntcDnsLogPage, 0);
} }
@RequestMapping(value = "/ntcMailLogs", method = RequestMethod.GET) @RequestMapping(value = "/ntcMailLogs", method = RequestMethod.GET)
@ApiOperation(value = "EMAIL日志查询", httpMethod = "GET", notes = "对应配置为“邮件管理”，存储动作为阻断与监测的命中日志。对日志功能“邮件管理”提供数据基础查询服务。") @ApiOperation(value = "EMAIL日志查询", httpMethod = "GET", notes = "对应配置为“邮件管理”，存储动作为阻断与监测的命中日志。对日志功能“邮件管理”提供数据基础查询服务。")
public Map<String, ?> ntcMailLogs(Page page, NtcMailLog ntcMailLog, Model model, HttpServletRequest request, HttpServletResponse response) { public Map<String, ?> ntcMailLogs(Page page, NtcMailLog ntcMailLog, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET,
request, null);
Page<NtcMailLog> ntcMailLogPage = new Page<>(); Page<NtcMailLog> ntcMailLogPage = new Page<>();
try { try {
ntcMailLogPage = testService.findNtcMailLogPage( ntcMailLogPage = testService.findNtcMailLogPage(new Page<NtcMailLog>(request, response, NtcMailLog.class),
new Page<NtcMailLog>(request, response, NtcMailLog.class), ntcMailLog); ntcMailLog);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "Mail日志检索成功",
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "Mail日志检索成功", ntcMailLogPage, 0); ntcMailLogPage, 0);
} }
@RequestMapping(value = "/ntcSslLogs", method = RequestMethod.GET) @RequestMapping(value = "/ntcSslLogs", method = RequestMethod.GET)
@ApiOperation(value = "SSL日志查询", httpMethod = "GET", notes = "对应配置为“网站管理-SSL”，存储动作为阻断与监测的命中日志。对日志功能“网站管理-SSL”提供数据基础查询服务。") @ApiOperation(value = "SSL日志查询", httpMethod = "GET", notes = "对应配置为“网站管理-SSL”，存储动作为阻断与监测的命中日志。对日志功能“网站管理-SSL”提供数据基础查询服务。")
public Map<String, ?> ntcSslLogs(Page page, NtcSslLog ntcSslLog, Model model, HttpServletRequest request, HttpServletResponse response) { public Map<String, ?> ntcSslLogs(Page page, NtcSslLog ntcSslLog, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET,
request, null);
Page<NtcSslLog> ntcSslLogPage = new Page<>(); Page<NtcSslLog> ntcSslLogPage = new Page<>();
try { try {
ntcSslLogPage = testService.findNtcSslLogPage( ntcSslLogPage = testService.findNtcSslLogPage(new Page<NtcSslLog>(request, response, NtcSslLog.class),
new Page<NtcSslLog>(request, response, NtcSslLog.class), ntcSslLog); ntcSslLog);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "SSL日志检索成功",
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "SSL日志检索成功", ntcSslLogPage, 0); ntcSslLogPage, 0);
} }
@RequestMapping(value = "/ntcPptpLogs", method = RequestMethod.GET) @RequestMapping(value = "/ntcPptpLogs", method = RequestMethod.GET)
@ApiOperation(value = "PPTP日志查询", httpMethod = "GET", notes = "对应配置为“隧道管理-PPTP”，存储动作为阻断与监测的命中日志。对日志功能“隧道管理-PPTP”提供数据基础查询服务。") @ApiOperation(value = "PPTP日志查询", httpMethod = "GET", notes = "对应配置为“隧道管理-PPTP”，存储动作为阻断与监测的命中日志。对日志功能“隧道管理-PPTP”提供数据基础查询服务。")
public Map<String, ?> ntcPptpLogs(Page page, NtcPptpLog ntcPptpLog, Model model, HttpServletRequest request, HttpServletResponse response) { public Map<String, ?> ntcPptpLogs(Page page, NtcPptpLog ntcPptpLog, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET,
request, null);
Page<NtcPptpLog> ntcPptpLogPage = new Page<>(); Page<NtcPptpLog> ntcPptpLogPage = new Page<>();
try { try {
ntcPptpLogPage = testService.findNtcPptpLogPage( ntcPptpLogPage = testService.findNtcPptpLogPage(new Page<NtcPptpLog>(request, response, NtcPptpLog.class),
new Page<NtcPptpLog>(request, response, NtcPptpLog.class), ntcPptpLog); ntcPptpLog);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "PPTP日志检索成功",
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "PPTP日志检索成功", ntcPptpLogPage, 0); ntcPptpLogPage, 0);
} }
@RequestMapping(value = "/ntcL2tpLogs", method = RequestMethod.GET) @RequestMapping(value = "/ntcL2tpLogs", method = RequestMethod.GET)
@ApiOperation(value = "L2TP日志查询", httpMethod = "GET", notes = "对应配置为“隧道管理-L2TP”，存储动作为阻断与监测的命中日志。对日志功能“隧道管理- L2TP”提供数据基础查询服务。") @ApiOperation(value = "L2TP日志查询", httpMethod = "GET", notes = "对应配置为“隧道管理-L2TP”，存储动作为阻断与监测的命中日志。对日志功能“隧道管理- L2TP”提供数据基础查询服务。")
public Map<String, ?> ntcL2tpLogs(Page page, NtcL2tpLog ntcL2tpLog, Model model, HttpServletRequest request, HttpServletResponse response) { public Map<String, ?> ntcL2tpLogs(Page page, NtcL2tpLog ntcL2tpLog, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET,
request, null);
Page<NtcL2tpLog> ntcL2tpLogPage = new Page<>(); Page<NtcL2tpLog> ntcL2tpLogPage = new Page<>();
try { try {
ntcL2tpLogPage = testService.findNtcL2tpLogPage( ntcL2tpLogPage = testService.findNtcL2tpLogPage(new Page<NtcL2tpLog>(request, response, NtcL2tpLog.class),
new Page<NtcL2tpLog>(request, response, NtcL2tpLog.class), ntcL2tpLog); ntcL2tpLog);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "L2TP日志检索成功",
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "L2TP日志检索成功", ntcL2tpLogPage, 0); ntcL2tpLogPage, 0);
} }
@RequestMapping(value = "/ntcOpenvpnLogs", method = RequestMethod.GET) @RequestMapping(value = "/ntcOpenvpnLogs", method = RequestMethod.GET)
@ApiOperation(value = "OPENVPN日志查询", httpMethod = "GET", notes = "对应配置为“隧道管理-OPENVPN”，存储动作为阻断与监测的命中日志。对日志功能“隧道管理- OPENVPN”提供数据基础查询服务。") @ApiOperation(value = "OPENVPN日志查询", httpMethod = "GET", notes = "对应配置为“隧道管理-OPENVPN”，存储动作为阻断与监测的命中日志。对日志功能“隧道管理- OPENVPN”提供数据基础查询服务。")
public Map<String, ?> ntcOpenvpnLogs(Page page, NtcOpenvpnLog ntcOpenvpnLog, Model model, HttpServletRequest request, HttpServletResponse response) { public Map<String, ?> ntcOpenvpnLogs(Page page, NtcOpenvpnLog ntcOpenvpnLog, Model model,
HttpServletRequest request, HttpServletResponse response) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET,
request, null);
Page<NtcOpenvpnLog> ntcOpenvpnLogPage = new Page<>(); Page<NtcOpenvpnLog> ntcOpenvpnLogPage = new Page<>();
try { try {
@@ -200,85 +260,119 @@ public class LogController extends BaseRestController{
e.printStackTrace(); e.printStackTrace();
} }
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "Openvpn日志检索成功",
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "Openvpn日志检索成功", ntcOpenvpnLogPage, 0); ntcOpenvpnLogPage, 0);
} }
@RequestMapping(value = "/ntcIpsecLogs", method = RequestMethod.GET) @RequestMapping(value = "/ntcIpsecLogs", method = RequestMethod.GET)
@ApiOperation(value = "IPSEC日志查询", httpMethod = "GET", notes = "对应配置为“隧道管理-IPSEC”，存储动作为监测的命中日志。对日志功能“隧道管理- IPSEC”提供数据基础查询服务。") @ApiOperation(value = "IPSEC日志查询", httpMethod = "GET", notes = "对应配置为“隧道管理-IPSEC”，存储动作为监测的命中日志。对日志功能“隧道管理- IPSEC”提供数据基础查询服务。")
public Map<String, ?> ntcIpsecLogs(Page page, NtcIpsecLog ntcIpsecLog, Model model, HttpServletRequest request, HttpServletResponse response) { public Map<String, ?> ntcIpsecLogs(Page page, NtcIpsecLog ntcIpsecLog, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET,
request, null);
Page<NtcIpsecLog> ntcIpsecLogPage = new Page<>(); Page<NtcIpsecLog> ntcIpsecLogPage = new Page<>();
try { try {
ntcIpsecLogPage = testService.findNtcIpsecLogPage( ntcIpsecLogPage = testService
new Page<NtcIpsecLog>(request, response, NtcIpsecLog.class), ntcIpsecLog); .findNtcIpsecLogPage(new Page<NtcIpsecLog>(request, response, NtcIpsecLog.class), ntcIpsecLog);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "IPSEC日志检索成功",
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "IPSEC日志检索成功", ntcIpsecLogPage, 0); ntcIpsecLogPage, 0);
} }
@RequestMapping(value = "/ntcSshLogs", method = RequestMethod.GET) @RequestMapping(value = "/ntcSshLogs", method = RequestMethod.GET)
@ApiOperation(value = "SSH日志查询", httpMethod = "GET", notes = "对应配置为“隧道管理-SSH”，存储动作为阻断与监测的命中日志。对日志功能“隧道管理- SSH”提供数据基础查询服务。") @ApiOperation(value = "SSH日志查询", httpMethod = "GET", notes = "对应配置为“隧道管理-SSH”，存储动作为阻断与监测的命中日志。对日志功能“隧道管理- SSH”提供数据基础查询服务。")
public Map<String, ?> ntcSshLogs(Page page, NtcSshLog ntcSshLog, Model model, HttpServletRequest request, HttpServletResponse response) { public Map<String, ?> ntcSshLogs(Page page, NtcSshLog ntcSshLog, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET,
request, null);
Page<NtcSshLog> ntcSshLogPage = new Page<>(); Page<NtcSshLog> ntcSshLogPage = new Page<>();
try { try {
ntcSshLogPage = testService.findNtcSshLogPage( ntcSshLogPage = testService.findNtcSshLogPage(new Page<NtcSshLog>(request, response, NtcSshLog.class),
new Page<NtcSshLog>(request, response, NtcSshLog.class), ntcSshLog); ntcSshLog);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "SSH日志检索成功",
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "SSH日志检索成功", ntcSshLogPage, 0); ntcSshLogPage, 0);
} }
@RequestMapping(value = "/ntcFtpLogs", method = RequestMethod.GET) @RequestMapping(value = "/ntcFtpLogs", method = RequestMethod.GET)
@ApiOperation(value = "FTP日志查询", httpMethod = "GET", notes = "对应配置为“文件传输-FTP”，存储动作为阻断与监测的命中日志。对日志功能“文件传输-FTP”提供数据基础查询服务。") @ApiOperation(value = "FTP日志查询", httpMethod = "GET", notes = "对应配置为“文件传输-FTP”，存储动作为阻断与监测的命中日志。对日志功能“文件传输-FTP”提供数据基础查询服务。")
public Map<String, ?> ntcFtpLogs(Page page, NtcFtpLog ntcFtpLog, Model model, HttpServletRequest request, HttpServletResponse response) { public Map<String, ?> ntcFtpLogs(Page page, NtcFtpLog ntcFtpLog, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET,
request, null);
Page<NtcFtpLog> ntcFtpLogPage = new Page<>(); Page<NtcFtpLog> ntcFtpLogPage = new Page<>();
try { try {
ntcFtpLogPage = testService.findNtcFtpLogPage( ntcFtpLogPage = testService.findNtcFtpLogPage(new Page<NtcFtpLog>(request, response, NtcSshLog.class),
new Page<NtcFtpLog>(request, response, NtcSshLog.class), ntcFtpLog); ntcFtpLog);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "FTP日志检索成功",
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "FTP日志检索成功", ntcFtpLogPage, 0); ntcFtpLogPage, 0);
} }
@RequestMapping(value = "/ntcPzReport", method = RequestMethod.GET) @RequestMapping(value = "/ntcPzReport", method = RequestMethod.GET)
@ApiOperation(value = "配置日志总量统计", httpMethod = "GET", notes = "配置命中日志数量实时统计报表，对外提供多种数据表现形式，具体可应用于界面配置命中总量业务、配置报表业务等") @ApiOperation(value = "配置日志总量统计", httpMethod = "GET", notes = "配置命中日志数量实时统计报表，对外提供多种数据表现形式，具体可应用于界面配置命中总量业务、配置报表业务等")
public Map<String, ?> ntcPzReport(Page page, NtcPzReport ntcPzReport, Model model, HttpServletRequest request, HttpServletResponse response) { public Map<String, ?> ntcPzReport(Page page, NtcPzReport ntcPzReport, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET,
request, null);
Page<?> ntcPzReportPage = new Page<>(); Page<?> ntcPzReportPage = new Page<>();
try { try {
ntcPzReportPage = testService.findNtcPzReport(new Page<NtcPzReport>(request, response,NtcPzReport.class), ntcPzReport); ntcPzReportPage = testService.findNtcPzReport(new Page<NtcPzReport>(request, response, NtcPzReport.class),
ntcPzReport);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "配置总量统计成功", ntcPzReportPage, 0); return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "配置总量统计成功",
ntcPzReportPage, 0);
} }
/**
*判断开始和结束时间是否为null,如果为null则初始化时间
* @param entity
* @throws Exception
*/
public void resetTime(LogEntity<?> entity) throws Exception {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Map<String, String> map = DateUtils.getLocalTime(entity.getSearchFoundStartTime(),
entity.getSearchFoundEndTime(), Constants.LOG_LOCAL_TIME, "log");
entity.setSearchFoundStartTime(map.get("startTime"));
entity.setSearchFoundEndTime(map.get("endTime"));
}
/**
* 将searchFoundStartTime,searchFoundEndTime与foundTime进行关联
* @return
*/
public Map<String, Map<String, String>> getCol2Col() {
Map<String, Map<String, String>> col2col = new HashMap<String, Map<String, String>>();
Map<String, String> startMap = new HashMap<String, String>();
startMap.put("start", "foundTime");
col2col.put("searchFoundStartTime", startMap);
Map<String, String> endMap = new HashMap<String, String>();
endMap.put("end", "foundTime");
col2col.put("searchFoundEndTime", endMap);
return col2col;
}
} }

24
src/main/java/com/nis/web/dao/DfLogSearchDao.java Normal file
View File

@@ -0,0 +1,24 @@
package com.nis.web.dao;
import java.util.List;
import com.nis.domain.LogEntity;
import com.nis.domain.restful.NtcIpLog;
/**
*
* <p>Title: DfLogSearchDao</p>
* <p>Description: 日志查询dao</p>
* <p>Company: IIE</p>
* @author rkg
* @date 2018年7月2日
*
*/
@MyBatisDao
public interface DfLogSearchDao extends CrudDao<LogEntity> {
// List<NtcIpLog> findNtcIpLog(NtcIpLog log);
}

49
src/main/java/com/nis/web/dao/DfLogSearchDao.xml Normal file
View File

@@ -0,0 +1,49 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.nis.web.dao.DfLogSearchDao">
<!-- 所有日志表的公共属性 -->
<sql id="commonPorperty">
cfg_id ,
found_time ,
recv_time ,
trans_proto ,
addr_type ,
d_ip ,
s_ip ,
d_port ,
s_port ,
service ,
entrance_id ,
device_id ,
direction ,
stream_dir ,
cap_ip ,
addr_list ,
user_region
</sql>
<resultMap id="NtcIpLogMap" type="com.nis.domain.restful.NtcIpLog">
<result column="cfg_id" jdbcType="BIGINT" property="cfgId" />
<result column="found_Time" jdbcType="TIMESTAMP" property="foundTime" />
<result column="recv_Time" jdbcType="TIMESTAMP" property="recvTime" />
<result column="trans_proto" jdbcType="VARCHAR" property="transProto" />
<result column="addr_type" jdbcType="INTEGER" property="addrType" />
<result column="d_ip" jdbcType="VARCHAR" property="dIp" />
<result column="s_ip" jdbcType="VARCHAR" property="sIp" />
<result column="d_port" jdbcType="VARCHAR" property="dPort" />
<result column="s_port" jdbcType="VARCHAR" property="sPort" />
<result column="service" jdbcType="INTEGER" property="service" />
<result column="entrance_id" jdbcType="BIGINT" property="entranceId" />
<result column="device_id" jdbcType="INTEGER" property="deviceId" />
<result column="direction" jdbcType="INTEGER" property="direction" />
<result column="stream_dir" jdbcType="INTEGER" property="streamDir" />
<result column="cap_ip" jdbcType="VARCHAR" property="capIp" />
<result column="addr_list" jdbcType="VARCHAR" property="addrList" />
<result column="user_region" jdbcType="VARCHAR" property="userRegion" />
</resultMap>
</mapper>

39
src/main/java/com/nis/web/service/BaseLogService.java
View File

@@ -40,6 +40,7 @@ public abstract class BaseLogService {
protected final Logger logger = Logger.getLogger(this.getClass()); protected final Logger logger = Logger.getLogger(this.getClass());
@Autowired @Autowired
protected ElasticsearchSqlDao elasticsearchSqlDao; protected ElasticsearchSqlDao elasticsearchSqlDao;
/** /**
* wx checkCloumnIsExist(这里用一句话描述这个方法的作用) (这里描述这个方法适用条件 – 可选) * wx checkCloumnIsExist(这里用一句话描述这个方法的作用) (这里描述这个方法适用条件 – 可选)
* *
@@ -126,7 +127,16 @@ public abstract class BaseLogService {
} }
} }
public void queryConditionCheck(SaveRequestLogThread thread,long start,LogEntity<?> entity,Class clazz,Page page) { /**
* 验证日志查询条件格式是否正确
* @param thread
* @param start
* @param entity
* @param clazz
* @param page
*/
public void queryConditionCheck(SaveRequestLogThread thread, long start, LogEntity<?> entity, Class clazz,
Page page) {
logger.info("请求参数校验开始----" + System.currentTimeMillis()); logger.info("请求参数校验开始----" + System.currentTimeMillis());
try { try {
if (!StringUtil.isBlank(entity.getSearchCfgId())) { if (!StringUtil.isBlank(entity.getSearchCfgId())) {
@@ -143,6 +153,21 @@ public abstract class BaseLogService {
throw new RestServiceException(thread, System.currentTimeMillis() - start, "searchCfgId参数错误"); throw new RestServiceException(thread, System.currentTimeMillis() - start, "searchCfgId参数错误");
} }
try {
if (!StringUtil.isBlank(entity.getSearchDirection())) {
Integer.parseInt(entity.getSearchDirection());
}
} catch (NumberFormatException e) {
thread.setExceptionInfo(e.getMessage() + " " + e.getCause());
logger.error(e);
throw new RestServiceException(thread, System.currentTimeMillis() - start, "getSearchDirection参数格式错误",
RestBusinessCode.param_formate_error.getValue());
} catch (Exception e) {
thread.setExceptionInfo(e.getMessage() + " " + e.getCause());
logger.error(e);
throw new RestServiceException(thread, System.currentTimeMillis() - start, "getSearchDirection参数错误");
}
try { try {
if (!StringUtil.isBlank(entity.getSearchEntranceId())) { if (!StringUtil.isBlank(entity.getSearchEntranceId())) {
Long.parseLong(entity.getSearchEntranceId()); Long.parseLong(entity.getSearchEntranceId());
@@ -189,8 +214,8 @@ public abstract class BaseLogService {
} }
try { try {
if (!StringUtil.isBlank(entity.getSearchServiceType())) { if (!StringUtil.isBlank(entity.getSearchService())) {
Integer.parseInt(entity.getSearchServiceType()); Integer.parseInt(entity.getSearchService());
} }
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
thread.setExceptionInfo(e.getMessage() + " " + e.getCause()); thread.setExceptionInfo(e.getMessage() + " " + e.getCause());
@@ -284,7 +309,6 @@ public abstract class BaseLogService {
} }
/** /**
* 管控实时统计公共查询字段验证 * 管控实时统计公共查询字段验证
* *
@@ -375,7 +399,8 @@ public abstract class BaseLogService {
* *
* @param entity * @param entity
*/ */
public void queryConditionCheck(SaveRequestLogThread thread, long start,DfReportEntity entity, Class clazz, Page page) { public void queryConditionCheck(SaveRequestLogThread thread, long start, DfReportEntity entity, Class clazz,
Page page) {
try { try {
if (!StringUtil.isBlank(entity.getSearchReportEndTime())) { if (!StringUtil.isBlank(entity.getSearchReportEndTime())) {
sdf.parse(entity.getSearchReportEndTime()); sdf.parse(entity.getSearchReportEndTime());
@@ -456,7 +481,6 @@ public abstract class BaseLogService {
throw new RestServiceException(thread, start, "searchService参数错误"); throw new RestServiceException(thread, start, "searchService参数错误");
} }
try { try {
checkCloumnIsExist(thread, start, clazz, page); checkCloumnIsExist(thread, start, clazz, page);
} catch (RestServiceException e) { } catch (RestServiceException e) {
@@ -538,8 +562,7 @@ public abstract class BaseLogService {
return key; return key;
} }
protected String getSearchField(Class<?> clazz, protected String getSearchField(Class<?> clazz, Serializable entity, String fieldName) {
Serializable entity, String fieldName) {
try { try {
Field field = clazz.getDeclaredField(fieldName); Field field = clazz.getDeclaredField(fieldName);
field.setAccessible(true); field.setAccessible(true);

54
src/main/java/com/nis/web/service/HiveSqlService.java
View File

@@ -14,7 +14,8 @@ import java.util.Map;
import org.apache.ibatis.mapping.ResultMap; import org.apache.ibatis.mapping.ResultMap;
import org.apache.ibatis.mapping.ResultMapping; import org.apache.ibatis.mapping.ResultMapping;
import org.apache.ibatis.session.SqlSessionFactory; import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.log4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.nis.domain.Page; import com.nis.domain.Page;
import com.nis.util.Configurations; import com.nis.util.Configurations;
@@ -25,13 +26,12 @@ import com.nis.util.StringUtil;
import com.nis.util.redis.SaveRedisThread; import com.nis.util.redis.SaveRedisThread;
public class HiveSqlService { public class HiveSqlService {
private final static Logger logger = Logger.getLogger(HiveJDBC.class); private final static Logger logger = LoggerFactory.getLogger(HiveSqlService.class);
private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
public static ResultSet getResultSet(Page page, Object bean, String tableName, public static ResultSet getResultSet(Page page, Object bean, String tableName,
Map<String, Map<String, String>> col2col, String orderBy, String searchActiveSys) throws Exception { Map<String, Map<String, String>> col2col, String orderBy, String searchActiveSys) throws Exception {
tableName = tableName.toLowerCase(); tableName = tableName.toLowerCase();
tableName = Configurations.getStringProperty(tableName, "t_" + tableName).trim();
String showColmun = getFiledsSql(bean.getClass().getSimpleName(), page.getFields()); String showColmun = getFiledsSql(bean.getClass().getSimpleName(), page.getFields());
StringBuffer sql = new StringBuffer(); StringBuffer sql = new StringBuffer();
Map<String, String> filedAndColumnMap = getFiledAndColumnMap(bean.getClass()); Map<String, String> filedAndColumnMap = getFiledAndColumnMap(bean.getClass());
@@ -59,8 +59,10 @@ public class HiveSqlService {
Long foundTimePartStart = null; Long foundTimePartStart = null;
Long foundTimePartEnd = null; Long foundTimePartEnd = null;
for (int i = 0; i < fields.length; i++) { for (int i = 0; i < fields.length; i++) {
// 现在gwall日志表结构中只有数值和字符串两种类型,数值都是int类型没有bigint所以不需要加L,
Field f = fields[i]; Field f = fields[i];
String key = f.getName();// 获取字段名 String key = f.getName();// 获取字段名
String typeName = f.getType().getName();
if (f.getType().getName().equals("java.lang.String") && key.startsWith("search")) { if (f.getType().getName().equals("java.lang.String") && key.startsWith("search")) {
Object value = getFieldValue(bean, key); Object value = getFieldValue(bean, key);
if (value != null) { if (value != null) {
@@ -81,7 +83,7 @@ public class HiveSqlService {
// value.toString().trim() // value.toString().trim()
// + "','yyyy-mm-dd HH24:mi:ss')"); // + "','yyyy-mm-dd HH24:mi:ss')");
sql.append(" and " + filedAndColumnMap.get(col2col.get(key).get("start")) + ">=" sql.append(" and " + filedAndColumnMap.get(col2col.get(key).get("start")) + ">="
+ value + "L"); + value);
} else { } else {
// sql.append(" and " + // sql.append(" and " +
// filedAndColumnMap.get(col2col.get(key).get("end")) // filedAndColumnMap.get(col2col.get(key).get("end"))
@@ -89,7 +91,7 @@ public class HiveSqlService {
// value.toString().trim() // value.toString().trim()
// + "','yyyy-mm-dd HH24:mi:ss')"); // + "','yyyy-mm-dd HH24:mi:ss')");
sql.append(" and " + filedAndColumnMap.get(col2col.get(key).get("end")) + "<" sql.append(" and " + filedAndColumnMap.get(col2col.get(key).get("end")) + "<"
+ value + "L"); + value);
} }
} }
} else { } else {
@@ -98,39 +100,29 @@ public class HiveSqlService {
key = key.substring(0, 1).toLowerCase() + key.substring(1); key = key.substring(0, 1).toLowerCase() + key.substring(1);
} }
if (!value.toString().trim().equals("") && filedAndColumnMap.containsKey(key) if (typeName.equals("java.lang.String")) {
&& (key.toLowerCase().equals("cfgid")
|| key.toLowerCase().equals("entranceid"))) {
sql.append(
" and " + filedAndColumnMap.get(key) + "=" + value.toString().trim() + "L");
} else if (!value.toString().trim().equals("") && filedAndColumnMap.containsKey(key)
&& (key.toLowerCase().equals("protocol") || key.toLowerCase().equals("serverip")
|| key.toLowerCase().equals("clientip")
|| key.toLowerCase().equals("url")
|| key.toLowerCase().equals("mailfrom")
|| key.toLowerCase().equals("mailto")
|| key.toLowerCase().equals("encryptmode")
|| key.toLowerCase().equals("exprotocol")
|| key.toLowerCase().equals("cljip"))) {
sql.append(" and " + filedAndColumnMap.get(key) + "='" + value.toString().trim() sql.append(" and " + filedAndColumnMap.get(key) + "='" + value.toString().trim()
+ "'"); + "'");
} else if (!value.toString().trim().equals("") && filedAndColumnMap.containsKey(key) } else if (typeName.equals("java.lang.Integer") || typeName.equals("int")) {
&& key.toLowerCase().equals("servicetype")) {
sql.append(" and " + filedAndColumnMap.get(key) + "=" + value.toString().trim()); sql.append(" and " + filedAndColumnMap.get(key) + "=" + value.toString().trim());
} else if (typeName.equals("java.lang.Long") || typeName.equals("long")) {
sql.append(
" and " + filedAndColumnMap.get(key) + "=" + value.toString().trim() + "L");
} }
} }
} }
} }
} }
if (null != searchActiveSys) {// 添加分区字段
//if (null != searchActiveSys && !searchActiveSys.equals(Constants.ACTIVESYS_A)) {// B版数据库才有found_time_partition字段,A版毛衣found_time_partition分区字段
if (null != searchActiveSys){//为A版日志库添加分区字段
if (null != foundTimePartStart) { if (null != foundTimePartStart) {
sql.append(" and found_time_partition>=" + foundTimePartStart + "L"); // sql.append(" and found_time_partition>=" + foundTimePartStart + "L");
sql.append(" and found_time_partition>=" + foundTimePartStart);
} }
if (null != foundTimePartEnd) { if (null != foundTimePartEnd) {
sql.append(" and found_time_partition<" + foundTimePartEnd + "L"); // sql.append(" and found_time_partition<" + foundTimePartEnd + "L");
sql.append(" and found_time_partition<" + foundTimePartEnd);
} }
} }
@@ -141,9 +133,9 @@ public class HiveSqlService {
// sql.append(" order by " + orderBy + " limit 10000) t1) t2 where // sql.append(" order by " + orderBy + " limit 10000) t1) t2 where
// row_Num between " + startNum + " and " + endNum); // row_Num between " + startNum + " and " + endNum);
sql.append(" limit " + Constants.EVERY_GETHIVEDATANUM); sql.append(" limit " + Constants.EVERY_GETHIVEDATANUM);
logger.info("获取数据中心日志sql===================" + sql); logger.info("获取数据中心日志sql==================={}", sql);
// ResultSet query = HiveJDBC.query(sql.toString()); // ResultSet query = HiveJDBC.query(sql.toString());
ResultSet query = HiveDataSource.query(sql.toString(), searchActiveSys); ResultSet query = HiveDataSource.query(sql.toString());
logger.info("获取数据中心日志成功"); logger.info("获取数据中心日志成功");
return query; return query;
} }
@@ -219,7 +211,9 @@ public class HiveSqlService {
} }
} }
//if (null != searchActiveSys && !searchActiveSys.equals(Constants.ACTIVESYS_A)) {// B版数据库才有found_time_partition字段,A版毛衣found_time_partition分区字段 // if (null != searchActiveSys &&
// !searchActiveSys.equals(Constants.ACTIVESYS_A)) {//
// B版数据库才有found_time_partition字段,A版毛衣found_time_partition分区字段
if (null != searchActiveSys) { if (null != searchActiveSys) {
if (null != foundTimePartStart) { if (null != foundTimePartStart) {
countSql.append(" and found_time_partition>=" + foundTimePartStart + "L"); countSql.append(" and found_time_partition>=" + foundTimePartStart + "L");
@@ -232,7 +226,7 @@ public class HiveSqlService {
} }
logger.info("获取数据中心日志总条数sql==================" + countSql.toString()); logger.info("获取数据中心日志总条数sql==================" + countSql.toString());
// ResultSet countRs = HiveJDBC.query(countSql.toString()); // ResultSet countRs = HiveJDBC.query(countSql.toString());
ResultSet countRs = HiveDataSource.query(countSql.toString(), searchActiveSys); ResultSet countRs = HiveDataSource.query(countSql.toString());
String countStr = null; String countStr = null;
while (countRs.next()) { while (countRs.next()) {
countStr = countRs.getObject(1).toString(); countStr = countRs.getObject(1).toString();

2
src/main/java/com/nis/web/service/SaveRequestLogThread.java
View File

@@ -51,7 +51,7 @@ public class SaveRequestLogThread implements Runnable {
public void run() { public void run() {
logger.info("线程开始执行！"); logger.info("线程开始执行！");
//新开线程切换数据源，不影响action中的数据源 //新开线程切换数据源，不影响action中的数据源
CustomerContextHolder.setCustomerType(CustomerContextHolder.DATA_SOURCE_C);//开启数据源C CustomerContextHolder.setCustomerType(CustomerContextHolder.DATA_SOURCE_B);//开启数据源B
// TODO Auto-generated method stub // TODO Auto-generated method stub
if(service!=null){ if(service!=null){
service.saveRequestLog(remoteAddr,requestURI,queryString,contextPath, operator, version, opAction, opTime, content, requestTime, consumerTime,businessCode,exceptionInfo,traceCode); service.saveRequestLog(remoteAddr,requestURI,queryString,contextPath, operator, version, opAction, opTime, content, requestTime, consumerTime,businessCode,exceptionInfo,traceCode);

61
src/main/resources/applicationContext-mybatis.xml
View File

@@ -78,13 +78,13 @@
</bean> </bean>
<!--A版hive数据源定义，采用BoneCP连接池 ,业务配置服务数据库 --> <!--hive数据源定义，采用BoneCP连接池 ,业务配置服务数据库 -->
<bean id="HiveADataSource" parent="parentDataSource" <bean id="HiveDataSource" parent="parentDataSource"
class="com.jolbox.bonecp.BoneCPDataSource" destroy-method="close"> class="com.jolbox.bonecp.BoneCPDataSource" destroy-method="close">
<property name="driverClass" value="${jdbc.hiveA.driver}"></property> <property name="driverClass" value="${jdbc.hive.driver}"></property>
<property name="jdbcUrl" value="${jdbc.hiveA.url}"></property> <property name="jdbcUrl" value="${jdbc.hive.url}"></property>
<property name="username" value="${jdbc.hiveA.username}"></property> <property name="username" value="${jdbc.hive.username}"></property>
<property name="password" value="${jdbc.hiveA.password}"></property> <property name="password" value="${jdbc.hive.password}"></property>
<!-- 检查数据库连接池中空闲连接的间隔时间，单位是分，默认值：240，如果要取消则设置为0 --> <!-- 检查数据库连接池中空闲连接的间隔时间，单位是分，默认值：240，如果要取消则设置为0 -->
<property name="idleConnectionTestPeriodInMinutes" <property name="idleConnectionTestPeriodInMinutes"
value="${bonecp.hive.idleConnectionTestPeriodInMinutes}"></property> value="${bonecp.hive.idleConnectionTestPeriodInMinutes}"></property>
@@ -102,59 +102,12 @@
<property name="statementsCacheSize" value="${bonecp.hive.statementsCacheSize}" /> <property name="statementsCacheSize" value="${bonecp.hive.statementsCacheSize}" />
</bean> </bean>
<!--B版hive数据源定义，采用BoneCP连接池 ,业务配置服务数据库 -->
<bean id="HiveBDataSource" parent="parentDataSource"
class="com.jolbox.bonecp.BoneCPDataSource" destroy-method="close">
<property name="driverClass" value="${jdbc.hiveB.driver}"></property>
<property name="jdbcUrl" value="${jdbc.hiveB.url}"></property>
<property name="username" value="${jdbc.hiveB.username}"></property>
<property name="password" value="${jdbc.hiveB.password}"></property>
<!-- 检查数据库连接池中空闲连接的间隔时间，单位是分，默认值：240，如果要取消则设置为0 -->
<property name="idleConnectionTestPeriodInMinutes"
value="${bonecp.hive.idleConnectionTestPeriodInMinutes}"></property>
<!-- 连接池中未使用的链接最大存活时间，单位是分，默认值：60，如果要永远存活设置为0 -->
<property name="idleMaxAgeInMinutes" value="${bonecp.hive.idleMaxAgeInMinutes}" />
<!-- 每个分区最大的连接数 -->
<property name="maxConnectionsPerPartition" value="${bonecp.hive.maxConnectionsPerPartition}" />
<!-- 每个分区最小的连接数 -->
<property name="minConnectionsPerPartition" value="${bonecp.hive.minConnectionsPerPartition}" />
<!-- 分区数 ，默认值2，最小1，推荐3-4，视应用而定 -->
<property name="partitionCount" value="${bonecp.hive.partitionCount}" />
<!-- 每次去拿数据库连接的时候一次性要拿几个,默认值：2 -->
<property name="acquireIncrement" value="${bonecp.hive.acquireIncrement}" />
<!-- 缓存prepared statements的大小，默认值：0 -->
<property name="statementsCacheSize" value="${bonecp.hive.statementsCacheSize}" />
</bean>
<!--数据源定义，采用BoneCP连接池 ,神通数据库 -->
<bean id="LogClusterDataSource" parent="parentDataSource"
class="com.jolbox.bonecp.BoneCPDataSource" destroy-method="close">
<property name="driverClass" value="${jdbc.log.cluster.driver}"></property>
<property name="jdbcUrl" value="${jdbc.log.cluster.url}"></property>
<property name="username" value="${jdbc.log.cluster.username}"></property>
<property name="password" value="${jdbc.log.cluster.password}"></property>
<!-- 检查数据库连接池中空闲连接的间隔时间，单位是分，默认值：240，如果要取消则设置为0 -->
<property name="idleConnectionTestPeriodInMinutes" value="${bonecp.cluster.idleConnectionTestPeriodInMinutes}"></property>
<!-- 连接池中未使用的链接最大存活时间，单位是分，默认值：60，如果要永远存活设置为0 -->
<property name="idleMaxAgeInMinutes" value="${bonecp.cluster.idleMaxAgeInMinutes}" />
<!-- 每个分区最大的连接数 -->
<property name="maxConnectionsPerPartition" value="${bonecp.cluster.maxConnectionsPerPartition}" />
<!-- 每个分区最小的连接数 -->
<property name="minConnectionsPerPartition" value="${bonecp.cluster.minConnectionsPerPartition}" />
<!-- 分区数 ，默认值2，最小1，推荐3-4，视应用而定 -->
<property name="partitionCount" value="${bonecp.cluster.partitionCount}" />
<!-- 每次去拿数据库连接的时候一次性要拿几个,默认值：2 -->
<property name="acquireIncrement" value="${bonecp.cluster.acquireIncrement}" />
<!-- 缓存prepared statements的大小，默认值：0 -->
<property name="statementsCacheSize" value="${bonecp.cluster.statementsCacheSize}" />
</bean>
<bean id="dynamicDataSource" class="com.nis.datasource.DynamicDataSource"> <bean id="dynamicDataSource" class="com.nis.datasource.DynamicDataSource">
<property name="targetDataSources"> <property name="targetDataSources">
<map key-type="java.lang.String"> <map key-type="java.lang.String">
<entry value-ref="DevlopDataSource" key="dataSourceA"></entry> <entry value-ref="DevlopDataSource" key="dataSourceA"></entry>
<entry value-ref="LogDataSource" key="dataSourceC"></entry> <entry value-ref="LogDataSource" key="dataSourceB"></entry>
<entry value-ref="LogClusterDataSource" key="dataSourceH"></entry>
</map> </map>
</property> </property>
<property name="defaultTargetDataSource" ref="DevlopDataSource"></property> <property name="defaultTargetDataSource" ref="DevlopDataSource"></property>

37
src/main/resources/jdbc.properties
View File

@@ -32,46 +32,23 @@ bonecp.partitionCount=3
bonecp.acquireIncrement=5 bonecp.acquireIncrement=5
bonecp.statementsCacheSize=100 bonecp.statementsCacheSize=100
############################################################################################################################################
#数据中心神通数据库接口配置
############################################################################################################################################
bonecp.cluster.idleMaxAgeInMinutes=60
bonecp.cluster.idleConnectionTestPeriodInMinutes=240
bonecp.cluster.maxConnectionsPerPartition=10
bonecp.cluster.minConnectionsPerPartition=5
bonecp.cluster.partitionCount=3
bonecp.cluster.acquireIncrement=5
bonecp.cluster.statementsCacheSize=100
jdbc.log.cluster.driver=com.oscar.cluster.BulkDriver
jdbc.log.cluster.url=jdbc:oscarclusterbulk://10.3.129.125:2010/logdb
jdbc.log.cluster.username=xa_z2_iie
jdbc.log.cluster.key=2fa3hQn28+4AOdJXL4Ud2w==
jdbc.log.cluster.password=uut+weC9of5ocPheagBJ4A==
############################################################################################################################################ ############################################################################################################################################
#数据中心hive接口配置 #数据中心hive接口配置
############################################################################################################################################ ############################################################################################################################################
#A版日志库 #A版日志库
jdbc.hiveA.driver=org.apache.hive.jdbc.HiveDriver jdbc.hive.driver=org.apache.hive.jdbc.HiveDriver
jdbc.hiveA.url=jdbc:hive2://10.3.130.24:10000/default jdbc.hive.url=jdbc:hive2://10.0.6.202:10000/maat
jdbc.hiveA.username=xa_z2_mesa jdbc.hive.username=xa_z2_mesa
jdbc.hiveA.key=aC/8fTC9vfPVhCk+CDzbAQ== jdbc.hive.key=aC/8fTC9vfPVhCk+CDzbAQ==
#加密后密码 #加密后密码
jdbc.hiveA.password=V3GyFlG8Mg01bTt8ykFVaA== jdbc.hive.password=V3GyFlG8Mg01bTt8ykFVaA==
#实际密码 #实际密码
#jdbc.hiveA.password=123!@#qwe #jdbc.hiveA.password=123!@#qwe
#B版日志库
jdbc.hiveB.driver=org.apache.hive.jdbc.HiveDriver
jdbc.hiveB.url=jdbc:hive2://10.3.130.25:10000/default
jdbc.hiveB.username=xa_z2_mesa
jdbc.hiveB.key=aC/8fTC9vfPVhCk+CDzbAQ==
#加密后密码
jdbc.hiveB.password=V3GyFlG8Mg01bTt8ykFVaA==
#实际密码
#jdbc.hiveB.password=123!@#qwe
bonecp.hive.idleMaxAgeInMinutes=60 bonecp.hive.idleMaxAgeInMinutes=60
bonecp.hive.idleConnectionTestPeriodInMinutes=240 bonecp.hive.idleConnectionTestPeriodInMinutes=240
bonecp.hive.maxConnectionsPerPartition=20 bonecp.hive.maxConnectionsPerPartition=20

10
src/main/resources/nis.properties
View File

@@ -183,7 +183,7 @@ selFromHive=false
isGetHiveCount=false isGetHiveCount=false
#每次获取数据中心多少条数据,咱们在对获取的数据进行分页处理 #每次获取数据中心多少条数据,咱们在对获取的数据进行分页处理
everyGetHiveDataNum=10000 everyGetHiveDataNum=100
#oracle数据库有问题不从oracle查询数据,所有日志数据均从数据中心查询 #oracle数据库有问题不从oracle查询数据,所有日志数据均从数据中心查询
onlySelFromHive=false onlySelFromHive=false
@@ -229,10 +229,8 @@ isUseES=false
#数据中心A版数据库名称,程序中每次查询时使用的数据库名称 use dbA #数据中心hive日志库数据库名称,程序中每次查询时使用的数据库名称 use dbName
jdbc.hive.AName=xa_dfbhit_hive jdbc.hive.Name=maat
#数据中心B版数据库名称
jdbc.hive.BName=xa_z2_mesalog_hive
maxPageSize=100000 maxPageSize=100000
@@ -243,8 +241,6 @@ isDebug=true
digest.gen.tool.path=maat-redis/digest_gen digest.gen.tool.path=maat-redis/digest_gen
#redis中有多少个数据库(需要加1,代码中用的小于不是小于等于) #redis中有多少个数据库(需要加1,代码中用的小于不是小于等于)
maxRedisDBIndex=12 maxRedisDBIndex=12
##存放编译,分组,域配置id关系的redis数据库编号 ##存放编译,分组,域配置id关系的redis数据库编号

14
src/main/resources/spring-mvc.xml
View File

@@ -179,11 +179,17 @@
<bean class="com.nis.interceptor.LogInterceptor" /> <bean class="com.nis.interceptor.LogInterceptor" />
</mvc:interceptor> </mvc:interceptor>
<!-- 数据源拦截器，该拦截路径下使用数据源C --> <!-- 日志数据源拦截器，该拦截路径下使用数据源C -->
<!-- <mvc:interceptor> -->
<!-- <mvc:mapping path="/service/log/**" /> -->
<!-- <mvc:mapping path="/log/**"/> -->
<!-- <bean class="com.nis.interceptor.DataSourceCInterceptor"></bean> -->
<!-- </mvc:interceptor> -->
<!-- 数据源拦截器，该拦截路径下使用数据源B -->
<mvc:interceptor> <mvc:interceptor>
<mvc:mapping path="/service/log/**" /> <mvc:mapping path="/service/cfg/**" />
<mvc:mapping path="/log/**"/> <bean class="com.nis.interceptor.DataSourceBInterceptor"></bean>
<bean class="com.nis.interceptor.DataSourceCInterceptor"></bean>
</mvc:interceptor> </mvc:interceptor>
</mvc:interceptors> </mvc:interceptors>

62
src/main/resources/table.properties
View File

@@ -1,62 +1,6 @@
#由于数据中心日志表的表名会变动,所以本系统中将日志中心的表名提取到配置文件中方便后期修改 #由于数据中心hive日志表的表名会变动,所以本系统中将日志的表名提取到配置文件中方便后期修改,key是bean的名称+HiveTable,value是hive的实际表名
#\u6570\u636E\u4E2D\u5FC3hive\u6570\u636E\u5E93\u8868\u540D NtcIpLogHiveTable=NTC_IP_LOG
df_ip_port_log=t_df_ip_port_log NtcHttpLogHiveTable=NTC_HTTP_LOG
df_http_req_log=t_df_http_req_log_v3
df_http_res_log=t_df_http_res_log_v3
df_http_keyword_log=t_df_http_keyword_log
df_mail_log=t_df_mail_log
df_dns_log=t_df_dns_log
df_ftp_log=t_df_ftp_log
df_pptp_log=t_df_pptp_log
df_l2tp_log=t_df_l2tp_log
df_ipsec_log=t_df_ipsec_log
df_openvpn_log=t_df_openvpn_log
df_ssh_log=t_df_ssh_log
df_ssl_log=t_df_ssl_log
df_tunnel_random_log=t_df_tunnel_random_log
#\u6570\u636E\u4E2D\u5FC3\u795E\u901A\u6570\u636E\u5E93A\u7248\u8868\u540D
DfIpPortLogA=t_xa_df_ip_port_log_hit_mpp
DfHttpReqLogA=t_xa_df_http_req_log_hit_mpp
DfHttpResLogA=t_xa_df_http_res_log_hit_mpp
DfHttpKeywordLogA=t_xa_df_http_keyword_log_hit_mpp
DfMailLogA=t_xa_df_mail_log_hit_mpp
DfDnsLogA=t_xa_df_dns_log_hit_mpp
DfFtpLogA=t_xa_df_ftp_log_hit_mpp
DfPptpLogA=t_xa_df_pptp_log_hit_mpp
DfL2tpLogA=t_xa_df_l2tp_log_hit_mpp
DfIpsecLogA=t_xa_df_ipsec_log_hit_mpp
DfOpenvpnLogA=t_xa_df_openvpn_log_hit_mpp
DfSshLogA=t_xa_df_ssh_log_hit_mpp
DfSslLogA=t_xa_df_ssl_log_hit_mpp
DfTunnelRandomLogA=t_xa_df_tunnel_random_log_hit_mpp
#\u795E\u901A\u6570\u636E\u5E93B\u7248\u8868\u540D
DfIpPortLogB=t_xa_df_ip_port_log_mpp
DfHttpReqLogB=t_xa_df_http_req_log_mpp
DfHttpResLogB=t_xa_df_http_res_log_mpp
DfHttpKeywordLogB=t_xa_df_http_keyword_log_mpp
DfMailLogB=t_xa_df_mail_log_mpp
DfDnsLogB=t_xa_df_dns_log_mpp
DfFtpLogB=t_xa_df_ftp_log_mpp
DfPptpLogB=t_xa_df_pptp_log_mpp
DfL2tpLogB=t_xa_df_l2tp_log_mpp
DfIpsecLogB=t_xa_df_ipsec_log_mpp
DfOpenvpnLogB=t_xa_df_openvpn_log_mpp
DfSshLogB=t_xa_df_ssh_log_mpp
DfSslLogB=t_xa_df_ssl_log_mpp
DfTunnelRandomLogB=t_xa_df_tunnel_random_log_mpp