gfwleak/k18-ntcs-web-argus-service
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' of https://git.mesalab.cn/galaxy/galaxy-service.git into develop

Browse Source
This commit is contained in:
renkaige
2019-01-16 10:09:19 +06:00
parent 0a77b39cd5 13e2c9d9ce
commit 5d6df905d7
6 changed files with 346 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/com/nis/domain/restful/NtcMailLog.java
View File

@@ -27,8 +27,19 @@ public class NtcMailLog extends LogEntity{
protected String searchMailFrom; //发件人
protected String searchMailTo; //收件人
protected String searchSubject; //主题
protected String flag;//泛收的标志
public String getMailProto() {
@JsonIgnore
public String getFlag() {
return flag;
}
public void setFlag(String flag) {
this.flag = flag;
}
public String getMailProto() {
return mailProto;
}

77
src/main/java/com/nis/domain/restful/NtcSslRecordLog.java Normal file
View File

@@ -0,0 +1,77 @@
package com.nis.domain.restful;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.nis.domain.LogEntity;
import com.wordnik.swagger.annotations.ApiModelProperty;
/**
* Created by darnell on 2018/6/11.
*/
public class NtcSslRecordLog extends LogEntity {
private static final long serialVersionUID = 164434539180653065L;
@ApiModelProperty(value="版本信息", required=true)
protected String version;
@ApiModelProperty(value="Server Name Indication", required=true, notes = "主要解决一台服务器只能使用一个证书-域名的缺点，通过sni可支持一台服务器为多台域名提供服务")
protected String sni;
@ApiModelProperty(value="SAN-Subject Alternative Name Certificate", required=true, notes = "数字证书的一种形式，运行SSL证书绑定多个域名和子域名。另EV证书，扩展验证证书，只能绑定一个域名")
protected String san;
@ApiModelProperty(value="证书授权机构", required=true)
protected String cn;
@ApiModelProperty(value="SNI", required=true)
protected String searchSni ;
public String getVersion() {
return version;
}
public void setVersion(String version) {
this.version = version;
}
public String getSni() {
return sni;
}
public void setSni(String sni) {
this.sni = sni;
}
public String getSan() {
return san;
}
public void setSan(String san) {
this.san = san;
}
public String getCn() {
return cn;
}
public void setCn(String cn) {
this.cn = cn;
}
/**
* @param searchSni the searchSni to set
*/
public void setSearchSni(String searchSni) {
this.searchSni = searchSni;
}
/**
* @return the searchSni
*/
@JsonIgnore
public String getSearchSni() {
return searchSni;
}
}

38
src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java
View File

@@ -34,6 +34,7 @@ import com.nis.domain.restful.NtcP2pLog;
import com.nis.domain.restful.NtcPptpLog;
import com.nis.domain.restful.NtcSshLog;
import com.nis.domain.restful.NtcSslLog;
import com.nis.domain.restful.NtcSslRecordLog;
import com.nis.domain.restful.NtcStreamingMediaLog;
import com.nis.domain.restful.NtcVoipLog;
import com.nis.domain.restful.dashboard.TrafficAsnStatistic;
@@ -159,7 +160,8 @@ public class NtcLogSearchController extends BaseRestController {
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "Dns日志检索成功", page, 0);
}
//日志 Mail 和 Mail泛收 用同一个接口
@RequestMapping(value = "/ntcMailLogs", method = RequestMethod.GET)
@ApiOperation(value = "EMAIL日志查询", httpMethod = "GET", notes = "对应配置为“邮件管理”，存储动作为阻断与监测的命中日志。对日志功能“邮件管理”提供数据基础查询服务。")
public Map<String, ?> ntcMailLogs(Page page, NtcMailLog ntcMailLog, Model model, HttpServletRequest request,
@@ -167,6 +169,10 @@ public class NtcLogSearchController extends BaseRestController {
long start = System.currentTimeMillis();
AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request,
null);
//判断searchCfgId是否为null
if(ntcMailLog.getSearchCfgId() ==null){
ntcMailLog.setSearchCfgId("-1");
}
try {
resetTime(ntcMailLog);
ntcLogService.queryConditionCheck(auditLogThread, start, ntcMailLog, NtcMailLog.class, page);
@@ -864,4 +870,34 @@ public class NtcLogSearchController extends BaseRestController {
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "ASN通联关系(源,目的)检索成功",
page, 0);
}
@RequestMapping(value = "/ntcSslRecordLogs", method = RequestMethod.GET)
@ApiOperation(value = "SSL泛收日志查询", httpMethod = "GET", notes = "对应配置为“网站管理-SSL”，存储动作为阻断与监测的命中日志。对日志功能“网站管理-SSL”提供数据基础查询服务。")
public Map<String, ?> ntcSslRecordLogs(Page page, NtcSslRecordLog ntcSslRecordLog, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis();
AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request,
null);
try {
resetTime(ntcSslRecordLog);
ntcLogService.queryConditionCheck(auditLogThread, start, ntcSslRecordLog, NtcSslRecordLog.class, page);
logDataService.getData(page, ntcSslRecordLog);
} catch (Exception e) {
auditLogThread.setExceptionInfo("SSL泛收日志检索失败:" + e.getMessage());
logger.error("SSL泛收日志检索失败：" + ExceptionUtil.getExceptionMsg(e));
if (e instanceof RestServiceException) {
throw new RestServiceException(auditLogThread, System.currentTimeMillis() - start,
"SSL泛收日志检索失败：" + e.getMessage(), ((RestServiceException) e).getErrorCode());
} else if (e instanceof ServiceRuntimeException) {
throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start,
"SSL泛收日志检索失败：" + e.getMessage(), ((ServiceRuntimeException) e).getErrorCode());
} else {
throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start,
"SSL泛收日志检索失败：" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
}
}
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "SSL泛收日志检索成功", page, 0);
}
}

35
src/main/java/com/nis/web/dao/DfLogSearchDao.xml
View File

@@ -1534,4 +1534,39 @@
</resultMap>
<resultMap id="NtcSslRecordLogMap" type="com.nis.domain.restful.NtcSslRecordLog">
<result column="cfg_id" jdbcType="BIGINT" property="cfgId" />
<result column="found_Time" jdbcType="TIMESTAMP" property="foundTime" />
<result column="recv_Time" jdbcType="TIMESTAMP" property="recvTime" />
<result column="trans_proto" jdbcType="VARCHAR" property="transProto" />
<result column="addr_type" jdbcType="INTEGER" property="addrType" />
<result column="d_ip" jdbcType="VARCHAR" property="dIp" />
<result column="s_ip" jdbcType="VARCHAR" property="sIp" />
<result column="d_port" jdbcType="VARCHAR" property="dPort" />
<result column="s_port" jdbcType="VARCHAR" property="sPort" />
<result column="service" jdbcType="INTEGER" property="service" />
<result column="entrance_id" jdbcType="BIGINT" property="entranceId" />
<result column="device_id" jdbcType="INTEGER" property="deviceId" />
<result column="direction" jdbcType="INTEGER" property="direction" />
<result column="stream_dir" jdbcType="INTEGER" property="streamDir" />
<result column="cap_ip" jdbcType="VARCHAR" property="capIp" />
<result column="addr_list" jdbcType="VARCHAR" property="addrList" />
<result column="user_region" jdbcType="VARCHAR" property="userRegion" />
<result column="server_locate" jdbcType="VARCHAR" property="serverLocate" />
<result column="client_locate" jdbcType="VARCHAR" property="clientLocate" />
<result column="s_asn" jdbcType="VARCHAR" property="sAsn" />
<result column="d_asn" jdbcType="VARCHAR" property="dAsn" />
<result column="s_subscribe_id" jdbcType="VARCHAR" property="sSubscribeId" />
<result column="d_subscribe_id" jdbcType="VARCHAR" property="dSubscribeId" />
<result column="scene_file" jdbcType="VARCHAR" property="sceneFile" />
<result column="link_id" jdbcType="BIGINT" property="linkId" />
<result column="encap_type" jdbcType="INTEGER" property="encapType" />
<result column="inner_smac" jdbcType="VARCHAR" property="innerSmac" />
<result column="inner_dmac" jdbcType="VARCHAR" property="innerDmac" />
<result column="version" jdbcType="VARCHAR" property="version" />
<result column="sni" jdbcType="VARCHAR" property="sni" />
<result column="san" jdbcType="VARCHAR" property="san" />
<result column="cn" jdbcType="VARCHAR" property="cn" />
</resultMap>
</mapper>

185
src/main/java/com/nis/web/service/LogDataService.java
View File

@@ -179,7 +179,12 @@ public class LogDataService {
orderBy = orderBy + "found_Time desc ";
}
if (Constants.ISUSECLICKHOUSE) {
getDataFromClickHouse(page, obj, tableName, className, orderBy.toLowerCase());
//判断实体类中是否存在某个字段
if(ifExistFiledName(obj,"flag")){//邮件泛收日志
getDataFromClickHouseFS(page, obj, tableName, className, orderBy.toLowerCase());
}else{
getDataFromClickHouse(page, obj, tableName, className, orderBy.toLowerCase());
}
} else {
getDataFromHive(page, obj, tableName, className, orderBy.toLowerCase());
}
@@ -316,7 +321,7 @@ public class LogDataService {
}
sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")) {
if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")|| tableName.toUpperCase().equals("TBS_ODS_NTC_COLLECT_SSL_LOG")) {
searchFromLocalCK(page, bean, sql, countSql);
} else {
searchFromDataCenter(page, bean, sql, countSql);
@@ -635,5 +640,181 @@ public class LogDataService {
Method method = bean.getClass().getMethod(methodName, classArr);
method.invoke(bean, value);
}
/**
* 利用反射查询一个对象中是否存在某个属性
*
* @param obj
* @param fieldName
* @throws Exception
*/
private static boolean ifExistFiledName(Object obj,String filedName) throws Exception{
Class class1=obj.getClass();//需要检测的类
/**
* 循环遍历所有的元素，检测有没有这个名字
*/
Field[] fields=class1.getDeclaredFields();
boolean b=false;
for (int i = 0; i < fields.length; i++) {
if(fields[i].getName().equals(filedName))
{
b=true;
break;
}
}
return b;
}
/**
* 有泛收的页面调用的方法，(邮箱泛收)
* 从clickhouse中查询数据,注意clickhouse区分大小写,目前和百分点商定都是用小写
*
* @param page 里面含有pagesize和pageno,order by
* @param bean 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名
* @param tableName 表名
* @param className 类名
* @param orderBy orderby条件
* @throws Exception
*/
private <T> void getDataFromClickHouseFS(Page<T> page, Object bean, String tableName, String className,
String orderBy) throws Exception {
tableName = tableName.toLowerCase();
String showColmun = getFiledsSql(className, page.getFields());
StringBuffer sql = new StringBuffer();
Map<String, String> filedAndColumnMap = getFiledAndColumnMap(bean.getClass());
if (null == showColmun || showColmun.equals("")) {
for (String key : filedAndColumnMap.keySet()) {
if (!filedAndColumnMap.get(key).toLowerCase().equals("id")) {
sql.append(filedAndColumnMap.get(key) + ",");
}
}
} else {
sql.append(showColmun);
}
String sqlTrim = sql.toString().trim();
if (sqlTrim.endsWith(",")) {
sqlTrim = sqlTrim.substring(0, sqlTrim.length() - 1);
}
sql.setLength(0);
sql.append(" select " + sqlTrim.toLowerCase() + " from " + tableName.toLowerCase() + " t where ");
StringBuffer whereFoundTime = new StringBuffer();
StringBuffer countSql = new StringBuffer();
countSql.append("select count(1) from " + tableName + " where ");
StringBuffer whereSB = new StringBuffer();
if (!StringUtil.isEmpty(bean)) {
Class<?> clazz = bean.getClass();
Map<String, String> filedsType = null;
filedsType = getFiledsType(bean);
for (; clazz != Object.class; clazz = clazz.getSuperclass()) {
Field[] fields = clazz.getDeclaredFields();
for (int i = 0; i < fields.length; i++) {
// 现在gwall日志表结构中只有数值和字符串两种类型,数值都是int类型没有bigint所以不需要加L,
Field f = fields[i];
String key = f.getName();// 获取字段名
if (f.getType().getName().equals("java.lang.String") && key.startsWith("search")) {
Object value = getFieldValue(bean, key);
if (!StringUtil.isEmpty(value)) {
setFieldValue(bean, key, value.toString().trim());
if (key.endsWith("Time")) {// 日期开始或结束的字段
if (col2col.containsKey(key)) {
value = sdf.parse(value.toString().trim()).getTime() / 1000;
if (col2col.get(key).get("start") != null) {
whereFoundTime.append(" and "
+ filedAndColumnMap.get(col2col.get(key).get("start")).toLowerCase()
+ ">=" + value);
} else {
whereFoundTime.append(" and "
+ filedAndColumnMap.get(col2col.get(key).get("end")).toLowerCase() + "<"
+ value);
}
}
} else {
if (key.toLowerCase().startsWith("search")) {
key = key.replace("search", "");
key = key.substring(0, 1).toLowerCase() + key.substring(1);
}
// clickhouse写法
String type = filedsType.get(key).trim();
String field = filedAndColumnMap.get(key).toLowerCase();
if (type.equals("java.lang.String")) {
if (field.contains("url") || field.equals("website")) {
whereSB.append(" and " + field + " like '"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'");
} else if (field.equals("client_locate") || field.equals("server_locate")) {
whereSB.append(" and " + field + " like '%"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'");
} else {
whereSB.append(" and " + field + "='"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'");
}
} else if (type.equals("java.lang.Integer") || type.equals("int")
|| type.equals("java.lang.Long") || type.equals("long")) {
if (field.equals("cfg_id")|| field.equals("web_id") || field.equals("app_id")
|| field.equals("proto_id")){
if(field.equals("cfg_id")){
if(value.toString().trim().equals("0")){
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " ="
+ value.toString().trim());
}else if(value.toString().trim().equals("-1")){
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " >0" );
}else{
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in("
+ value.toString().trim() + ")");
}
}else{
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in("
+ value.toString().trim() + ")");
}
}else {
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "="
+ value.toString().trim());
}
}
}
}
}
}
}
}
Integer startNum = (page.getPageNo() - 1) * page.getPageSize();
StringBuffer foundTimeSql = new StringBuffer();
foundTimeSql.append("select found_time from " + tableName + " where ");
Integer limitCount = startNum + page.getPageSize();
if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件
if (whereFoundTime.length() > 0) {
int indexOf = whereFoundTime.indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(indexOf));
foundTimeSql
.append(whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + " limit " + limitCount);
sql.append(" found_time in(" + foundTimeSql + ") ");
} else {
throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件");
}
} else {
int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(foundIndexOf));
foundTimeSql
.append(whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + " limit " + limitCount);
int indexOf = whereSB.indexOf("and") + "and".length();
sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") ");
}
sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG") ) {
searchFromLocalCK(page, bean, sql, countSql);
} else {
searchFromDataCenter(page, bean, sql, countSql);
}
}
}

4
src/main/resources/applicationLog-clickhouse.properties
View File

@@ -1,4 +1,4 @@
#由于数据中心ClickHouse日志表的表名会变动,所以本系统中将日志的表名提取到配置文件中方便后期修改,key是bean的名称+ClickHouseTable,value是ClickHouseTable的实际表名
#\u7531\u4e8e\u6570\u636e\u4e2d\u5fc3ClickHouse\u65e5\u5fd7\u8868\u7684\u8868\u540d\u4f1a\u53d8\u52a8,\u6240\u4ee5\u672c\u7cfb\u7edf\u4e2d\u5c06\u65e5\u5fd7\u7684\u8868\u540d\u63d0\u53d6\u5230\u914d\u7f6e\u6587\u4ef6\u4e2d\u65b9\u4fbf\u540e\u671f\u4fee\u6539,key\u662fbean\u7684\u540d\u79f0+ClickHouseTable,value\u662fClickHouseTable\u7684\u5b9e\u9645\u8868\u540d
NtcIpLogClickHouseTable=TBS_ODS_NTC_IP_LOG
NtcHttpLogClickHouseTable=TBS_ODS_NTC_HTTP_LOG
NtcDnsLogClickHouseTable=TBS_ODS_NTC_DNS_LOG
@@ -38,7 +38,7 @@ NtcKeywordsUrlLogClickHouseTable=TBS_ODS_NTC_KEYWORDS_URL_LOG
NtcCollectVoipLogClickHouseTable=TBS_ODS_NTC_COLLECT_VOIP_LOG
NtcCollectRadiusLogClickHouseTable=TBS_ODS_NTC_COLLECT_RADIUS_LOG
NtcConnRecordLogClickHouseTable=TBS_ODS_NTC_CONN_RECORD_LOG
NtcSslRecordLogClickHouseTable=TBS_ODS_NTC_COLLECT_SSL_LOG