diff --git a/src/main/java/com/nis/domain/restful/NtcMailLog.java b/src/main/java/com/nis/domain/restful/NtcMailLog.java index a6ed73c..09782f3 100644 --- a/src/main/java/com/nis/domain/restful/NtcMailLog.java +++ b/src/main/java/com/nis/domain/restful/NtcMailLog.java @@ -27,8 +27,19 @@ public class NtcMailLog extends LogEntity{ protected String searchMailFrom; //发件人 protected String searchMailTo; //收件人 protected String searchSubject; //主题 + + protected String flag;//泛收的标志 - public String getMailProto() { + @JsonIgnore + public String getFlag() { + return flag; + } + + public void setFlag(String flag) { + this.flag = flag; + } + + public String getMailProto() { return mailProto; } diff --git a/src/main/java/com/nis/domain/restful/NtcSslRecordLog.java b/src/main/java/com/nis/domain/restful/NtcSslRecordLog.java new file mode 100644 index 0000000..c889104 --- /dev/null +++ b/src/main/java/com/nis/domain/restful/NtcSslRecordLog.java @@ -0,0 +1,77 @@ +package com.nis.domain.restful; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.nis.domain.LogEntity; +import com.wordnik.swagger.annotations.ApiModelProperty; + +/** + * Created by darnell on 2018/6/11. + */ +public class NtcSslRecordLog extends LogEntity { + + private static final long serialVersionUID = 164434539180653065L; + + + @ApiModelProperty(value="版本信息", required=true) + protected String version; + + @ApiModelProperty(value="Server Name Indication", required=true, notes = "主要解决一台服务器只能使用一个证书-域名的缺点，通过sni可支持一台服务器为多台域名提供服务") + protected String sni; + + @ApiModelProperty(value="SAN-Subject Alternative Name Certificate", required=true, notes = "数字证书的一种形式，运行SSL证书绑定多个域名和子域名。另EV证书，扩展验证证书，只能绑定一个域名") + protected String san; + + @ApiModelProperty(value="证书授权机构", required=true) + protected String cn; + + @ApiModelProperty(value="SNI", required=true) + protected String searchSni ; + + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getSni() { + return sni; + } + + public void setSni(String sni) { + this.sni = sni; + } + + public String getSan() { + return san; + } + + public void setSan(String san) { + this.san = san; + } + + public String getCn() { + return cn; + } + + public void setCn(String cn) { + this.cn = cn; + } + + /** + * @param searchSni the searchSni to set + */ + public void setSearchSni(String searchSni) { + this.searchSni = searchSni; + } + + /** + * @return the searchSni + */ + @JsonIgnore + public String getSearchSni() { + return searchSni; + } +} diff --git a/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java b/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java index 9ac09b3..00af934 100644 --- a/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java +++ b/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java @@ -34,6 +34,7 @@ import com.nis.domain.restful.NtcP2pLog; import com.nis.domain.restful.NtcPptpLog; import com.nis.domain.restful.NtcSshLog; import com.nis.domain.restful.NtcSslLog; +import com.nis.domain.restful.NtcSslRecordLog; import com.nis.domain.restful.NtcStreamingMediaLog; import com.nis.domain.restful.NtcVoipLog; import com.nis.domain.restful.dashboard.TrafficAsnStatistic; @@ -159,7 +160,8 @@ public class NtcLogSearchController extends BaseRestController { return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "Dns日志检索成功", page, 0); } - + + //日志 Mail 和 Mail泛收 用同一个接口 @RequestMapping(value = "/ntcMailLogs", method = RequestMethod.GET) @ApiOperation(value = "EMAIL日志查询", httpMethod = "GET", notes = "对应配置为“邮件管理”，存储动作为阻断与监测的命中日志。对日志功能“邮件管理”提供数据基础查询服务。") public Map ntcMailLogs(Page page, NtcMailLog ntcMailLog, Model model, HttpServletRequest request, @@ -167,6 +169,10 @@ public class NtcLogSearchController extends BaseRestController { long start = System.currentTimeMillis(); AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, null); + //判断searchCfgId是否为null + if(ntcMailLog.getSearchCfgId() ==null){ + ntcMailLog.setSearchCfgId("-1"); + } try { resetTime(ntcMailLog); ntcLogService.queryConditionCheck(auditLogThread, start, ntcMailLog, NtcMailLog.class, page); @@ -864,4 +870,34 @@ public class NtcLogSearchController extends BaseRestController { return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "ASN通联关系(源,目的)检索成功", page, 0); } + + @RequestMapping(value = "/ntcSslRecordLogs", method = RequestMethod.GET) + @ApiOperation(value = "SSL泛收日志查询", httpMethod = "GET", notes = "对应配置为“网站管理-SSL”，存储动作为阻断与监测的命中日志。对日志功能“网站管理-SSL”提供数据基础查询服务。") + public Map ntcSslRecordLogs(Page page, NtcSslRecordLog ntcSslRecordLog, Model model, HttpServletRequest request, + HttpServletResponse response) { + long start = System.currentTimeMillis(); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); + + try { + resetTime(ntcSslRecordLog); + ntcLogService.queryConditionCheck(auditLogThread, start, ntcSslRecordLog, NtcSslRecordLog.class, page); + logDataService.getData(page, ntcSslRecordLog); + } catch (Exception e) { + auditLogThread.setExceptionInfo("SSL泛收日志检索失败:" + e.getMessage()); + logger.error("SSL泛收日志检索失败：" + ExceptionUtil.getExceptionMsg(e)); + if (e instanceof RestServiceException) { + throw new RestServiceException(auditLogThread, System.currentTimeMillis() - start, + "SSL泛收日志检索失败：" + e.getMessage(), ((RestServiceException) e).getErrorCode()); + } else if (e instanceof ServiceRuntimeException) { + throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start, + "SSL泛收日志检索失败：" + e.getMessage(), ((ServiceRuntimeException) e).getErrorCode()); + } else { + throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start, + "SSL泛收日志检索失败：" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue()); + } + } + + return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "SSL泛收日志检索成功", page, 0); + } } diff --git a/src/main/java/com/nis/web/dao/DfLogSearchDao.xml b/src/main/java/com/nis/web/dao/DfLogSearchDao.xml index 2782054..ac25120 100644 --- a/src/main/java/com/nis/web/dao/DfLogSearchDao.xml +++ b/src/main/java/com/nis/web/dao/DfLogSearchDao.xml @@ -1534,4 +1534,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/main/java/com/nis/web/service/LogDataService.java b/src/main/java/com/nis/web/service/LogDataService.java index b37b4cb..c51d49d 100644 --- a/src/main/java/com/nis/web/service/LogDataService.java +++ b/src/main/java/com/nis/web/service/LogDataService.java @@ -179,7 +179,12 @@ public class LogDataService { orderBy = orderBy + "found_Time desc "; } if (Constants.ISUSECLICKHOUSE) { - getDataFromClickHouse(page, obj, tableName, className, orderBy.toLowerCase()); + //判断实体类中是否存在某个字段 + if(ifExistFiledName(obj,"flag")){//邮件泛收日志 + getDataFromClickHouseFS(page, obj, tableName, className, orderBy.toLowerCase()); + }else{ + getDataFromClickHouse(page, obj, tableName, className, orderBy.toLowerCase()); + } } else { getDataFromHive(page, obj, tableName, className, orderBy.toLowerCase()); } @@ -316,7 +321,7 @@ public class LogDataService { } sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 - if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")) { + if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")|| tableName.toUpperCase().equals("TBS_ODS_NTC_COLLECT_SSL_LOG")) { searchFromLocalCK(page, bean, sql, countSql); } else { searchFromDataCenter(page, bean, sql, countSql); @@ -635,5 +640,181 @@ public class LogDataService { Method method = bean.getClass().getMethod(methodName, classArr); method.invoke(bean, value); } + + /** + * 利用反射查询一个对象中是否存在某个属性 + * + * @param obj + * @param fieldName + * @throws Exception + */ + private static boolean ifExistFiledName(Object obj,String filedName) throws Exception{ + Class class1=obj.getClass();//需要检测的类 + /** + * 循环遍历所有的元素，检测有没有这个名字 + */ + Field[] fields=class1.getDeclaredFields(); + + boolean b=false; + for (int i = 0; i < fields.length; i++) { + if(fields[i].getName().equals(filedName)) + { + b=true; + break; + } + } + return b; + } + + /** + * 有泛收的页面调用的方法，(邮箱泛收) + * 从clickhouse中查询数据,注意clickhouse区分大小写,目前和百分点商定都是用小写 + * + * @param page 里面含有pagesize和pageno,order by + * @param bean 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名 + * @param tableName 表名 + * @param className 类名 + * @param orderBy orderby条件 + * @throws Exception + */ + private void getDataFromClickHouseFS(Page page, Object bean, String tableName, String className, + String orderBy) throws Exception { + tableName = tableName.toLowerCase(); + String showColmun = getFiledsSql(className, page.getFields()); + StringBuffer sql = new StringBuffer(); + Map filedAndColumnMap = getFiledAndColumnMap(bean.getClass()); + if (null == showColmun || showColmun.equals("")) { + for (String key : filedAndColumnMap.keySet()) { + if (!filedAndColumnMap.get(key).toLowerCase().equals("id")) { + sql.append(filedAndColumnMap.get(key) + ","); + } + } + } else { + sql.append(showColmun); + } + String sqlTrim = sql.toString().trim(); + if (sqlTrim.endsWith(",")) { + sqlTrim = sqlTrim.substring(0, sqlTrim.length() - 1); + } + sql.setLength(0); + sql.append(" select " + sqlTrim.toLowerCase() + " from " + tableName.toLowerCase() + " t where "); + StringBuffer whereFoundTime = new StringBuffer(); + StringBuffer countSql = new StringBuffer(); + countSql.append("select count(1) from " + tableName + " where "); + StringBuffer whereSB = new StringBuffer(); + if (!StringUtil.isEmpty(bean)) { + Class clazz = bean.getClass(); + Map filedsType = null; + filedsType = getFiledsType(bean); + for (; clazz != Object.class; clazz = clazz.getSuperclass()) { + Field[] fields = clazz.getDeclaredFields(); + for (int i = 0; i < fields.length; i++) { + // 现在gwall日志表结构中只有数值和字符串两种类型,数值都是int类型没有bigint所以不需要加L, + Field f = fields[i]; + String key = f.getName();// 获取字段名 + if (f.getType().getName().equals("java.lang.String") && key.startsWith("search")) { + Object value = getFieldValue(bean, key); + if (!StringUtil.isEmpty(value)) { + setFieldValue(bean, key, value.toString().trim()); + if (key.endsWith("Time")) {// 日期开始或结束的字段 + if (col2col.containsKey(key)) { + value = sdf.parse(value.toString().trim()).getTime() / 1000; + if (col2col.get(key).get("start") != null) { + whereFoundTime.append(" and " + + filedAndColumnMap.get(col2col.get(key).get("start")).toLowerCase() + + ">=" + value); + } else { + whereFoundTime.append(" and " + + filedAndColumnMap.get(col2col.get(key).get("end")).toLowerCase() + "<" + + value); + } + } + } else { + if (key.toLowerCase().startsWith("search")) { + key = key.replace("search", ""); + key = key.substring(0, 1).toLowerCase() + key.substring(1); + } + // clickhouse写法 + String type = filedsType.get(key).trim(); + String field = filedAndColumnMap.get(key).toLowerCase(); + if (type.equals("java.lang.String")) { + + if (field.contains("url") || field.equals("website")) { + whereSB.append(" and " + field + " like '" + + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); + } else if (field.equals("client_locate") || field.equals("server_locate")) { + whereSB.append(" and " + field + " like '%" + + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); + } else { + whereSB.append(" and " + field + "='" + + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'"); + } + + } else if (type.equals("java.lang.Integer") || type.equals("int") + || type.equals("java.lang.Long") || type.equals("long")) { + if (field.equals("cfg_id")|| field.equals("web_id") || field.equals("app_id") + || field.equals("proto_id")){ + if(field.equals("cfg_id")){ + if(value.toString().trim().equals("0")){ + whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " =" + + value.toString().trim()); + }else if(value.toString().trim().equals("-1")){ + whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " >0" ); + }else{ + whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in(" + + value.toString().trim() + ")"); + } + }else{ + whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in(" + + value.toString().trim() + ")"); + } + }else { + whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "=" + + value.toString().trim()); + } + + } + + } + } + + } + } + + } + } + Integer startNum = (page.getPageNo() - 1) * page.getPageSize(); + StringBuffer foundTimeSql = new StringBuffer(); + foundTimeSql.append("select found_time from " + tableName + " where "); + Integer limitCount = startNum + page.getPageSize(); + if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件 + if (whereFoundTime.length() > 0) { + int indexOf = whereFoundTime.indexOf("and") + "and".length(); + countSql.append(whereFoundTime.substring(indexOf)); + + foundTimeSql + .append(whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + " limit " + limitCount); + sql.append(" found_time in(" + foundTimeSql + ") "); + } else { + throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件"); + } + } else { + int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length(); + countSql.append(whereFoundTime.substring(foundIndexOf)); + foundTimeSql + .append(whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + " limit " + limitCount); + + int indexOf = whereSB.indexOf("and") + "and".length(); + sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") "); + } + sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 + + if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG") ) { + searchFromLocalCK(page, bean, sql, countSql); + } else { + searchFromDataCenter(page, bean, sql, countSql); + } + + } } diff --git a/src/main/resources/applicationLog-clickhouse.properties b/src/main/resources/applicationLog-clickhouse.properties index 9845875..6b69dac 100644 --- a/src/main/resources/applicationLog-clickhouse.properties +++ b/src/main/resources/applicationLog-clickhouse.properties @@ -1,4 +1,4 @@ -#由于数据中心ClickHouse日志表的表名会变动,所以本系统中将日志的表名提取到配置文件中方便后期修改,key是bean的名称+ClickHouseTable,value是ClickHouseTable的实际表名 +#\u7531\u4e8e\u6570\u636e\u4e2d\u5fc3ClickHouse\u65e5\u5fd7\u8868\u7684\u8868\u540d\u4f1a\u53d8\u52a8,\u6240\u4ee5\u672c\u7cfb\u7edf\u4e2d\u5c06\u65e5\u5fd7\u7684\u8868\u540d\u63d0\u53d6\u5230\u914d\u7f6e\u6587\u4ef6\u4e2d\u65b9\u4fbf\u540e\u671f\u4fee\u6539,key\u662fbean\u7684\u540d\u79f0+ClickHouseTable,value\u662fClickHouseTable\u7684\u5b9e\u9645\u8868\u540d NtcIpLogClickHouseTable=TBS_ODS_NTC_IP_LOG NtcHttpLogClickHouseTable=TBS_ODS_NTC_HTTP_LOG NtcDnsLogClickHouseTable=TBS_ODS_NTC_DNS_LOG @@ -38,7 +38,7 @@ NtcKeywordsUrlLogClickHouseTable=TBS_ODS_NTC_KEYWORDS_URL_LOG NtcCollectVoipLogClickHouseTable=TBS_ODS_NTC_COLLECT_VOIP_LOG NtcCollectRadiusLogClickHouseTable=TBS_ODS_NTC_COLLECT_RADIUS_LOG NtcConnRecordLogClickHouseTable=TBS_ODS_NTC_CONN_RECORD_LOG - +NtcSslRecordLogClickHouseTable=TBS_ODS_NTC_COLLECT_SSL_LOG