diff --git a/src/main/java/com/nis/web/service/LogDataService.java b/src/main/java/com/nis/web/service/LogDataService.java index 69ab607..00b8a1c 100644 --- a/src/main/java/com/nis/web/service/LogDataService.java +++ b/src/main/java/com/nis/web/service/LogDataService.java @@ -37,7 +37,8 @@ import com.zdjizhi.utils.StringUtil; */ @Service public class LogDataService { - private final static Logger logger = LoggerFactory.getLogger(LogDataService.class); + private final static Logger logger = LoggerFactory + .getLogger(LogDataService.class); @Autowired private LogJDBCByDruid logJDBCByDruid; @@ -46,7 +47,8 @@ public class LogDataService { private IspInfoDao ispInfoDao; @Autowired private LocalLogJDBCByDruid localLogJDBCByDruid; - private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private static SimpleDateFormat sdf = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); private static SimpleDateFormat sdf2 = new SimpleDateFormat("yyyyMMdd"); private static Map> col2col = new HashMap>(); static { @@ -58,11 +60,15 @@ public class LogDataService { col2col.put("searchFoundEndTime", endMap); } - public NtcConnRecordPercent getNtcConnRecordPercent(NtcConnRecordPercent ntcConnRecordPercent) throws Exception { - long startTime = sdf.parse(ntcConnRecordPercent.getSearchFoundStartTime().toString().trim()).getTime() / 1000; - long endTime = sdf.parse(ntcConnRecordPercent.getSearchFoundEndTime().toString().trim()).getTime() / 1000; + public NtcConnRecordPercent getNtcConnRecordPercent( + NtcConnRecordPercent ntcConnRecordPercent) throws Exception { + long startTime = sdf.parse( + ntcConnRecordPercent.getSearchFoundStartTime().toString().trim()).getTime() / 1000; + long endTime = sdf.parse( + ntcConnRecordPercent.getSearchFoundEndTime().toString().trim()).getTime() / 1000; if (endTime - startTime < 0) { - throw new RestServiceException("searchFoundStartTime() can not exceed searchFoundEndTime", + throw new RestServiceException( + "searchFoundStartTime() can not exceed searchFoundEndTime", RestBusinessCode.param_formate_error.getValue()); } long second = endTime - startTime; @@ -71,8 +77,7 @@ public class LogDataService { sql.append(second); sql.append(" AS pps , SUM(c2s_byte_num + s2c_byte_num)*8/1024/1024/1024/"); sql.append(second); - sql.append( - " AS bps FROM tbs_ods_ntc_conn_record_log t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log WHERE found_time >= "); + sql.append(" AS bps FROM tbs_ods_ntc_conn_record_log t WHERE found_time>= "); sql.append(startTime); sql.append(" and found_time< "); sql.append(endTime); @@ -80,25 +85,32 @@ public class LogDataService { totalSql.append(sql); if (ntcConnRecordPercent.getSearchEntranceId() != null && !ntcConnRecordPercent.getSearchEntranceId().trim().equals("")) { - totalSql.append(" and stream_dir in(1,2,3) and entrance_id =" + ntcConnRecordPercent.getSearchEntranceId() - + " ) and stream_dir in(1,2,3) and entrance_id =" + ntcConnRecordPercent.getSearchEntranceId()); - sql.append(" and stream_dir in(1,2) and entrance_id =" + ntcConnRecordPercent.getSearchEntranceId() - + " ) and stream_dir in(1,2) and entrance_id =" + ntcConnRecordPercent.getSearchEntranceId()); + totalSql.append(" and stream_dir in(1,2,3) and entrance_id =" + + ntcConnRecordPercent.getSearchEntranceId()); + sql.append(" and stream_dir in(1,2) and entrance_id =" + + ntcConnRecordPercent.getSearchEntranceId()); } else { - totalSql.append(" and stream_dir in(1,2,3)) and stream_dir in(1,2,3)"); - sql.append(" and stream_dir in(1,2) ) and stream_dir in(1,2) "); + totalSql.append(" and stream_dir in(1,2,3) "); + sql.append(" and stream_dir in(1,2) "); } - - localLogJDBCByDruid.getNtcConnRecordPercentData(ntcConnRecordPercent, sql.toString(), false); - localLogJDBCByDruid.getNtcConnRecordPercentData(ntcConnRecordPercent, totalSql.toString(), true); + localLogJDBCByDruid.getNtcConnRecordPercentData(ntcConnRecordPercent, + sql.toString(), false); + localLogJDBCByDruid.getNtcConnRecordPercentData(ntcConnRecordPercent, + totalSql.toString(), true); return ntcConnRecordPercent; } - public void getNtcAsnRecord(Page page, TrafficAsnStatistic ntcAsnRecord) throws Exception { - long startTime = sdf.parse(ntcAsnRecord.getSearchFoundStartTime().toString().trim()).getTime() / 1000; - long endTime = sdf.parse(ntcAsnRecord.getSearchFoundEndTime().toString().trim()).getTime() / 1000; + public void getNtcAsnRecord(Page page, TrafficAsnStatistic ntcAsnRecord) + throws Exception { + long startTime = sdf.parse( + ntcAsnRecord.getSearchFoundStartTime().toString().trim()) + .getTime() / 1000; + long endTime = sdf.parse( + ntcAsnRecord.getSearchFoundEndTime().toString().trim()) + .getTime() / 1000; if (endTime - startTime < 0) { - throw new RestServiceException("searchFoundStartTime() can not exceed searchFoundEndTime", + throw new RestServiceException( + "searchFoundStartTime() can not exceed searchFoundEndTime", RestBusinessCode.param_formate_error.getValue()); } @@ -121,17 +133,17 @@ public class LogDataService { sql.append(" and found_time< "); sql.append(endTime); -// sql.append(" IN ( SELECT found_time FROM tbs_ods_ntc_conn_record_log WHERE found_time >= "); -// sql.append(startTime); -// sql.append(" and found_time< "); -// sql.append(endTime); + // sql.append(" IN ( SELECT found_time FROM tbs_ods_ntc_conn_record_log WHERE found_time >= "); + // sql.append(startTime); + // sql.append(" and found_time< "); + // sql.append(endTime); if (searchAsn != null && !searchAsn.equals("")) { sql.append(" and " + asn + "='" + searchAsn + "'"); } sql.append(" group by " + asn); -// countSql.append(" ) "); + // countSql.append(" ) "); StringBuffer countSql = new StringBuffer(); countSql.append("select count(1) from ("); @@ -139,9 +151,10 @@ public class LogDataService { countSql.append(")"); Integer startNum = (page.getPageNo() - 1) * page.getPageSize(); -// Integer limitCount = startNum + page.getPageSize(); -// sql.append(" limit " + limitCount + " ) "); - sql.append(" order by bps desc,pps desc limit " + startNum + "," + page.getPageSize()); + // Integer limitCount = startNum + page.getPageSize(); + // sql.append(" limit " + limitCount + " ) "); + sql.append(" order by bps desc,pps desc limit " + startNum + "," + + page.getPageSize()); localLogJDBCByDruid.getCount(page, countSql.toString()); if (page.getCount() > 0) { page.setList(localLogJDBCByDruid.getNtcAsnRecordData(sql.toString())); @@ -178,19 +191,23 @@ public class LogDataService { } String orderBy = " order by "; if (null != page.getOrderBy() && !page.getOrderBy().equals("")) { - orderBy = orderBy + Page.getOrderBySql(className, page.getOrderBy()); + orderBy = orderBy + + Page.getOrderBySql(className, page.getOrderBy()); } else { orderBy = orderBy + "found_Time desc "; } if (Constants.ISUSECLICKHOUSE) { // 判断实体类中是否存在某个字段 if (ifExistFiledName(obj, "flag")) {// 邮件泛收日志 - getDataFromClickHouseFS(page, obj, tableName, className, orderBy.toLowerCase(), getIspNum(obj)); + getDataFromClickHouseFS(page, obj, tableName, className, + orderBy.toLowerCase(), getIspNum(obj)); } else { - getDataFromClickHouse(page, obj, tableName, className, orderBy.toLowerCase(), getIspNum(obj)); + getDataFromClickHouse(page, obj, tableName, className, + orderBy.toLowerCase(), getIspNum(obj)); } } else { - getDataFromHive(page, obj, tableName, className, orderBy.toLowerCase()); + getDataFromHive(page, obj, tableName, className, + orderBy.toLowerCase()); } } @@ -214,19 +231,26 @@ public class LogDataService { /** * 从clickhouse中查询数据,注意clickhouse区分大小写,目前和百分点商定都是用小写 * - * @param page 里面含有pagesize和pageno,order by - * @param bean 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名 - * @param tableName 表名 - * @param className 类名 - * @param orderBy orderby条件 + * @param page + * 里面含有pagesize和pageno,order by + * @param bean + * 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名 + * @param tableName + * 表名 + * @param className + * 类名 + * @param orderBy + * orderby条件 * @throws Exception */ - private void getDataFromClickHouse(Page page, Object bean, String tableName, String className, - String orderBy, String ispNum) throws Exception { + private void getDataFromClickHouse(Page page, Object bean, + String tableName, String className, String orderBy, String ispNum) + throws Exception { tableName = tableName.toLowerCase(); String showColmun = getFiledsSql(className, page.getFields()); StringBuffer sql = new StringBuffer(); - Map filedAndColumnMap = getFiledAndColumnMap(bean.getClass()); + Map filedAndColumnMap = getFiledAndColumnMap(bean + .getClass()); if (null == showColmun || showColmun.equals("")) { for (String key : filedAndColumnMap.keySet()) { if (!filedAndColumnMap.get(key).toLowerCase().equals("id")) { @@ -246,15 +270,16 @@ public class LogDataService { StringBuffer countSql = new StringBuffer(); if (ispNum != null) { - sql.append(" select " + sqlTrim.toLowerCase() - + ", concat(toString(entrance_id),toString(device_id)) as ispNum from " + tableName.toLowerCase() - + " t where "); - countSql.append( - "select count(1) from (select concat(toString(entrance_id),toString(device_id)) as ispNum from " - + tableName + " where "); + sql.append(" select " + + sqlTrim.toLowerCase() + + ", concat(toString(entrance_id),toString(device_id)) as ispNum from " + + tableName.toLowerCase() + " t where "); + countSql.append("select count(1) from (select concat(toString(entrance_id),toString(device_id)) as ispNum from " + + tableName + " where "); } else { - sql.append(" select " + sqlTrim.toLowerCase() + " from " + tableName.toLowerCase() + " t where "); + sql.append(" select " + sqlTrim.toLowerCase() + " from " + + tableName.toLowerCase() + " t where "); countSql.append("select count(1) from " + tableName + " where "); } @@ -269,20 +294,28 @@ public class LogDataService { // 现在gwall日志表结构中只有数值和字符串两种类型,数值都是int类型没有bigint所以不需要加L, Field f = fields[i]; String key = f.getName();// 获取字段名 - if (f.getType().getName().equals("java.lang.String") && key.startsWith("search")) { + if (f.getType().getName().equals("java.lang.String") + && key.startsWith("search")) { Object value = getFieldValue(bean, key); if (!StringUtil.isEmpty(value)) { setFieldValue(bean, key, value.toString().trim()); if (key.endsWith("Time")) {// 日期开始或结束的字段 if (col2col.containsKey(key)) { - value = sdf.parse(value.toString().trim()).getTime() / 1000; + value = sdf.parse(value.toString().trim()) + .getTime() / 1000; if (col2col.get(key).get("start") != null) { whereFoundTime.append(" and " - + filedAndColumnMap.get(col2col.get(key).get("start")).toLowerCase() - + ">=" + value); + + filedAndColumnMap.get( + col2col.get(key).get( + "start")) + .toLowerCase() + ">=" + + value); } else { whereFoundTime.append(" and " - + filedAndColumnMap.get(col2col.get(key).get("end")).toLowerCase() + "<" + + filedAndColumnMap.get( + col2col.get(key).get( + "end")) + .toLowerCase() + "<" + value); } } @@ -290,32 +323,68 @@ public class LogDataService { if (key != "searchIspCode") {// 将运营商的查询条件排除在外 if (key.toLowerCase().startsWith("search")) { key = key.replace("search", ""); - key = key.substring(0, 1).toLowerCase() + key.substring(1); + key = key.substring(0, 1).toLowerCase() + + key.substring(1); } // clickhouse写法 String type = filedsType.get(key).trim(); - String field = filedAndColumnMap.get(key).toLowerCase(); + String field = filedAndColumnMap.get(key) + .toLowerCase(); if (type.equals("java.lang.String")) { - if (field.contains("url") || field.equals("website")) { - whereSB.append(" and " + field + " like '" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); - } else if (field.equals("client_locate") || field.equals("server_locate")) { - whereSB.append(" and " + field + " like '%" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); + if (field.contains("url") + || field.equals("website")) { + whereSB.append(" and " + + field + + " like '" + + StringEscapeUtils + .unescapeHtml4(value + .toString() + .trim()) + + "%'"); + } else if (field + .equals("client_locate") + || field.equals("server_locate")) { + whereSB.append(" and " + + field + + " like '%" + + StringEscapeUtils + .unescapeHtml4(value + .toString() + .trim()) + + "%'"); } else { - whereSB.append(" and " + field + "='" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'"); + whereSB.append(" and " + + field + + "='" + + StringEscapeUtils + .unescapeHtml4(value + .toString() + .trim()) + + "'"); } - } else if (type.equals("java.lang.Integer") || type.equals("int") - || type.equals("java.lang.Long") || type.equals("long")) { - if (field.equals("cfg_id") || field.equals("web_id") || field.equals("app_id") + } else if (type.equals("java.lang.Integer") + || type.equals("int") + || type.equals("java.lang.Long") + || type.equals("long")) { + if (field.equals("cfg_id") + || field.equals("web_id") + || field.equals("app_id") || field.equals("proto_id")) { - whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in(" - + value.toString().trim() + ")"); + whereSB.append(" and " + + filedAndColumnMap + .get(key) + .toLowerCase() + + " in(" + + value.toString().trim() + + ")"); } else { - whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "=" + whereSB.append(" and " + + filedAndColumnMap + .get(key) + .toLowerCase() + + "=" + value.toString().trim()); } } @@ -330,53 +399,68 @@ public class LogDataService { } Integer startNum = (page.getPageNo() - 1) * page.getPageSize(); StringBuffer foundTimeSql = new StringBuffer(); - foundTimeSql.append("select found_time from " + tableName + " where "); + foundTimeSql + .append("select found_time from " + tableName + " where "); Integer limitCount = startNum + page.getPageSize(); if (ispNum != null) { if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件 if (whereFoundTime.length() > 0) { - int indexOf = whereFoundTime.indexOf("and") + "and".length(); - countSql.append(whereFoundTime.substring(indexOf) + " and ispNum in(" + ispNum + "))"); - sql.append(whereFoundTime.substring(indexOf) + " and ispNum in(" + ispNum + ")"); + int indexOf = whereFoundTime.indexOf("and") + + "and".length(); + countSql.append(whereFoundTime.substring(indexOf) + + " and ispNum in(" + ispNum + "))"); + sql.append(whereFoundTime.substring(indexOf) + + " and ispNum in(" + ispNum + ")"); } else { - throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件"); + throw new RuntimeException("从clickhouse的" + tableName + + "表查询时,必须要有一个where条件"); } } else { - int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length(); - countSql.append(whereFoundTime.substring(foundIndexOf) + " and ispNum in(" + ispNum + "))"); + int foundIndexOf = whereFoundTime.append(whereSB) + .indexOf("and") + "and".length(); + countSql.append(whereFoundTime.substring(foundIndexOf) + + " and ispNum in(" + ispNum + "))"); int indexOf = whereSB.indexOf("and") + "and".length(); - sql.append(whereSB.substring(indexOf) + whereFoundTime + " and ispNum in(" + ispNum + ")"); + sql.append(whereSB.substring(indexOf) + whereFoundTime + + " and ispNum in(" + ispNum + ")"); } - sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 + sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + + page.getPageSize());// clickhouse的分页与mysql相同 } else { if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件 if (whereFoundTime.length() > 0) { - int indexOf = whereFoundTime.indexOf("and") + "and".length(); + int indexOf = whereFoundTime.indexOf("and") + + "and".length(); countSql.append(whereFoundTime.substring(indexOf)); - foundTimeSql.append( - whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + " limit " + limitCount); + foundTimeSql.append(whereFoundTime.substring(indexOf) + + orderBy.toLowerCase() + " limit " + limitCount); sql.append(" found_time in(" + foundTimeSql + ") "); } else { - throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件"); + throw new RuntimeException("从clickhouse的" + tableName + + "表查询时,必须要有一个where条件"); } } else { - int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length(); + int foundIndexOf = whereFoundTime.append(whereSB) + .indexOf("and") + "and".length(); countSql.append(whereFoundTime.substring(foundIndexOf)); - foundTimeSql.append( - whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + " limit " + limitCount); + foundTimeSql.append(whereFoundTime.substring(foundIndexOf) + + orderBy.toLowerCase() + " limit " + limitCount); int indexOf = whereSB.indexOf("and") + "and".length(); - sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") "); + sql.append(whereSB.substring(indexOf) + " and found_time in(" + + foundTimeSql + ") "); } - sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 + sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + + page.getPageSize());// clickhouse的分页与mysql相同 } if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG") - || tableName.toUpperCase().equals("TBS_ODS_NTC_COLLECT_SSL_LOG")) { + || tableName.toUpperCase() + .equals("TBS_ODS_NTC_COLLECT_SSL_LOG")) { searchFromLocalCK(page, bean, sql, countSql); } else { searchFromDataCenter(page, bean, sql, countSql); @@ -387,19 +471,26 @@ public class LogDataService { /** * 从hive中查询数据 * - * @param page 里面含有pagesize和pageno,order by - * @param bean 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名 - * @param tableName 表名 - * @param className 类名 - * @param orderBy orderby条件 + * @param page + * 里面含有pagesize和pageno,order by + * @param bean + * 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名 + * @param tableName + * 表名 + * @param className + * 类名 + * @param orderBy + * orderby条件 * @throws Exception */ - private void getDataFromHive(Page page, Object bean, String tableName, String className, String orderBy) + private void getDataFromHive(Page page, Object bean, + String tableName, String className, String orderBy) throws Exception { tableName = tableName.toLowerCase(); String showColmun = getFiledsSql(className, page.getFields()); StringBuffer sql = new StringBuffer(); - Map filedAndColumnMap = getFiledAndColumnMap(bean.getClass()); + Map filedAndColumnMap = getFiledAndColumnMap(bean + .getClass()); if (null == showColmun || showColmun.equals("")) { for (String key : filedAndColumnMap.keySet()) { if (!filedAndColumnMap.get(key).toLowerCase().equals("id")) { @@ -414,9 +505,11 @@ public class LogDataService { sqlTrim = sqlTrim.substring(0, sqlTrim.length() - 1); } sql.setLength(0); - sql.append(" select " + sqlTrim.toLowerCase() + " from (select " + sqlTrim.toLowerCase() - + ",row_number() over(partition by found_time_partition " + orderBy + ") as row_num from " - + tableName.toLowerCase() + " "); + sql.append(" select " + sqlTrim.toLowerCase() + " from (select " + + sqlTrim.toLowerCase() + + ",row_number() over(partition by found_time_partition " + + orderBy + ") as row_num from " + tableName.toLowerCase() + + " "); StringBuffer countSql = new StringBuffer(); countSql.append("select count(1) from " + tableName + " "); @@ -434,18 +527,24 @@ public class LogDataService { Field f = fields[i]; String key = f.getName();// 获取字段名 String typeName = f.getType().getName(); - if (f.getType().getName().equals("java.lang.String") && key.startsWith("search")) { + if (f.getType().getName().equals("java.lang.String") + && key.startsWith("search")) { Object value = getFieldValue(bean, key); if (!StringUtil.isEmpty(value)) { setFieldValue(bean, key, value.toString().trim()); if (key.endsWith("Time")) {// 日期开始或结束的字段 if (col2col.containsKey(key)) { - Long partition = Long.parseLong(sdf2.format(sdf.parse(value.toString().trim()))); - value = sdf.parse(value.toString().trim()).getTime() / 1000; - if (key.toLowerCase().equals("searchfoundstarttime")) { + Long partition = Long.parseLong(sdf2 + .format(sdf.parse(value.toString() + .trim()))); + value = sdf.parse(value.toString().trim()) + .getTime() / 1000; + if (key.toLowerCase().equals( + "searchfoundstarttime")) { foundTimePartStart = partition; } - if (key.toLowerCase().equals("searchfoundendtime")) { + if (key.toLowerCase().equals( + "searchfoundendtime")) { foundTimePartEnd = partition; } if (col2col.get(key).get("start") != null) { @@ -455,8 +554,11 @@ public class LogDataService { // value.toString().trim() // + "','yyyy-mm-dd HH24:mi:ss')"); whereSB.append(" and " - + filedAndColumnMap.get(col2col.get(key).get("start")).toLowerCase() - + ">=" + value); + + filedAndColumnMap.get( + col2col.get(key).get( + "start")) + .toLowerCase() + ">=" + + value); } else { // sql.append(" and " + // filedAndColumnMap.get(col2col.get(key).get("end")) @@ -464,32 +566,51 @@ public class LogDataService { // value.toString().trim() // + "','yyyy-mm-dd HH24:mi:ss')"); whereSB.append(" and " - + filedAndColumnMap.get(col2col.get(key).get("end")).toLowerCase() + "<" + + filedAndColumnMap.get( + col2col.get(key).get( + "end")) + .toLowerCase() + "<" + value); } } } else { if (key.toLowerCase().startsWith("search")) { key = key.replace("search", ""); - key = key.substring(0, 1).toLowerCase() + key.substring(1); + key = key.substring(0, 1).toLowerCase() + + key.substring(1); } if (typeName.equals("java.lang.String")) { String field = filedAndColumnMap.get(key); - if (field.contains("url") || field.equals("website")) { - whereSB.append(" and " + field + " like '" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); + if (field.contains("url") + || field.equals("website")) { + whereSB.append(" and " + + field + + " like '" + + StringEscapeUtils + .unescapeHtml4(value + .toString() + .trim()) + "%'"); } else { - whereSB.append(" and " + field + "='" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'"); + whereSB.append(" and " + + field + + "='" + + StringEscapeUtils + .unescapeHtml4(value + .toString() + .trim()) + "'"); } - } else if (typeName.equals("java.lang.Integer") || typeName.equals("int")) { - whereSB.append( - " and " + filedAndColumnMap.get(key) + "=" + value.toString().trim()); + } else if (typeName.equals("java.lang.Integer") + || typeName.equals("int")) { + whereSB.append(" and " + + filedAndColumnMap.get(key) + "=" + + value.toString().trim()); - } else if (typeName.equals("java.lang.Long") || typeName.equals("long")) { - whereSB.append( - " and " + filedAndColumnMap.get(key) + "=" + value.toString().trim() + "L"); + } else if (typeName.equals("java.lang.Long") + || typeName.equals("long")) { + whereSB.append(" and " + + filedAndColumnMap.get(key) + "=" + + value.toString().trim() + "L"); } } @@ -498,12 +619,16 @@ public class LogDataService { } } if (null != foundTimePartStart) { - // sql.append(" and found_time_partition>=" + foundTimePartStart + "L"); - whereSB.append(" and found_time_partition>=" + foundTimePartStart); + // sql.append(" and found_time_partition>=" + + // foundTimePartStart + "L"); + whereSB.append(" and found_time_partition>=" + + foundTimePartStart); } if (null != foundTimePartEnd) { - // sql.append(" and found_time_partition<" + foundTimePartEnd + "L"); - whereSB.append(" and found_time_partition<=" + foundTimePartEnd); + // sql.append(" and found_time_partition<" + + // foundTimePartEnd + "L"); + whereSB.append(" and found_time_partition<=" + + foundTimePartEnd); } } @@ -516,7 +641,8 @@ public class LogDataService { } Integer startNum = (page.getPageNo() - 1) * page.getPageSize() + 1; Integer endNum = startNum - 1 + page.getPageSize(); - sql.append(" ) t where row_Num between " + startNum + " and " + endNum); + sql.append(" ) t where row_Num between " + startNum + " and " + + endNum); searchFromDataCenter(page, bean, sql, countSql); } @@ -529,13 +655,14 @@ public class LogDataService { * @param countSql * @throws Exception */ - private void searchFromDataCenter(Page page, Object bean, StringBuffer selSql, StringBuffer countSql) - throws Exception { -// if (Constants.ISOPENLOGCOUNTANDLAST) { + private void searchFromDataCenter(Page page, Object bean, + StringBuffer selSql, StringBuffer countSql) throws Exception { + // if (Constants.ISOPENLOGCOUNTANDLAST) { logJDBCByDruid.getCount(page, countSql.toString()); -// } + // } if (page.getCount() > 0) { - logJDBCByDruid.getTableData(page, selSql.toString(), bean.getClass()); + logJDBCByDruid.getTableData(page, selSql.toString(), + bean.getClass()); if (page.getLast() > 100) { page.setLast(100); } @@ -545,13 +672,14 @@ public class LogDataService { } - private void searchFromLocalCK(Page page, Object bean, StringBuffer selSql, StringBuffer countSql) - throws Exception { -// if (Constants.ISOPENLOGCOUNTANDLAST) { + private void searchFromLocalCK(Page page, Object bean, + StringBuffer selSql, StringBuffer countSql) throws Exception { + // if (Constants.ISOPENLOGCOUNTANDLAST) { localLogJDBCByDruid.getCount(page, countSql.toString()); -// } + // } if (page.getCount() > 0) { - localLogJDBCByDruid.getTableData(page, selSql.toString(), bean.getClass()); + localLogJDBCByDruid.getTableData(page, selSql.toString(), + bean.getClass()); if (page.getLast() > 100) { page.setLast(100); } @@ -572,10 +700,12 @@ public class LogDataService { Field[] superfields = obj.getClass().getDeclaredFields(); Map infoMap = new HashMap(); for (int i = 0; i < fields.length; i++) { - infoMap.put(fields[i].getName(), fields[i].getType().toString().replace("class", "")); + infoMap.put(fields[i].getName(), fields[i].getType().toString() + .replace("class", "")); } for (int i = 0; i < superfields.length; i++) { - infoMap.put(superfields[i].getName(), superfields[i].getType().toString().replace("class", "")); + infoMap.put(superfields[i].getName(), superfields[i].getType() + .toString().replace("class", "")); } return infoMap; } @@ -584,11 +714,13 @@ public class LogDataService { * 将fileds中的字段根据DfLogSearchDao.xml中对应的resultMap转换为数据库中的字段 * * @param mapName - * @param fileds与界面商定好的是传日志类中的对象名(界面没有表结构不知道对象属性对应的数据库字段名称是什么),不是数据库中的字段名 + * @param fileds与界面商定好的是传日志类中的对象名 + * (界面没有表结构不知道对象属性对应的数据库字段名称是什么),不是数据库中的字段名 * @return * @throws Exception */ - private static String getFiledsSql(String mapName, String fileds) throws Exception { + private static String getFiledsSql(String mapName, String fileds) + throws Exception { if (!StringUtil.isBlank(fileds)) { String[] fieldsColoumn = null; // 所有字段名 @@ -602,8 +734,10 @@ public class LogDataService { // 从resultMap中获取字段名称和属性名称 if (fieldsColoumn != null) { - SqlSessionFactory sqlSessionFactory = SpringContextHolder.getBean(SqlSessionFactory.class); - ResultMap map = sqlSessionFactory.getConfiguration().getResultMap(mapName + "Map"); + SqlSessionFactory sqlSessionFactory = SpringContextHolder + .getBean(SqlSessionFactory.class); + ResultMap map = sqlSessionFactory.getConfiguration() + .getResultMap(mapName + "Map"); List mapping = map.getResultMappings(); for (ResultMapping mapp : mapping) { columnList.add(mapp.getColumn().toLowerCase()); @@ -641,8 +775,10 @@ public class LogDataService { */ private static Map getFiledAndColumnMap(Class clazz) { Map map = new HashMap(); - SqlSessionFactory sqlSessionFactory = SpringContextHolder.getBean(SqlSessionFactory.class); - ResultMap resultMap = sqlSessionFactory.getConfiguration().getResultMap(clazz.getSimpleName() + "Map"); + SqlSessionFactory sqlSessionFactory = SpringContextHolder + .getBean(SqlSessionFactory.class); + ResultMap resultMap = sqlSessionFactory.getConfiguration() + .getResultMap(clazz.getSimpleName() + "Map"); List mapping = resultMap.getResultMappings(); for (ResultMapping mapp : mapping) { map.put(mapp.getProperty(), mapp.getColumn().toLowerCase()); @@ -659,9 +795,11 @@ public class LogDataService { * @return * @throws Exception */ - private static Object getFieldValue(Object bean, String fieldName) throws Exception { + private static Object getFieldValue(Object bean, String fieldName) + throws Exception { StringBuffer result = new StringBuffer(); - String methodName = result.append("get").append(fieldName.substring(0, 1).toUpperCase()) + String methodName = result.append("get") + .append(fieldName.substring(0, 1).toUpperCase()) .append(fieldName.substring(1)).toString(); Object rObject = null; @@ -683,9 +821,11 @@ public class LogDataService { * @param value * @throws Exception */ - private static void setFieldValue(Object bean, String fieldName, Object value) throws Exception { + private static void setFieldValue(Object bean, String fieldName, + Object value) throws Exception { StringBuffer result = new StringBuffer(); - String methodName = result.append("set").append(fieldName.substring(0, 1).toUpperCase()) + String methodName = result.append("set") + .append(fieldName.substring(0, 1).toUpperCase()) .append(fieldName.substring(1)).toString(); /** * 利用发射调用bean.set方法将value设置到字段 @@ -703,7 +843,8 @@ public class LogDataService { * @param fieldName * @throws Exception */ - private static boolean ifExistFiledName(Object obj, String filedName) throws Exception { + private static boolean ifExistFiledName(Object obj, String filedName) + throws Exception { Class class1 = obj.getClass();// 需要检测的类 /** * 循环遍历所有的元素,检测有没有这个名字 @@ -723,19 +864,26 @@ public class LogDataService { /** * 有泛收的页面调用的方法,(邮箱泛收) 从clickhouse中查询数据,注意clickhouse区分大小写,目前和百分点商定都是用小写 * - * @param page 里面含有pagesize和pageno,order by - * @param bean 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名 - * @param tableName 表名 - * @param className 类名 - * @param orderBy orderby条件 + * @param page + * 里面含有pagesize和pageno,order by + * @param bean + * 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名 + * @param tableName + * 表名 + * @param className + * 类名 + * @param orderBy + * orderby条件 * @throws Exception */ - private void getDataFromClickHouseFS(Page page, Object bean, String tableName, String className, - String orderBy, String ispNum) throws Exception { + private void getDataFromClickHouseFS(Page page, Object bean, + String tableName, String className, String orderBy, String ispNum) + throws Exception { tableName = tableName.toLowerCase(); String showColmun = getFiledsSql(className, page.getFields()); StringBuffer sql = new StringBuffer(); - Map filedAndColumnMap = getFiledAndColumnMap(bean.getClass()); + Map filedAndColumnMap = getFiledAndColumnMap(bean + .getClass()); if (null == showColmun || showColmun.equals("")) { for (String key : filedAndColumnMap.keySet()) { if (!filedAndColumnMap.get(key).toLowerCase().equals("id")) { @@ -754,15 +902,16 @@ public class LogDataService { StringBuffer countSql = new StringBuffer(); if (ispNum != null) { - sql.append(" select " + sqlTrim.toLowerCase() - + ", concat(toString(entrance_id),toString(device_id)) as ispNum from " + tableName.toLowerCase() - + " t where "); - countSql.append( - "select count(1) from (select concat(toString(entrance_id),toString(device_id)) as ispNum from " - + tableName + " where "); + sql.append(" select " + + sqlTrim.toLowerCase() + + ", concat(toString(entrance_id),toString(device_id)) as ispNum from " + + tableName.toLowerCase() + " t where "); + countSql.append("select count(1) from (select concat(toString(entrance_id),toString(device_id)) as ispNum from " + + tableName + " where "); } else { - sql.append(" select " + sqlTrim.toLowerCase() + " from " + tableName.toLowerCase() + " t where "); + sql.append(" select " + sqlTrim.toLowerCase() + " from " + + tableName.toLowerCase() + " t where "); countSql.append("select count(1) from " + tableName + " where "); } @@ -777,20 +926,28 @@ public class LogDataService { // 现在gwall日志表结构中只有数值和字符串两种类型,数值都是int类型没有bigint所以不需要加L, Field f = fields[i]; String key = f.getName();// 获取字段名 - if (f.getType().getName().equals("java.lang.String") && key.startsWith("search")) { + if (f.getType().getName().equals("java.lang.String") + && key.startsWith("search")) { Object value = getFieldValue(bean, key); if (!StringUtil.isEmpty(value)) { setFieldValue(bean, key, value.toString().trim()); if (key.endsWith("Time")) {// 日期开始或结束的字段 if (col2col.containsKey(key)) { - value = sdf.parse(value.toString().trim()).getTime() / 1000; + value = sdf.parse(value.toString().trim()) + .getTime() / 1000; if (col2col.get(key).get("start") != null) { whereFoundTime.append(" and " - + filedAndColumnMap.get(col2col.get(key).get("start")).toLowerCase() - + ">=" + value); + + filedAndColumnMap.get( + col2col.get(key).get( + "start")) + .toLowerCase() + ">=" + + value); } else { whereFoundTime.append(" and " - + filedAndColumnMap.get(col2col.get(key).get("end")).toLowerCase() + "<" + + filedAndColumnMap.get( + col2col.get(key).get( + "end")) + .toLowerCase() + "<" + value); } } @@ -798,46 +955,98 @@ public class LogDataService { if (key != "searchIspCode") {// 将运营商的查询条件排除在外 if (key.toLowerCase().startsWith("search")) { key = key.replace("search", ""); - key = key.substring(0, 1).toLowerCase() + key.substring(1); + key = key.substring(0, 1).toLowerCase() + + key.substring(1); } // clickhouse写法 String type = filedsType.get(key).trim(); - String field = filedAndColumnMap.get(key).toLowerCase(); + String field = filedAndColumnMap.get(key) + .toLowerCase(); if (type.equals("java.lang.String")) { - if (field.contains("url") || field.equals("website")) { - whereSB.append(" and " + field + " like '" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); - } else if (field.equals("client_locate") || field.equals("server_locate")) { - whereSB.append(" and " + field + " like '%" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); + if (field.contains("url") + || field.equals("website")) { + whereSB.append(" and " + + field + + " like '" + + StringEscapeUtils + .unescapeHtml4(value + .toString() + .trim()) + + "%'"); + } else if (field + .equals("client_locate") + || field.equals("server_locate")) { + whereSB.append(" and " + + field + + " like '%" + + StringEscapeUtils + .unescapeHtml4(value + .toString() + .trim()) + + "%'"); } else { - whereSB.append(" and " + field + "='" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'"); + whereSB.append(" and " + + field + + "='" + + StringEscapeUtils + .unescapeHtml4(value + .toString() + .trim()) + + "'"); } - } else if (type.equals("java.lang.Integer") || type.equals("int") - || type.equals("java.lang.Long") || type.equals("long")) { - if (field.equals("cfg_id") || field.equals("web_id") || field.equals("app_id") + } else if (type.equals("java.lang.Integer") + || type.equals("int") + || type.equals("java.lang.Long") + || type.equals("long")) { + if (field.equals("cfg_id") + || field.equals("web_id") + || field.equals("app_id") || field.equals("proto_id")) { if (field.equals("cfg_id")) { - if (value.toString().trim().equals("0")) { - whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() - + " =" + value.toString().trim()); - } else if (value.toString().trim().equals("-1")) { - whereSB.append( - " and " + filedAndColumnMap.get(key).toLowerCase() + " >0"); + if (value.toString().trim() + .equals("0")) { + whereSB.append(" and " + + filedAndColumnMap + .get(key) + .toLowerCase() + + " =" + + value.toString() + .trim()); + } else if (value.toString() + .trim().equals("-1")) { + whereSB.append(" and " + + filedAndColumnMap + .get(key) + .toLowerCase() + + " >0"); } else { - whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() - + " in(" + value.toString().trim() + ")"); + whereSB.append(" and " + + filedAndColumnMap + .get(key) + .toLowerCase() + + " in(" + + value.toString() + .trim() + + ")"); } } else { - whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() - + " in(" + value.toString().trim() + ")"); + whereSB.append(" and " + + filedAndColumnMap + .get(key) + .toLowerCase() + + " in(" + + value.toString() + .trim() + ")"); } } else { - whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "=" + whereSB.append(" and " + + filedAndColumnMap + .get(key) + .toLowerCase() + + "=" + value.toString().trim()); } } @@ -853,48 +1062,62 @@ public class LogDataService { } Integer startNum = (page.getPageNo() - 1) * page.getPageSize(); StringBuffer foundTimeSql = new StringBuffer(); - foundTimeSql.append("select found_time from " + tableName + " where "); + foundTimeSql + .append("select found_time from " + tableName + " where "); Integer limitCount = startNum + page.getPageSize(); if (ispNum != null) { if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件 if (whereFoundTime.length() > 0) { - int indexOf = whereFoundTime.indexOf("and") + "and".length(); - countSql.append(whereFoundTime.substring(indexOf) + " and ispNum in(" + ispNum + "))"); - sql.append(whereFoundTime.substring(indexOf) + " and ispNum in(" + ispNum + ")"); + int indexOf = whereFoundTime.indexOf("and") + + "and".length(); + countSql.append(whereFoundTime.substring(indexOf) + + " and ispNum in(" + ispNum + "))"); + sql.append(whereFoundTime.substring(indexOf) + + " and ispNum in(" + ispNum + ")"); } else { - throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件"); + throw new RuntimeException("从clickhouse的" + tableName + + "表查询时,必须要有一个where条件"); } } else { - int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length(); - countSql.append(whereFoundTime.substring(foundIndexOf) + " and ispNum in(" + ispNum + "))"); + int foundIndexOf = whereFoundTime.append(whereSB) + .indexOf("and") + "and".length(); + countSql.append(whereFoundTime.substring(foundIndexOf) + + " and ispNum in(" + ispNum + "))"); int indexOf = whereSB.indexOf("and") + "and".length(); - sql.append(whereSB.substring(indexOf) + whereFoundTime + " and ispNum in(" + ispNum + ")"); + sql.append(whereSB.substring(indexOf) + whereFoundTime + + " and ispNum in(" + ispNum + ")"); } - sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 + sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + + page.getPageSize());// clickhouse的分页与mysql相同 } else { if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件 if (whereFoundTime.length() > 0) { - int indexOf = whereFoundTime.indexOf("and") + "and".length(); + int indexOf = whereFoundTime.indexOf("and") + + "and".length(); countSql.append(whereFoundTime.substring(indexOf)); - foundTimeSql.append( - whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + " limit " + limitCount); + foundTimeSql.append(whereFoundTime.substring(indexOf) + + orderBy.toLowerCase() + " limit " + limitCount); sql.append(" found_time in(" + foundTimeSql + ") "); } else { - throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件"); + throw new RuntimeException("从clickhouse的" + tableName + + "表查询时,必须要有一个where条件"); } } else { - int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length(); + int foundIndexOf = whereFoundTime.append(whereSB) + .indexOf("and") + "and".length(); countSql.append(whereFoundTime.substring(foundIndexOf)); - foundTimeSql.append( - whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + " limit " + limitCount); + foundTimeSql.append(whereFoundTime.substring(foundIndexOf) + + orderBy.toLowerCase() + " limit " + limitCount); int indexOf = whereSB.indexOf("and") + "and".length(); - sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") "); + sql.append(whereSB.substring(indexOf) + " and found_time in(" + + foundTimeSql + ") "); } - sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 + sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + + page.getPageSize());// clickhouse的分页与mysql相同 } if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")) {