From 453f8ae5dd2de65eab7714551d68099285757481 Mon Sep 17 00:00:00 2001 From: renkaige Date: Wed, 16 Jan 2019 19:16:52 +0600 Subject: [PATCH] =?UTF-8?q?1:=E4=B8=BA=E5=8E=9F=E5=A7=8B=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=B7=BB=E5=8A=A0=E8=BF=90=E8=90=A5=E5=95=86?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=9D=A1=E4=BB=B6=202:=E6=B3=A8=E6=84=8F?= =?UTF-8?q?=E5=A6=82=E6=9E=9C=E9=9C=80=E8=A6=81=E4=B8=BA=E6=9F=90=E4=B8=AA?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E6=B7=BB=E5=8A=A0=E8=BF=90=E8=90=A5=E5=95=86?= =?UTF-8?q?=E6=90=9C=E7=B4=A2=E6=9D=A1=E4=BB=B6,=E9=9C=80=E8=A6=81?= =?UTF-8?q?=E5=9C=A8=E5=AE=9E=E4=BD=93=E7=B1=BB=E4=B8=AD=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=20protected=20String=20searchIspCode;//=20=E8=BF=90=E8=90=A5?= =?UTF-8?q?=E5=95=86=20protected=20String=20ispNum;=20=E4=B8=A4=E4=B8=AA?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../nis/domain/restful/NtcConnRecordLog.java | 41 +- .../com/nis/domain/restful/NtcMailLog.java | 25 +- src/main/java/com/nis/web/dao/IspInfoDao.java | 19 + src/main/java/com/nis/web/dao/IspInfoDao.xml | 8 + .../nis/web/dao/impl/LocalLogJDBCByDruid.java | 41 +- .../com/nis/web/service/LogDataService.java | 357 ++++++++++++------ 6 files changed, 335 insertions(+), 156 deletions(-) create mode 100644 src/main/java/com/nis/web/dao/IspInfoDao.java create mode 100644 src/main/java/com/nis/web/dao/IspInfoDao.xml diff --git a/src/main/java/com/nis/domain/restful/NtcConnRecordLog.java b/src/main/java/com/nis/domain/restful/NtcConnRecordLog.java index 0c95f55..5c1692f 100644 --- a/src/main/java/com/nis/domain/restful/NtcConnRecordLog.java +++ b/src/main/java/com/nis/domain/restful/NtcConnRecordLog.java @@ -13,7 +13,6 @@ import com.wordnik.swagger.annotations.ApiModelProperty; * @version V1.0 */ public class NtcConnRecordLog extends LogEntity { - /** * @@ -29,7 +28,7 @@ public class NtcConnRecordLog extends LogEntity { protected String c2sByteNum; @ApiModelProperty(value = "", required = true) protected String s2cByteNum; - + @ApiModelProperty(value = "", required = true) protected Long protoId; @ApiModelProperty(value = "", required = true) @@ -37,13 +36,17 @@ public class NtcConnRecordLog extends LogEntity { @ApiModelProperty(value = "", required = true) protected Long webId; - - protected String searchAppId;//app - protected String searchWebId;//web - protected String searchProtoId;//proto - //protected String searchServerLocate;//serverLocate - //protected String searchClientLocate;//clientLocate - + @ApiModelProperty(value = "运营商标识,entrance_id,device_id", required = true) + protected String ispNum; + + protected String searchAppId;// app + protected String searchWebId;// web + protected String searchProtoId;// proto + protected String searchIspCode;// 运营商 + + // protected String searchServerLocate;//serverLocate + // protected String searchClientLocate;//clientLocate + @JsonIgnore public String getAppLabel() { return appLabel; @@ -135,7 +138,23 @@ public class NtcConnRecordLog extends LogEntity { public void setSearchProtoId(String searchProtoId) { this.searchProtoId = searchProtoId; } - - + + @JsonIgnore + public String getSearchIspCode() { + return searchIspCode; + } + + public void setSearchIspCode(String searchIspCode) { + this.searchIspCode = searchIspCode; + } + + public String getIspNum() { + return ispNum; + } + + public void setIspNum(String ispNum) { + this.ispNum = ispNum; + } + } diff --git a/src/main/java/com/nis/domain/restful/NtcMailLog.java b/src/main/java/com/nis/domain/restful/NtcMailLog.java index 09782f3..8ca8dac 100644 --- a/src/main/java/com/nis/domain/restful/NtcMailLog.java +++ b/src/main/java/com/nis/domain/restful/NtcMailLog.java @@ -23,11 +23,12 @@ public class NtcMailLog extends LogEntity{ protected String emlKey; @ApiModelProperty(value="EML文件转储路径", required=true) protected String emlFile; - + @ApiModelProperty(value = "运营商标识,entrance_id,device_id", required = true) + protected String ispNum; protected String searchMailFrom; //发件人 protected String searchMailTo; //收件人 protected String searchSubject; //主题 - + protected String searchIspCode;// 运营商 protected String flag;//泛收的标志 @JsonIgnore @@ -114,4 +115,24 @@ public class NtcMailLog extends LogEntity{ public String getSearchSubject() { return searchSubject; } + + public String getIspNum() { + return ispNum; + } + + public void setIspNum(String ispNum) { + this.ispNum = ispNum; + } + @JsonIgnore + public String getSearchIspCode() { + return searchIspCode; + } + + public void setSearchIspCode(String searchIspCode) { + this.searchIspCode = searchIspCode; + } + + + + } diff --git a/src/main/java/com/nis/web/dao/IspInfoDao.java b/src/main/java/com/nis/web/dao/IspInfoDao.java new file mode 100644 index 0000000..bcc1e14 --- /dev/null +++ b/src/main/java/com/nis/web/dao/IspInfoDao.java @@ -0,0 +1,19 @@ +package com.nis.web.dao; + +import java.util.List; + +import org.apache.ibatis.annotations.Param; + +/** + * + * @ClassName:IspInfoDao + * @Description:TODO(这里用一句话描述这个类的作用) + * @author (rkg) + * @date 2019年1月16日 下午5:47:55 + * @version V1.0 + */ +@MyBatisDao +public interface IspInfoDao { + + List getIspNum(@Param("ispCode") String ispCode); +} diff --git a/src/main/java/com/nis/web/dao/IspInfoDao.xml b/src/main/java/com/nis/web/dao/IspInfoDao.xml new file mode 100644 index 0000000..1eff3b8 --- /dev/null +++ b/src/main/java/com/nis/web/dao/IspInfoDao.xml @@ -0,0 +1,8 @@ + + + + + \ No newline at end of file diff --git a/src/main/java/com/nis/web/dao/impl/LocalLogJDBCByDruid.java b/src/main/java/com/nis/web/dao/impl/LocalLogJDBCByDruid.java index a3b011d..cb2183c 100644 --- a/src/main/java/com/nis/web/dao/impl/LocalLogJDBCByDruid.java +++ b/src/main/java/com/nis/web/dao/impl/LocalLogJDBCByDruid.java @@ -90,17 +90,19 @@ public class LocalLogJDBCByDruid { for (int i = 1; i <= metaData.getColumnCount(); i++) { Object value = rs.getObject(i); String filedName = filedAndColumnMap.get(metaData.getColumnName(i).toString().toLowerCase()); - if (!StringUtil.isEmpty(value)) { - // 如果是日期类型的属性需要把时间戳转换成日期，如果时间戳为0直接把值设置为null - if (null != columnList && columnList.contains(filedName.toLowerCase())) { - long time = 0L; - time = Long.parseLong(value.toString()); - map.put(filedName, time == 0L ? null : new Date(time * 1000)); + if (filedName != null) { + if (!StringUtil.isEmpty(value)) { + // 如果是日期类型的属性需要把时间戳转换成日期，如果时间戳为0直接把值设置为null + if (null != columnList && columnList.contains(filedName.toLowerCase())) { + long time = 0L; + time = Long.parseLong(value.toString()); + map.put(filedName, time == 0L ? null : new Date(time * 1000)); + } else { + map.put(filedName, value); + } } else { - map.put(filedName, value); + map.put(filedName, null); } - } else { - map.put(filedName, null); } } listObject.add((T) map2Obj(map, entityClass)); @@ -148,10 +150,10 @@ public class LocalLogJDBCByDruid { if (!allPps.equals("0")) { ntcConnRecordPercent.setPpsPercent( num.format(Double.valueOf(ntcConnRecordPercent.getPps()) / Double.valueOf(allPps))); - String ppsNum =ntcConnRecordPercent.getPps(); - if(ntcConnRecordPercent.getPps().contains(".")) { - int idx =ntcConnRecordPercent.getPps().lastIndexOf(".");//查找小数点的位置 - ppsNum= ntcConnRecordPercent.getPps().substring(0,idx);//截取从字符串开始到小数点位置的字符串，就是整数部分 + String ppsNum = ntcConnRecordPercent.getPps(); + if (ntcConnRecordPercent.getPps().contains(".")) { + int idx = ntcConnRecordPercent.getPps().lastIndexOf(".");// 查找小数点的位置 + ppsNum = ntcConnRecordPercent.getPps().substring(0, idx);// 截取从字符串开始到小数点位置的字符串，就是整数部分 } ntcConnRecordPercent.setPps(ppsNum); } else { @@ -193,10 +195,10 @@ public class LocalLogJDBCByDruid { TrafficAsnStatistic ntcAsnRecord = new TrafficAsnStatistic(); String bps = rs.getString("bps"); String pps = rs.getString("pps"); - String ppsNum =pps; - if(pps.contains(".")) { - int idx =pps.lastIndexOf(".");//查找小数点的位置 - ppsNum= pps.substring(0,idx);//截取从字符串开始到小数点位置的字符串，就是整数部分 + String ppsNum = pps; + if (pps.contains(".")) { + int idx = pps.lastIndexOf(".");// 查找小数点的位置 + ppsNum = pps.substring(0, idx);// 截取从字符串开始到小数点位置的字符串，就是整数部分 } ntcAsnRecord.setBps(subStr(bps)); ntcAsnRecord.setPps(ppsNum); @@ -214,7 +216,8 @@ public class LocalLogJDBCByDruid { List list = new ArrayList<>(); try { // String sql = "select distinct ip from ip_location_database_local where ip like '145%' and frontier='"+frontier+"'"; - String sql = "select distinct ip from (select ip,frontier from ip_location_database ) where frontier='"+frontier+"'"; + String sql = "select distinct ip from (select ip,frontier from ip_location_database ) where frontier='" + + frontier + "'"; conn = getConnection(); logger.info("连接数据中心日志库成功--------------------------"); st = conn.createStatement(); @@ -223,7 +226,7 @@ public class LocalLogJDBCByDruid { num.setMinimumFractionDigits(2);// 保留两位小数 while (rs.next()) { String bps = rs.getString("ip"); - if(bps!=null&&!bps.trim().equals("")) { + if (bps != null && !bps.trim().equals("")) { list.add(bps); } } diff --git a/src/main/java/com/nis/web/service/LogDataService.java b/src/main/java/com/nis/web/service/LogDataService.java index c51d49d..69ab607 100644 --- a/src/main/java/com/nis/web/service/LogDataService.java +++ b/src/main/java/com/nis/web/service/LogDataService.java @@ -24,6 +24,7 @@ import com.nis.restful.RestBusinessCode; import com.nis.restful.RestServiceException; import com.nis.util.Configurations; import com.nis.util.Constants; +import com.nis.web.dao.IspInfoDao; import com.nis.web.dao.impl.LocalLogJDBCByDruid; import com.nis.web.dao.impl.LogJDBCByDruid; import com.zdjizhi.utils.StringUtil; @@ -40,6 +41,9 @@ public class LogDataService { @Autowired private LogJDBCByDruid logJDBCByDruid; + + @Autowired + private IspInfoDao ispInfoDao; @Autowired private LocalLogJDBCByDruid localLogJDBCByDruid; private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); @@ -179,11 +183,11 @@ public class LogDataService { orderBy = orderBy + "found_Time desc "; } if (Constants.ISUSECLICKHOUSE) { - //判断实体类中是否存在某个字段 - if(ifExistFiledName(obj,"flag")){//邮件泛收日志 - getDataFromClickHouseFS(page, obj, tableName, className, orderBy.toLowerCase()); - }else{ - getDataFromClickHouse(page, obj, tableName, className, orderBy.toLowerCase()); + // 判断实体类中是否存在某个字段 + if (ifExistFiledName(obj, "flag")) {// 邮件泛收日志 + getDataFromClickHouseFS(page, obj, tableName, className, orderBy.toLowerCase(), getIspNum(obj)); + } else { + getDataFromClickHouse(page, obj, tableName, className, orderBy.toLowerCase(), getIspNum(obj)); } } else { getDataFromHive(page, obj, tableName, className, orderBy.toLowerCase()); @@ -191,6 +195,22 @@ public class LogDataService { } + /** + * 判断日志查询条件中是否有运营商的查询条件,如果有则去数据库中查询运营商的组合信息 + * + * @param obj + * @return + * @throws Exception + */ + private String getIspNum(Object obj) throws Exception { + String ispNum = null; + Object fieldValue = getFieldValue(obj, "searchIspCode"); + if (fieldValue != null) { + ispNum = getIspNum(fieldValue.toString()); + } + return ispNum; + } + /** * 从clickhouse中查询数据,注意clickhouse区分大小写,目前和百分点商定都是用小写 * @@ -202,7 +222,7 @@ public class LogDataService { * @throws Exception */ private void getDataFromClickHouse(Page page, Object bean, String tableName, String className, - String orderBy) throws Exception { + String orderBy, String ispNum) throws Exception { tableName = tableName.toLowerCase(); String showColmun = getFiledsSql(className, page.getFields()); StringBuffer sql = new StringBuffer(); @@ -221,10 +241,22 @@ public class LogDataService { sqlTrim = sqlTrim.substring(0, sqlTrim.length() - 1); } sql.setLength(0); - sql.append(" select " + sqlTrim.toLowerCase() + " from " + tableName.toLowerCase() + " t where "); + StringBuffer whereFoundTime = new StringBuffer(); StringBuffer countSql = new StringBuffer(); - countSql.append("select count(1) from " + tableName + " where "); + + if (ispNum != null) { + sql.append(" select " + sqlTrim.toLowerCase() + + ", concat(toString(entrance_id),toString(device_id)) as ispNum from " + tableName.toLowerCase() + + " t where "); + countSql.append( + "select count(1) from (select concat(toString(entrance_id),toString(device_id)) as ispNum from " + + tableName + " where "); + + } else { + sql.append(" select " + sqlTrim.toLowerCase() + " from " + tableName.toLowerCase() + " t where "); + countSql.append("select count(1) from " + tableName + " where "); + } StringBuffer whereSB = new StringBuffer(); if (!StringUtil.isEmpty(bean)) { @@ -255,38 +287,39 @@ public class LogDataService { } } } else { - if (key.toLowerCase().startsWith("search")) { - key = key.replace("search", ""); - key = key.substring(0, 1).toLowerCase() + key.substring(1); - } - // clickhouse写法 - String type = filedsType.get(key).trim(); - String field = filedAndColumnMap.get(key).toLowerCase(); - if (type.equals("java.lang.String")) { - - if (field.contains("url") || field.equals("website")) { - whereSB.append(" and " + field + " like '" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); - } else if (field.equals("client_locate") || field.equals("server_locate")) { - whereSB.append(" and " + field + " like '%" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); - } else { - whereSB.append(" and " + field + "='" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'"); + if (key != "searchIspCode") {// 将运营商的查询条件排除在外 + if (key.toLowerCase().startsWith("search")) { + key = key.replace("search", ""); + key = key.substring(0, 1).toLowerCase() + key.substring(1); } + // clickhouse写法 + String type = filedsType.get(key).trim(); + String field = filedAndColumnMap.get(key).toLowerCase(); + if (type.equals("java.lang.String")) { - } else if (type.equals("java.lang.Integer") || type.equals("int") - || type.equals("java.lang.Long") || type.equals("long")) { - if (field.equals("cfg_id") || field.equals("web_id") || field.equals("app_id") - || field.equals("proto_id")) { - whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in(" - + value.toString().trim() + ")"); - } else { - whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "=" - + value.toString().trim()); + if (field.contains("url") || field.equals("website")) { + whereSB.append(" and " + field + " like '" + + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); + } else if (field.equals("client_locate") || field.equals("server_locate")) { + whereSB.append(" and " + field + " like '%" + + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); + } else { + whereSB.append(" and " + field + "='" + + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'"); + } + + } else if (type.equals("java.lang.Integer") || type.equals("int") + || type.equals("java.lang.Long") || type.equals("long")) { + if (field.equals("cfg_id") || field.equals("web_id") || field.equals("app_id") + || field.equals("proto_id")) { + whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in(" + + value.toString().trim() + ")"); + } else { + whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "=" + + value.toString().trim()); + } } } - } } @@ -299,29 +332,51 @@ public class LogDataService { StringBuffer foundTimeSql = new StringBuffer(); foundTimeSql.append("select found_time from " + tableName + " where "); Integer limitCount = startNum + page.getPageSize(); - if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件 - if (whereFoundTime.length() > 0) { - int indexOf = whereFoundTime.indexOf("and") + "and".length(); - countSql.append(whereFoundTime.substring(indexOf)); - - foundTimeSql - .append(whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + " limit " + limitCount); - sql.append(" found_time in(" + foundTimeSql + ") "); + if (ispNum != null) { + if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件 + if (whereFoundTime.length() > 0) { + int indexOf = whereFoundTime.indexOf("and") + "and".length(); + countSql.append(whereFoundTime.substring(indexOf) + " and ispNum in(" + ispNum + "))"); + sql.append(whereFoundTime.substring(indexOf) + " and ispNum in(" + ispNum + ")"); + } else { + throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件"); + } } else { - throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件"); + int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length(); + countSql.append(whereFoundTime.substring(foundIndexOf) + " and ispNum in(" + ispNum + "))"); + int indexOf = whereSB.indexOf("and") + "and".length(); + sql.append(whereSB.substring(indexOf) + whereFoundTime + " and ispNum in(" + ispNum + ")"); } + sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 + } else { - int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length(); - countSql.append(whereFoundTime.substring(foundIndexOf)); - foundTimeSql - .append(whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + " limit " + limitCount); - int indexOf = whereSB.indexOf("and") + "and".length(); - sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") "); + if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件 + if (whereFoundTime.length() > 0) { + int indexOf = whereFoundTime.indexOf("and") + "and".length(); + countSql.append(whereFoundTime.substring(indexOf)); + + foundTimeSql.append( + whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + " limit " + limitCount); + sql.append(" found_time in(" + foundTimeSql + ") "); + } else { + throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件"); + } + } else { + int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length(); + countSql.append(whereFoundTime.substring(foundIndexOf)); + foundTimeSql.append( + whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + " limit " + limitCount); + + int indexOf = whereSB.indexOf("and") + "and".length(); + sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") "); + } + sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 + } - sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 - if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")|| tableName.toUpperCase().equals("TBS_ODS_NTC_COLLECT_SSL_LOG")) { + if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG") + || tableName.toUpperCase().equals("TBS_ODS_NTC_COLLECT_SSL_LOG")) { searchFromLocalCK(page, bean, sql, countSql); } else { searchFromDataCenter(page, bean, sql, countSql); @@ -640,7 +695,7 @@ public class LogDataService { Method method = bean.getClass().getMethod(methodName, classArr); method.invoke(bean, value); } - + /** * 利用反射查询一个对象中是否存在某个属性 * @@ -648,27 +703,25 @@ public class LogDataService { * @param fieldName * @throws Exception */ - private static boolean ifExistFiledName(Object obj,String filedName) throws Exception{ - Class class1=obj.getClass();//需要检测的类 + private static boolean ifExistFiledName(Object obj, String filedName) throws Exception { + Class class1 = obj.getClass();// 需要检测的类 /** * 循环遍历所有的元素，检测有没有这个名字 */ - Field[] fields=class1.getDeclaredFields(); - - boolean b=false; + Field[] fields = class1.getDeclaredFields(); + + boolean b = false; for (int i = 0; i < fields.length; i++) { - if(fields[i].getName().equals(filedName)) - { - b=true; - break; - } + if (fields[i].getName().equals(filedName)) { + b = true; + break; + } } return b; } - + /** - * 有泛收的页面调用的方法，(邮箱泛收) - * 从clickhouse中查询数据,注意clickhouse区分大小写,目前和百分点商定都是用小写 + * 有泛收的页面调用的方法，(邮箱泛收) 从clickhouse中查询数据,注意clickhouse区分大小写,目前和百分点商定都是用小写 * * @param page 里面含有pagesize和pageno,order by * @param bean 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名 @@ -678,7 +731,7 @@ public class LogDataService { * @throws Exception */ private void getDataFromClickHouseFS(Page page, Object bean, String tableName, String className, - String orderBy) throws Exception { + String orderBy, String ispNum) throws Exception { tableName = tableName.toLowerCase(); String showColmun = getFiledsSql(className, page.getFields()); StringBuffer sql = new StringBuffer(); @@ -697,10 +750,21 @@ public class LogDataService { sqlTrim = sqlTrim.substring(0, sqlTrim.length() - 1); } sql.setLength(0); - sql.append(" select " + sqlTrim.toLowerCase() + " from " + tableName.toLowerCase() + " t where "); StringBuffer whereFoundTime = new StringBuffer(); StringBuffer countSql = new StringBuffer(); - countSql.append("select count(1) from " + tableName + " where "); + + if (ispNum != null) { + sql.append(" select " + sqlTrim.toLowerCase() + + ", concat(toString(entrance_id),toString(device_id)) as ispNum from " + tableName.toLowerCase() + + " t where "); + countSql.append( + "select count(1) from (select concat(toString(entrance_id),toString(device_id)) as ispNum from " + + tableName + " where "); + + } else { + sql.append(" select " + sqlTrim.toLowerCase() + " from " + tableName.toLowerCase() + " t where "); + countSql.append("select count(1) from " + tableName + " where "); + } StringBuffer whereSB = new StringBuffer(); if (!StringUtil.isEmpty(bean)) { @@ -731,49 +795,52 @@ public class LogDataService { } } } else { - if (key.toLowerCase().startsWith("search")) { - key = key.replace("search", ""); - key = key.substring(0, 1).toLowerCase() + key.substring(1); - } - // clickhouse写法 - String type = filedsType.get(key).trim(); - String field = filedAndColumnMap.get(key).toLowerCase(); - if (type.equals("java.lang.String")) { - - if (field.contains("url") || field.equals("website")) { - whereSB.append(" and " + field + " like '" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); - } else if (field.equals("client_locate") || field.equals("server_locate")) { - whereSB.append(" and " + field + " like '%" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); - } else { - whereSB.append(" and " + field + "='" - + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'"); + if (key != "searchIspCode") {// 将运营商的查询条件排除在外 + if (key.toLowerCase().startsWith("search")) { + key = key.replace("search", ""); + key = key.substring(0, 1).toLowerCase() + key.substring(1); } - } else if (type.equals("java.lang.Integer") || type.equals("int") - || type.equals("java.lang.Long") || type.equals("long")) { - if (field.equals("cfg_id")|| field.equals("web_id") || field.equals("app_id") - || field.equals("proto_id")){ - if(field.equals("cfg_id")){ - if(value.toString().trim().equals("0")){ - whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " =" - + value.toString().trim()); - }else if(value.toString().trim().equals("-1")){ - whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " >0" ); - }else{ - whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in(" - + value.toString().trim() + ")"); + // clickhouse写法 + String type = filedsType.get(key).trim(); + String field = filedAndColumnMap.get(key).toLowerCase(); + if (type.equals("java.lang.String")) { + + if (field.contains("url") || field.equals("website")) { + whereSB.append(" and " + field + " like '" + + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); + } else if (field.equals("client_locate") || field.equals("server_locate")) { + whereSB.append(" and " + field + " like '%" + + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'"); + } else { + whereSB.append(" and " + field + "='" + + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'"); + } + + } else if (type.equals("java.lang.Integer") || type.equals("int") + || type.equals("java.lang.Long") || type.equals("long")) { + if (field.equals("cfg_id") || field.equals("web_id") || field.equals("app_id") + || field.equals("proto_id")) { + if (field.equals("cfg_id")) { + if (value.toString().trim().equals("0")) { + whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + + " =" + value.toString().trim()); + } else if (value.toString().trim().equals("-1")) { + whereSB.append( + " and " + filedAndColumnMap.get(key).toLowerCase() + " >0"); + } else { + whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + + " in(" + value.toString().trim() + ")"); } - }else{ - whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in(" - + value.toString().trim() + ")"); + } else { + whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + + " in(" + value.toString().trim() + ")"); } - }else { + } else { whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "=" + value.toString().trim()); } - + } } } @@ -788,33 +855,75 @@ public class LogDataService { StringBuffer foundTimeSql = new StringBuffer(); foundTimeSql.append("select found_time from " + tableName + " where "); Integer limitCount = startNum + page.getPageSize(); - if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件 - if (whereFoundTime.length() > 0) { - int indexOf = whereFoundTime.indexOf("and") + "and".length(); - countSql.append(whereFoundTime.substring(indexOf)); - - foundTimeSql - .append(whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + " limit " + limitCount); - sql.append(" found_time in(" + foundTimeSql + ") "); + if (ispNum != null) { + if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件 + if (whereFoundTime.length() > 0) { + int indexOf = whereFoundTime.indexOf("and") + "and".length(); + countSql.append(whereFoundTime.substring(indexOf) + " and ispNum in(" + ispNum + "))"); + sql.append(whereFoundTime.substring(indexOf) + " and ispNum in(" + ispNum + ")"); + } else { + throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件"); + } } else { - throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件"); + int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length(); + countSql.append(whereFoundTime.substring(foundIndexOf) + " and ispNum in(" + ispNum + "))"); + int indexOf = whereSB.indexOf("and") + "and".length(); + sql.append(whereSB.substring(indexOf) + whereFoundTime + " and ispNum in(" + ispNum + ")"); } + sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 + } else { - int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length(); - countSql.append(whereFoundTime.substring(foundIndexOf)); - foundTimeSql - .append(whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + " limit " + limitCount); - int indexOf = whereSB.indexOf("and") + "and".length(); - sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") "); + if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件 + if (whereFoundTime.length() > 0) { + int indexOf = whereFoundTime.indexOf("and") + "and".length(); + countSql.append(whereFoundTime.substring(indexOf)); + + foundTimeSql.append( + whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + " limit " + limitCount); + sql.append(" found_time in(" + foundTimeSql + ") "); + } else { + throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件"); + } + } else { + int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length(); + countSql.append(whereFoundTime.substring(foundIndexOf)); + foundTimeSql.append( + whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + " limit " + limitCount); + + int indexOf = whereSB.indexOf("and") + "and".length(); + sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") "); + } + sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 + } - sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 - - if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG") ) { + if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")) { searchFromLocalCK(page, bean, sql, countSql); } else { searchFromDataCenter(page, bean, sql, countSql); } } + + /** + * 根据ispcode获取当前运营商下所有的entrance_id,device_id,link_id组合 + * + * @param ispCode + * @return + */ + private String getIspNum(String ispCode) { + List ispNum = ispInfoDao.getIspNum(ispCode); + if (ispNum != null && ispNum.size() > 0) { + StringBuffer sb = new StringBuffer(); + for (String ispNumStr : ispNum) { + sb.append("'"); + sb.append(ispNumStr); + sb.append("'"); + sb.append(","); + } + return sb.substring(0, sb.length() - 1); + } + return null; + } + }