gfwleak/galaxy-k18-galaxy-service
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1:为原始日志查询添加运营商查询条件

Browse Source 
2:注意如果需要为某个接口添加运营商搜索条件,需要在实体类中新增
protected String searchIspCode;// 运营商
protected String ispNum;
两个字段
This commit is contained in:
renkaige
2019-01-16 19:16:52 +06:00
parent d86cce8dc6
commit 453f8ae5dd
6 changed files with 335 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

357
src/main/java/com/nis/web/service/LogDataService.java
View File

@@ -24,6 +24,7 @@ import com.nis.restful.RestBusinessCode;
import com.nis.restful.RestServiceException;
import com.nis.util.Configurations;
import com.nis.util.Constants;
import com.nis.web.dao.IspInfoDao;
import com.nis.web.dao.impl.LocalLogJDBCByDruid;
import com.nis.web.dao.impl.LogJDBCByDruid;
import com.zdjizhi.utils.StringUtil;
@@ -40,6 +41,9 @@ public class LogDataService {
@Autowired
private LogJDBCByDruid logJDBCByDruid;
@Autowired
private IspInfoDao ispInfoDao;
@Autowired
private LocalLogJDBCByDruid localLogJDBCByDruid;
private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
@@ -179,11 +183,11 @@ public class LogDataService {
orderBy = orderBy + "found_Time desc ";
}
if (Constants.ISUSECLICKHOUSE) {
//判断实体类中是否存在某个字段
if(ifExistFiledName(obj,"flag")){//邮件泛收日志
getDataFromClickHouseFS(page, obj, tableName, className, orderBy.toLowerCase());
}else{
getDataFromClickHouse(page, obj, tableName, className, orderBy.toLowerCase());
// 判断实体类中是否存在某个字段
if (ifExistFiledName(obj, "flag")) {// 邮件泛收日志
getDataFromClickHouseFS(page, obj, tableName, className, orderBy.toLowerCase(), getIspNum(obj));
} else {
getDataFromClickHouse(page, obj, tableName, className, orderBy.toLowerCase(), getIspNum(obj));
}
} else {
getDataFromHive(page, obj, tableName, className, orderBy.toLowerCase());
@@ -191,6 +195,22 @@ public class LogDataService {
}
/**
* 判断日志查询条件中是否有运营商的查询条件,如果有则去数据库中查询运营商的组合信息
*
* @param obj
* @return
* @throws Exception
*/
private String getIspNum(Object obj) throws Exception {
String ispNum = null;
Object fieldValue = getFieldValue(obj, "searchIspCode");
if (fieldValue != null) {
ispNum = getIspNum(fieldValue.toString());
}
return ispNum;
}
/**
* 从clickhouse中查询数据,注意clickhouse区分大小写,目前和百分点商定都是用小写
*
@@ -202,7 +222,7 @@ public class LogDataService {
* @throws Exception
*/
private <T> void getDataFromClickHouse(Page<T> page, Object bean, String tableName, String className,
String orderBy) throws Exception {
String orderBy, String ispNum) throws Exception {
tableName = tableName.toLowerCase();
String showColmun = getFiledsSql(className, page.getFields());
StringBuffer sql = new StringBuffer();
@@ -221,10 +241,22 @@ public class LogDataService {
sqlTrim = sqlTrim.substring(0, sqlTrim.length() - 1);
}
sql.setLength(0);
sql.append(" select " + sqlTrim.toLowerCase() + " from " + tableName.toLowerCase() + " t where ");
StringBuffer whereFoundTime = new StringBuffer();
StringBuffer countSql = new StringBuffer();
countSql.append("select count(1) from " + tableName + " where ");
if (ispNum != null) {
sql.append(" select " + sqlTrim.toLowerCase()
+ ", concat(toString(entrance_id),toString(device_id)) as ispNum from " + tableName.toLowerCase()
+ " t where ");
countSql.append(
"select count(1) from (select concat(toString(entrance_id),toString(device_id)) as ispNum from "
+ tableName + " where ");
} else {
sql.append(" select " + sqlTrim.toLowerCase() + " from " + tableName.toLowerCase() + " t where ");
countSql.append("select count(1) from " + tableName + " where ");
}
StringBuffer whereSB = new StringBuffer();
if (!StringUtil.isEmpty(bean)) {
@@ -255,38 +287,39 @@ public class LogDataService {
}
}
} else {
if (key.toLowerCase().startsWith("search")) {
key = key.replace("search", "");
key = key.substring(0, 1).toLowerCase() + key.substring(1);
}
// clickhouse写法
String type = filedsType.get(key).trim();
String field = filedAndColumnMap.get(key).toLowerCase();
if (type.equals("java.lang.String")) {
if (field.contains("url") || field.equals("website")) {
whereSB.append(" and " + field + " like '"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'");
} else if (field.equals("client_locate") || field.equals("server_locate")) {
whereSB.append(" and " + field + " like '%"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'");
} else {
whereSB.append(" and " + field + "='"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'");
if (key != "searchIspCode") {// 将运营商的查询条件排除在外
if (key.toLowerCase().startsWith("search")) {
key = key.replace("search", "");
key = key.substring(0, 1).toLowerCase() + key.substring(1);
}
// clickhouse写法
String type = filedsType.get(key).trim();
String field = filedAndColumnMap.get(key).toLowerCase();
if (type.equals("java.lang.String")) {
} else if (type.equals("java.lang.Integer") || type.equals("int")
|| type.equals("java.lang.Long") || type.equals("long")) {
if (field.equals("cfg_id") || field.equals("web_id") || field.equals("app_id")
|| field.equals("proto_id")) {
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in("
+ value.toString().trim() + ")");
} else {
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "="
+ value.toString().trim());
if (field.contains("url") || field.equals("website")) {
whereSB.append(" and " + field + " like '"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'");
} else if (field.equals("client_locate") || field.equals("server_locate")) {
whereSB.append(" and " + field + " like '%"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'");
} else {
whereSB.append(" and " + field + "='"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'");
}
} else if (type.equals("java.lang.Integer") || type.equals("int")
|| type.equals("java.lang.Long") || type.equals("long")) {
if (field.equals("cfg_id") || field.equals("web_id") || field.equals("app_id")
|| field.equals("proto_id")) {
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in("
+ value.toString().trim() + ")");
} else {
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "="
+ value.toString().trim());
}
}
}
}
}
@@ -299,29 +332,51 @@ public class LogDataService {
StringBuffer foundTimeSql = new StringBuffer();
foundTimeSql.append("select found_time from " + tableName + " where ");
Integer limitCount = startNum + page.getPageSize();
if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件
if (whereFoundTime.length() > 0) {
int indexOf = whereFoundTime.indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(indexOf));
foundTimeSql
.append(whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + " limit " + limitCount);
sql.append(" found_time in(" + foundTimeSql + ") ");
if (ispNum != null) {
if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件
if (whereFoundTime.length() > 0) {
int indexOf = whereFoundTime.indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(indexOf) + " and ispNum in(" + ispNum + "))");
sql.append(whereFoundTime.substring(indexOf) + " and ispNum in(" + ispNum + ")");
} else {
throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件");
}
} else {
throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件");
int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(foundIndexOf) + " and ispNum in(" + ispNum + "))");
int indexOf = whereSB.indexOf("and") + "and".length();
sql.append(whereSB.substring(indexOf) + whereFoundTime + " and ispNum in(" + ispNum + ")");
}
sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
} else {
int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(foundIndexOf));
foundTimeSql
.append(whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + " limit " + limitCount);
int indexOf = whereSB.indexOf("and") + "and".length();
sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") ");
if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件
if (whereFoundTime.length() > 0) {
int indexOf = whereFoundTime.indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(indexOf));
foundTimeSql.append(
whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + " limit " + limitCount);
sql.append(" found_time in(" + foundTimeSql + ") ");
} else {
throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件");
}
} else {
int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(foundIndexOf));
foundTimeSql.append(
whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + " limit " + limitCount);
int indexOf = whereSB.indexOf("and") + "and".length();
sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") ");
}
sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
}
sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")|| tableName.toUpperCase().equals("TBS_ODS_NTC_COLLECT_SSL_LOG")) {
if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")
|| tableName.toUpperCase().equals("TBS_ODS_NTC_COLLECT_SSL_LOG")) {
searchFromLocalCK(page, bean, sql, countSql);
} else {
searchFromDataCenter(page, bean, sql, countSql);
@@ -640,7 +695,7 @@ public class LogDataService {
Method method = bean.getClass().getMethod(methodName, classArr);
method.invoke(bean, value);
}
/**
* 利用反射查询一个对象中是否存在某个属性
*
@@ -648,27 +703,25 @@ public class LogDataService {
* @param fieldName
* @throws Exception
*/
private static boolean ifExistFiledName(Object obj,String filedName) throws Exception{
Class class1=obj.getClass();//需要检测的类
private static boolean ifExistFiledName(Object obj, String filedName) throws Exception {
Class class1 = obj.getClass();// 需要检测的类
/**
* 循环遍历所有的元素，检测有没有这个名字
*/
Field[] fields=class1.getDeclaredFields();
boolean b=false;
Field[] fields = class1.getDeclaredFields();
boolean b = false;
for (int i = 0; i < fields.length; i++) {
if(fields[i].getName().equals(filedName))
{
b=true;
break;
}
if (fields[i].getName().equals(filedName)) {
b = true;
break;
}
}
return b;
}
/**
* 有泛收的页面调用的方法，(邮箱泛收)
* 从clickhouse中查询数据,注意clickhouse区分大小写,目前和百分点商定都是用小写
* 有泛收的页面调用的方法，(邮箱泛收) 从clickhouse中查询数据,注意clickhouse区分大小写,目前和百分点商定都是用小写
*
* @param page 里面含有pagesize和pageno,order by
* @param bean 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名
@@ -678,7 +731,7 @@ public class LogDataService {
* @throws Exception
*/
private <T> void getDataFromClickHouseFS(Page<T> page, Object bean, String tableName, String className,
String orderBy) throws Exception {
String orderBy, String ispNum) throws Exception {
tableName = tableName.toLowerCase();
String showColmun = getFiledsSql(className, page.getFields());
StringBuffer sql = new StringBuffer();
@@ -697,10 +750,21 @@ public class LogDataService {
sqlTrim = sqlTrim.substring(0, sqlTrim.length() - 1);
}
sql.setLength(0);
sql.append(" select " + sqlTrim.toLowerCase() + " from " + tableName.toLowerCase() + " t where ");
StringBuffer whereFoundTime = new StringBuffer();
StringBuffer countSql = new StringBuffer();
countSql.append("select count(1) from " + tableName + " where ");
if (ispNum != null) {
sql.append(" select " + sqlTrim.toLowerCase()
+ ", concat(toString(entrance_id),toString(device_id)) as ispNum from " + tableName.toLowerCase()
+ " t where ");
countSql.append(
"select count(1) from (select concat(toString(entrance_id),toString(device_id)) as ispNum from "
+ tableName + " where ");
} else {
sql.append(" select " + sqlTrim.toLowerCase() + " from " + tableName.toLowerCase() + " t where ");
countSql.append("select count(1) from " + tableName + " where ");
}
StringBuffer whereSB = new StringBuffer();
if (!StringUtil.isEmpty(bean)) {
@@ -731,49 +795,52 @@ public class LogDataService {
}
}
} else {
if (key.toLowerCase().startsWith("search")) {
key = key.replace("search", "");
key = key.substring(0, 1).toLowerCase() + key.substring(1);
}
// clickhouse写法
String type = filedsType.get(key).trim();
String field = filedAndColumnMap.get(key).toLowerCase();
if (type.equals("java.lang.String")) {
if (field.contains("url") || field.equals("website")) {
whereSB.append(" and " + field + " like '"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'");
} else if (field.equals("client_locate") || field.equals("server_locate")) {
whereSB.append(" and " + field + " like '%"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'");
} else {
whereSB.append(" and " + field + "='"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'");
if (key != "searchIspCode") {// 将运营商的查询条件排除在外
if (key.toLowerCase().startsWith("search")) {
key = key.replace("search", "");
key = key.substring(0, 1).toLowerCase() + key.substring(1);
}
} else if (type.equals("java.lang.Integer") || type.equals("int")
|| type.equals("java.lang.Long") || type.equals("long")) {
if (field.equals("cfg_id")|| field.equals("web_id") || field.equals("app_id")
|| field.equals("proto_id")){
if(field.equals("cfg_id")){
if(value.toString().trim().equals("0")){
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " ="
+ value.toString().trim());
}else if(value.toString().trim().equals("-1")){
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " >0" );
}else{
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in("
+ value.toString().trim() + ")");
// clickhouse写法
String type = filedsType.get(key).trim();
String field = filedAndColumnMap.get(key).toLowerCase();
if (type.equals("java.lang.String")) {
if (field.contains("url") || field.equals("website")) {
whereSB.append(" and " + field + " like '"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'");
} else if (field.equals("client_locate") || field.equals("server_locate")) {
whereSB.append(" and " + field + " like '%"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'");
} else {
whereSB.append(" and " + field + "='"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'");
}
} else if (type.equals("java.lang.Integer") || type.equals("int")
|| type.equals("java.lang.Long") || type.equals("long")) {
if (field.equals("cfg_id") || field.equals("web_id") || field.equals("app_id")
|| field.equals("proto_id")) {
if (field.equals("cfg_id")) {
if (value.toString().trim().equals("0")) {
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase()
+ " =" + value.toString().trim());
} else if (value.toString().trim().equals("-1")) {
whereSB.append(
" and " + filedAndColumnMap.get(key).toLowerCase() + " >0");
} else {
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase()
+ " in(" + value.toString().trim() + ")");
}
}else{
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in("
+ value.toString().trim() + ")");
} else {
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase()
+ " in(" + value.toString().trim() + ")");
}
}else {
} else {
whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "="
+ value.toString().trim());
}
}
}
}
@@ -788,33 +855,75 @@ public class LogDataService {
StringBuffer foundTimeSql = new StringBuffer();
foundTimeSql.append("select found_time from " + tableName + " where ");
Integer limitCount = startNum + page.getPageSize();
if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件
if (whereFoundTime.length() > 0) {
int indexOf = whereFoundTime.indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(indexOf));
foundTimeSql
.append(whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + " limit " + limitCount);
sql.append(" found_time in(" + foundTimeSql + ") ");
if (ispNum != null) {
if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件
if (whereFoundTime.length() > 0) {
int indexOf = whereFoundTime.indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(indexOf) + " and ispNum in(" + ispNum + "))");
sql.append(whereFoundTime.substring(indexOf) + " and ispNum in(" + ispNum + ")");
} else {
throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件");
}
} else {
throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件");
int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(foundIndexOf) + " and ispNum in(" + ispNum + "))");
int indexOf = whereSB.indexOf("and") + "and".length();
sql.append(whereSB.substring(indexOf) + whereFoundTime + " and ispNum in(" + ispNum + ")");
}
sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
} else {
int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(foundIndexOf));
foundTimeSql
.append(whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + " limit " + limitCount);
int indexOf = whereSB.indexOf("and") + "and".length();
sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") ");
if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件
if (whereFoundTime.length() > 0) {
int indexOf = whereFoundTime.indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(indexOf));
foundTimeSql.append(
whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + " limit " + limitCount);
sql.append(" found_time in(" + foundTimeSql + ") ");
} else {
throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件");
}
} else {
int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(foundIndexOf));
foundTimeSql.append(
whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + " limit " + limitCount);
int indexOf = whereSB.indexOf("and") + "and".length();
sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") ");
}
sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
}
sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG") ) {
if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")) {
searchFromLocalCK(page, bean, sql, countSql);
} else {
searchFromDataCenter(page, bean, sql, countSql);
}
}
/**
* 根据ispcode获取当前运营商下所有的entrance_id,device_id,link_id组合
*
* @param ispCode
* @return
*/
private String getIspNum(String ispCode) {
List<String> ispNum = ispInfoDao.getIspNum(ispCode);
if (ispNum != null && ispNum.size() > 0) {
StringBuffer sb = new StringBuffer();
for (String ispNumStr : ispNum) {
sb.append("'");
sb.append(ispNumStr);
sb.append("'");
sb.append(",");
}
return sb.substring(0, sb.length() - 1);
}
return null;
}
}