diff --git a/src/main/java/com/nis/domain/Page.java b/src/main/java/com/nis/domain/Page.java index 366ded9..733c2fb 100644 --- a/src/main/java/com/nis/domain/Page.java +++ b/src/main/java/com/nis/domain/Page.java @@ -641,6 +641,15 @@ public class Page { public int getMaxResults(){ return getPageSize(); } + + + + + public void setLast(int last) { + this.last = last; + } + + /** * @Title: getFiledsSql * @Description: 将fields的属性名称替换为字段名称 diff --git a/src/main/java/com/nis/util/LogJDBCByDruid.java b/src/main/java/com/nis/util/LogJDBCByDruid.java index 7f4d3dc..164de1b 100644 --- a/src/main/java/com/nis/util/LogJDBCByDruid.java +++ b/src/main/java/com/nis/util/LogJDBCByDruid.java @@ -29,11 +29,18 @@ import com.nis.web.service.SpringContextHolder; /** * - *

Title: HiveJDBCByDruid

- *

Description: 使用druid连接池对hive进行查询并解析结果

- *

Company: IIE

+ *

+ * Title: HiveJDBCByDruid + *

+ *

+ * Description: 使用druid连接池对hive进行查询并解析结果 + *

+ *

+ * Company: IIE + *

+ * * @author rkg - * @date 2018年8月20日 + * @date 2018年8月20日 * */ public class LogJDBCByDruid { @@ -169,6 +176,7 @@ public class LogJDBCByDruid { /** * 反射获取类中date类型的字段名称 + * * @param type * @return * @throws Exception @@ -240,7 +248,8 @@ public class LogJDBCByDruid { return map; } - public ResultSet query(String sql) throws Exception { + + public ResultSet query(String sql) throws Exception { conn = getConnection(); logger.info("连接数据中心日志库成功--------------------------"); st = conn.createStatement(); @@ -250,9 +259,30 @@ public class LogJDBCByDruid { // logger.info("选择数据库{}成功,开始执行查询", Constants.HIVEDBNAME); // logger.info("选择数据库{}成功,开始执行查询", Constants.HIVEDBNAME); rs = st.executeQuery(sql); - logger.info("执行查询语句成功sql={}",sql); + logger.info("执行查询语句成功sql={}", sql); return rs; } + public long getCount(String sql) throws Exception { + try { + conn = getConnection(); + st = conn.createStatement(); + rs = st.executeQuery(sql); + String countStr = null; + while (rs.next()) { + countStr = rs.getObject(1).toString(); + break; + } + if (countStr == null || countStr.trim().equals("")) { + logger.info("获取数据中心日志总条数成功总共===================0条配置"); + return 0l; + } else { + return Long.valueOf(countStr); + } + } finally { + closeConn(); + } + } + } diff --git a/src/main/java/com/nis/web/controller/BaseRestController.java b/src/main/java/com/nis/web/controller/BaseRestController.java index f35b091..53a2177 100644 --- a/src/main/java/com/nis/web/controller/BaseRestController.java +++ b/src/main/java/com/nis/web/controller/BaseRestController.java @@ -140,6 +140,7 @@ public class BaseRestController { /** * 不将日志插入到数据库中,只返回到前台 + * * @param time * @param request * @param response @@ -215,7 +216,7 @@ public class BaseRestController { restResult.setMsg(msg); restResult.setTraceCode(thread.getTraceCode()); thread.setConsumerTime(time); -// thread.setExceptionInfo(msg); + // thread.setExceptionInfo(msg); thread.setBusinessCode(restResult.getBusinessCode().getValue()); new Thread(thread).start(); return convert(restResult); @@ -223,6 +224,7 @@ public class BaseRestController { /** * 日志结果响应格式规范 + * * @param auditLogThread * @param executedTime * @param request @@ -408,4 +410,11 @@ public class BaseRestController { return Configurations.getStringProperty(key, defaultTableName); } + protected int getLastPageNum(int totalCount, int pageSize) { + int pageNum = totalCount / pageSize; + if (totalCount % pageSize > 0) { + pageNum++; + } + return pageNum; + } } diff --git a/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java b/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java index 5ea2145..b50d367 100644 --- a/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java +++ b/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java @@ -16,6 +16,7 @@ import org.springframework.web.bind.annotation.RestController; import com.nis.domain.LogEntity; import com.nis.domain.Page; +import com.nis.domain.restful.DkBehaviorLog; import com.nis.domain.restful.NtcAppLog; import com.nis.domain.restful.NtcBgpLog; import com.nis.domain.restful.NtcDdosLog; @@ -61,7 +62,7 @@ import com.zdjizhi.utils.StringUtil; public class NtcLogSearchController extends BaseRestController { @Autowired protected ServicesRequestLogService servicesRequestLogService; - + @Autowired protected NtcLogService ntcLogService; @@ -70,8 +71,8 @@ public class NtcLogSearchController extends BaseRestController { public Map ntcIpLogs(Page page, NtcIpLog ntcIpLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcIpLogPage = null; try { resetTime(ntcIpLog); @@ -85,38 +86,39 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } -// ResultSet rs = HiveSqlService.getResultSet(page, ntcIpLog, -// getTableName(NtcIpLog.class.getSimpleName() + "HiveTable", "ntc_ip_log"), -// getCol2Col(), orderBy, null); -// Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcIpLog.class, "foundTime", -// "recvTime"); + // ResultSet rs = HiveSqlService.getResultSet(page, ntcIpLog, + // getTableName(NtcIpLog.class.getSimpleName() + "HiveTable", "ntc_ip_log"), + // getCol2Col(), orderBy, null); + // Map tableMapping = HiveJDBC.tableMapping(page, null, rs, + // NtcIpLog.class, "foundTime", + // "recvTime"); String sql = HiveSqlService.getSql(page, ntcIpLog, - getTableName(NtcIpLog.class.getSimpleName() + "HiveTable", "ntc_ip_log"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null,sql , NtcIpLog.class, "foundTime", - "recvTime"); + getTableName(NtcIpLog.class.getSimpleName() + "HiveTable", "ntc_ip_log"), getCol2Col(), orderBy, + null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcIpLog.class, + "foundTime", "recvTime"); if (tableMapping == null) { ntcIpLogPage.setList(new ArrayList()); } else { List list = new ArrayList(); list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, NtcIpLog.class); ntcIpLogPage.setList(List); ntcIpLogPage.setCount(List.size()); -// ntcIpLogPage -// .setCount( -// HiveSqlService.getHivePageCount(ntcIpLog, null, -// getTableName( -// NtcIpLog.class.getSimpleName() + "HiveTable", "ntc_ip_log"), -// getCol2Col(), null)); + // ntcIpLogPage + // .setCount( + // HiveSqlService.getHivePageCount(ntcIpLog, null, + // getTableName( + // NtcIpLog.class.getSimpleName() + "HiveTable", "ntc_ip_log"), + // getCol2Col(), null)); } else { ntcIpLogPage.setList(new ArrayList()); ntcIpLogPage.setCount(0L); @@ -140,8 +142,8 @@ public class NtcLogSearchController extends BaseRestController { public Map ntcHttpLogs(Page page, NtcHttpLog ntcHttpLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcHttpLogPage = null; try { @@ -157,19 +159,19 @@ public class NtcLogSearchController extends BaseRestController { orderBy = "found_Time"; } String sql = HiveSqlService.getSql(page, ntcHttpLog, - getTableName(NtcHttpLog.class.getSimpleName() + "HiveTable", "ntc_http_log"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcHttpLog.class, "foundTime", - "recvTime"); + getTableName(NtcHttpLog.class.getSimpleName() + "HiveTable", "ntc_http_log"), getCol2Col(), orderBy, + null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, + NtcHttpLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcHttpLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -200,8 +202,8 @@ public class NtcLogSearchController extends BaseRestController { public Map ntcDnsLogs(Page page, NtcDnsLog ntcDnsLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcDnsLogPage = null; try { @@ -217,19 +219,19 @@ public class NtcLogSearchController extends BaseRestController { orderBy = "found_Time"; } String sql = HiveSqlService.getSql(page, ntcDnsLog, - getTableName(NtcDnsLog.class.getSimpleName() + "HiveTable", "ntc_dns_log"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcDnsLog.class, "foundTime", - "recvTime"); + getTableName(NtcDnsLog.class.getSimpleName() + "HiveTable", "ntc_dns_log"), getCol2Col(), orderBy, + null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcDnsLog.class, + "foundTime", "recvTime"); if (tableMapping == null) { ntcDnsLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -260,8 +262,8 @@ public class NtcLogSearchController extends BaseRestController { public Map ntcMailLogs(Page page, NtcMailLog ntcMailLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcMailLogPage = null; try { @@ -277,19 +279,19 @@ public class NtcLogSearchController extends BaseRestController { orderBy = "found_Time"; } String sql = HiveSqlService.getSql(page, ntcMailLog, - getTableName(NtcMailLog.class.getSimpleName() + "HiveTable", "ntc_mail_log"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcMailLog.class, "foundTime", - "recvTime"); + getTableName(NtcMailLog.class.getSimpleName() + "HiveTable", "ntc_mail_log"), getCol2Col(), orderBy, + null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, + NtcMailLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcMailLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -320,8 +322,8 @@ public class NtcLogSearchController extends BaseRestController { public Map ntcSslLogs(Page page, NtcSslLog ntcSslLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcSslLogPage = null; try { @@ -337,19 +339,19 @@ public class NtcLogSearchController extends BaseRestController { orderBy = "found_Time"; } String sql = HiveSqlService.getSql(page, ntcSslLog, - getTableName(NtcSslLog.class.getSimpleName() + "HiveTable", "ntc_ssl_log"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcSslLog.class, "foundTime", - "recvTime"); + getTableName(NtcSslLog.class.getSimpleName() + "HiveTable", "ntc_ssl_log"), getCol2Col(), orderBy, + null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcSslLog.class, + "foundTime", "recvTime"); if (tableMapping == null) { ntcSslLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -380,8 +382,8 @@ public class NtcLogSearchController extends BaseRestController { public Map ntcPptpLogs(Page page, NtcPptpLog ntcPptpLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcPptpLogPage = null; try { @@ -397,19 +399,19 @@ public class NtcLogSearchController extends BaseRestController { orderBy = "found_Time"; } String sql = HiveSqlService.getSql(page, ntcPptpLog, - getTableName(NtcPptpLog.class.getSimpleName() + "HiveTable", "ntc_pptp_log"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcPptpLog.class, "foundTime", - "recvTime"); + getTableName(NtcPptpLog.class.getSimpleName() + "HiveTable", "ntc_pptp_log"), getCol2Col(), orderBy, + null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, + NtcPptpLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcPptpLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -439,8 +441,8 @@ public class NtcLogSearchController extends BaseRestController { public Map ntcL2tpLogs(Page page, NtcL2tpLog ntcL2tpLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcL2tpLogPage = null; try { @@ -456,19 +458,19 @@ public class NtcLogSearchController extends BaseRestController { orderBy = "found_Time"; } String sql = HiveSqlService.getSql(page, ntcL2tpLog, - getTableName(NtcL2tpLog.class.getSimpleName() + "HiveTable", "ntc_l2tp_log"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcL2tpLog.class, "foundTime", - "recvTime"); + getTableName(NtcL2tpLog.class.getSimpleName() + "HiveTable", "ntc_l2tp_log"), getCol2Col(), orderBy, + null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, + NtcL2tpLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcL2tpLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -499,8 +501,8 @@ public class NtcLogSearchController extends BaseRestController { public Map ntcOpenvpnLogs(Page page, NtcOpenvpnLog ntcOpenvpnLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcOpenvpnLogPage = null; try { @@ -516,19 +518,19 @@ public class NtcLogSearchController extends BaseRestController { orderBy = "found_Time"; } String sql = HiveSqlService.getSql(page, ntcOpenvpnLog, - getTableName(NtcOpenvpnLog.class.getSimpleName() + "HiveTable", "ntc_openvpn_log"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcOpenvpnLog.class, "foundTime", - "recvTime"); + getTableName(NtcOpenvpnLog.class.getSimpleName() + "HiveTable", "ntc_openvpn_log"), getCol2Col(), + orderBy, null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, + NtcOpenvpnLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcOpenvpnLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -559,8 +561,8 @@ public class NtcLogSearchController extends BaseRestController { public Map ntcIpsecLogs(Page page, NtcIpsecLog ntcIpsecLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcIpsecLogPage = null; try { @@ -576,19 +578,19 @@ public class NtcLogSearchController extends BaseRestController { orderBy = "found_Time"; } String sql = HiveSqlService.getSql(page, ntcIpsecLog, - getTableName(NtcIpsecLog.class.getSimpleName() + "HiveTable", "ntc_ipsec_log"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcIpsecLog.class, "foundTime", - "recvTime"); + getTableName(NtcIpsecLog.class.getSimpleName() + "HiveTable", "ntc_ipsec_log"), getCol2Col(), + orderBy, null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, + NtcIpsecLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcIpsecLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -619,8 +621,8 @@ public class NtcLogSearchController extends BaseRestController { public Map ntcSshLogs(Page page, NtcSshLog ntcSshLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcSshLogPage = null; try { @@ -636,19 +638,19 @@ public class NtcLogSearchController extends BaseRestController { orderBy = "found_Time"; } String sql = HiveSqlService.getSql(page, ntcSshLog, - getTableName(NtcSshLog.class.getSimpleName() + "HiveTable", "ntc_ssh_log"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcSshLog.class, "foundTime", - "recvTime"); + getTableName(NtcSshLog.class.getSimpleName() + "HiveTable", "ntc_ssh_log"), getCol2Col(), orderBy, + null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcSshLog.class, + "foundTime", "recvTime"); if (tableMapping == null) { ntcSshLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -679,8 +681,8 @@ public class NtcLogSearchController extends BaseRestController { public Map ntcFtpLogs(Page page, NtcFtpLog ntcFtpLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcFtpLogPage = null; try { @@ -696,19 +698,19 @@ public class NtcLogSearchController extends BaseRestController { orderBy = "found_Time"; } String sql = HiveSqlService.getSql(page, ntcFtpLog, - getTableName(NtcFtpLog.class.getSimpleName() + "HiveTable", "ntc_ftp_log"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcFtpLog.class, "foundTime", - "recvTime"); + getTableName(NtcFtpLog.class.getSimpleName() + "HiveTable", "ntc_ftp_log"), getCol2Col(), orderBy, + null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcFtpLog.class, + "foundTime", "recvTime"); if (tableMapping == null) { ntcFtpLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -732,13 +734,14 @@ public class NtcLogSearchController extends BaseRestController { return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "FTP日志检索成功", ntcFtpLogPage, 0); } + @RequestMapping(value = "/ntcAppLogs", method = RequestMethod.GET) @ApiOperation(value = "App日志查询", httpMethod = "GET", notes = "对应配置为“App管理”，存储动作为阻断与监测的命中日志。对日志功能“APP策略日志”提供数据基础查询服务") public Map ntcAppLogs(Page page, NtcAppLog ntcAppLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcAppLogPage = null; try { @@ -754,19 +757,19 @@ public class NtcLogSearchController extends BaseRestController { orderBy = "found_Time"; } String sql = HiveSqlService.getSql(page, ntcAppLog, - getTableName(NtcAppLog.class.getSimpleName() + "HiveTable", "ntc_app_log"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcAppLog.class, "foundTime", - "recvTime"); + getTableName(NtcAppLog.class.getSimpleName() + "HiveTable", "ntc_app_log"), getCol2Col(), orderBy, + null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcAppLog.class, + "foundTime", "recvTime"); if (tableMapping == null) { ntcAppLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -790,13 +793,14 @@ public class NtcLogSearchController extends BaseRestController { return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "App日志检索成功", ntcAppLogPage, 0); } + @RequestMapping(value = "/ntcDdosLogs", method = RequestMethod.GET) @ApiOperation(value = "DDos日志查询", httpMethod = "GET", notes = "对应配置为“DDOS日志监控”，存储动作为丢弃的命中日志。对日志功能“DDOS日志监控”提供数据基础查询服务") public Map ntcDdosLogs(Page page, NtcDdosLog ntcDdosLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcDdosLogPage = null; try { @@ -812,19 +816,19 @@ public class NtcLogSearchController extends BaseRestController { orderBy = "found_Time"; } String sql = HiveSqlService.getSql(page, ntcDdosLog, - getTableName(NtcDdosLog.class.getSimpleName() + "HiveTable", "ntc_ddos_log"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcDdosLog.class, "foundTime", - "recvTime"); + getTableName(NtcDdosLog.class.getSimpleName() + "HiveTable", "ntc_ddos_log"), getCol2Col(), orderBy, + null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, + NtcDdosLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcDdosLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -848,14 +852,14 @@ public class NtcLogSearchController extends BaseRestController { return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "DDos日志检索成功", ntcDdosLogPage, 0); } - + @RequestMapping(value = "/ntcP2pLogs", method = RequestMethod.GET) @ApiOperation(value = "P2P日志查询", httpMethod = "GET", notes = "对应配置为“文件传输管理-P2P”，存储动作为阻断的命中日志。对日志功能“文件传输管理-P2P日志”提供数据基础查询服务") public Map ntcP2pLogs(Page page, NtcP2pLog ntcP2pLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcP2pLogPage = null; try { @@ -870,22 +874,22 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - + String sql = HiveSqlService.getSql(page, ntcP2pLog, - getTableName(NtcP2pLog.class.getSimpleName() + "HiveTable", "ntc_p2p_log"), - getCol2Col(), orderBy, null); - - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcP2pLog.class, "foundTime", - "recvTime"); + getTableName(NtcP2pLog.class.getSimpleName() + "HiveTable", "ntc_p2p_log"), getCol2Col(), orderBy, + null); + + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcP2pLog.class, + "foundTime", "recvTime"); if (tableMapping == null) { ntcP2pLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -909,14 +913,14 @@ public class NtcLogSearchController extends BaseRestController { return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "P2P日志检索成功", ntcP2pLogPage, 0); } - + @RequestMapping(value = "/ntcBgpLogs", method = RequestMethod.GET) @ApiOperation(value = "BGP日志查询", httpMethod = "GET", notes = "对应配置为“其他-BGP”，存储动作为阻断和监测的命中日志。对日志功能“BGP日志”提供数据基础查询服务") public Map ntcBgpLogs(Page page, NtcBgpLog ntcBgpLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page ntcBgpLogPage = null; try { @@ -931,22 +935,22 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - + String sql = HiveSqlService.getSql(page, ntcBgpLog, - getTableName(NtcBgpLog.class.getSimpleName() + "HiveTable", "ntc_bgp_log"), - getCol2Col(), orderBy, null); - - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcBgpLog.class, "foundTime", - "recvTime"); + getTableName(NtcBgpLog.class.getSimpleName() + "HiveTable", "ntc_bgp_log"), getCol2Col(), orderBy, + null); + + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcBgpLog.class, + "foundTime", "recvTime"); if (tableMapping == null) { ntcBgpLogPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); -// if (tableMapping.get("obj").size() > page.getPageSize()) { -// list = tableMapping.get("obj").subList(0, page.getPageSize()); -// } else { -// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); -// } + // if (tableMapping.get("obj").size() > page.getPageSize()) { + // list = tableMapping.get("obj").subList(0, page.getPageSize()); + // } else { + // list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size()); + // } if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); List List = (java.util.List) JsonMapper.fromJsonList(jsonString, @@ -970,14 +974,14 @@ public class NtcLogSearchController extends BaseRestController { return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "BGP日志检索成功", ntcBgpLogPage, 0); } - + @RequestMapping(value = "/ntcVoipLogs", method = RequestMethod.GET) @ApiOperation(value = "NTC VoIP日志查询", httpMethod = "GET", notes = "对日志功能“NTC VoIP日志”提供数据基础查询服务") public Map ntcVoipLogs(Page page, NtcVoipLog ntcVoipLog, Model model, HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page logPage = null; try { @@ -993,10 +997,10 @@ public class NtcLogSearchController extends BaseRestController { orderBy = "found_Time"; } String sql = HiveSqlService.getSql(page, ntcVoipLog, - getTableName(NtcVoipLog.class.getSimpleName() + "HiveTable", "NTC_VOIP_LOG"), - getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcVoipLog.class, "foundTime", - "recvTime"); + getTableName(NtcVoipLog.class.getSimpleName() + "HiveTable", "NTC_VOIP_LOG"), getCol2Col(), orderBy, + null); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, + NtcVoipLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); } else { @@ -1024,19 +1028,20 @@ public class NtcLogSearchController extends BaseRestController { return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "NTC VoIP日志检索成功", logPage, 0); } - + @RequestMapping(value = "/ntcStreamMediaLogs", method = RequestMethod.GET) @ApiOperation(value = "流媒体协议日志查询", httpMethod = "GET", notes = "对日志功能“流媒体协议日志”提供数据基础查询服务") - public Map ntcStreamMediaLogs(Page page, NtcStreamingMediaLog ntcStreamMediaLog, Model model, HttpServletRequest request, - HttpServletResponse response) { + public Map ntcStreamMediaLogs(Page page, NtcStreamingMediaLog ntcStreamMediaLog, Model model, + HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); Page logPage = null; try { resetTime(ntcStreamMediaLog); - ntcLogService.queryConditionCheck(auditLogThread, start, ntcStreamMediaLog, NtcStreamingMediaLog.class, page); + ntcLogService.queryConditionCheck(auditLogThread, start, ntcStreamMediaLog, NtcStreamingMediaLog.class, + page); logPage = new Page(); logPage.setPageNo(page.getPageNo()); logPage.setPageSize(page.getPageSize()); @@ -1049,16 +1054,16 @@ public class NtcLogSearchController extends BaseRestController { String sql = HiveSqlService.getSql(page, ntcStreamMediaLog, getTableName(NtcStreamingMediaLog.class.getSimpleName() + "HiveTable", "NTC_STREAMING_MEDIA_LOG"), getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcStreamingMediaLog.class, "foundTime", - "recvTime"); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, + NtcStreamingMediaLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); - List List = (java.util.List) JsonMapper.fromJsonList(jsonString, - NtcStreamingMediaLog.class); + List List = (java.util.List) JsonMapper + .fromJsonList(jsonString, NtcStreamingMediaLog.class); logPage.setList(List); logPage.setCount(List.size()); @@ -1075,18 +1080,18 @@ public class NtcLogSearchController extends BaseRestController { } throw ((RestServiceException) e); } - return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "流媒体协议日志检索成功", - logPage, 0); + return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "流媒体协议日志检索成功", logPage, + 0); } - + @RequestMapping(value = "/ntcKeywordsUrlLogs", method = RequestMethod.GET) @ApiOperation(value = "NTC关键字转URL日志查询", httpMethod = "GET", notes = "对日志功能“NTC关键字转URL日志”提供数据基础查询服务") - public Map ntcKeywordsUrlLogs(Page page, NtcKeywordsUrlLog ntcKeywordsUrlLog, Model model, HttpServletRequest request, - HttpServletResponse response) { + public Map ntcKeywordsUrlLogs(Page page, NtcKeywordsUrlLog ntcKeywordsUrlLog, Model model, + HttpServletRequest request, HttpServletResponse response) { long start = System.currentTimeMillis(); - AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, - request, null); - + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); + Page logPage = null; try { resetTime(ntcKeywordsUrlLog); @@ -1103,19 +1108,23 @@ public class NtcLogSearchController extends BaseRestController { String sql = HiveSqlService.getSql(page, ntcKeywordsUrlLog, getTableName(NtcKeywordsUrlLog.class.getSimpleName() + "HiveTable", "NTC_KEYWORDS_URL_LOG"), getCol2Col(), orderBy, null); - Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, NtcKeywordsUrlLog.class, "foundTime", - "recvTime"); + Map> tableMapping = new LogJDBCByDruid().tableMapping(page, null, sql, + NtcKeywordsUrlLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); } else { List list = tableMapping.get("obj"); if (list.size() > 0) { String jsonString = JsonMapper.toJsonString(list); - List List = (java.util.List) JsonMapper.fromJsonList(jsonString, - NtcKeywordsUrlLog.class); + List List = (java.util.List) JsonMapper + .fromJsonList(jsonString, NtcKeywordsUrlLog.class); logPage.setList(List); - logPage.setCount(List.size()); - + Long hivePageCount = HiveSqlService.getLogCount(ntcKeywordsUrlLog, + getTableName(NtcKeywordsUrlLog.class.getSimpleName() + "HiveTable", "NTC_KEYWORDS_URL_LOG"), + getCol2Col()); + + logPage.setLast(getLastPageNum(hivePageCount.intValue(), page.getPageSize())); + logPage.setCount(hivePageCount); } else { logPage.setList(new ArrayList()); } @@ -1132,14 +1141,16 @@ public class NtcLogSearchController extends BaseRestController { return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "NTC关键字转URL日志检索成功", logPage, 0); } - + /** - *判断开始和结束时间是否为null,如果为null则初始化时间 + * 判断开始和结束时间是否为null,如果为null则初始化时间 + * * @param entity * @throws Exception */ public void resetTime(LogEntity entity) throws Exception { - if (StringUtil.isEmpty(entity.getSearchFoundStartTime())&& StringUtil.isEmpty(entity.getSearchFoundEndTime())) { + if (StringUtil.isEmpty(entity.getSearchFoundStartTime()) + && StringUtil.isEmpty(entity.getSearchFoundEndTime())) { Map map = DateUtils.getLocalTime(entity.getSearchFoundStartTime(), entity.getSearchFoundEndTime(), Constants.LOG_LOCAL_TIME, "minute"); entity.setSearchFoundStartTime(map.get("startTime")); @@ -1149,6 +1160,7 @@ public class NtcLogSearchController extends BaseRestController { /** * 将searchFoundStartTime,searchFoundEndTime与foundTime进行关联 + * * @return */ public Map> getCol2Col() { diff --git a/src/main/java/com/nis/web/service/HiveSqlService.java b/src/main/java/com/nis/web/service/HiveSqlService.java index 07cf736..a4fcac6 100644 --- a/src/main/java/com/nis/web/service/HiveSqlService.java +++ b/src/main/java/com/nis/web/service/HiveSqlService.java @@ -50,6 +50,15 @@ public class HiveSqlService { public static String getSql(Page page, Object bean, String tableName, Map> col2col, String orderBy, String searchActiveSys) throws Exception { + if (Constants.ISUSECLICKHOUSE) { + return getSqlByClickHouse(page, bean, tableName, col2col, orderBy, searchActiveSys); + } else { + return getSqlByHive(page, bean, tableName, col2col, orderBy, searchActiveSys); + } + } + + public static String getSqlByClickHouse(Page page, Object bean, String tableName, + Map> col2col, String orderBy, String searchActiveSys) throws Exception { tableName = tableName.toLowerCase(); String showColmun = getFiledsSql(bean.getClass().getSimpleName(), page.getFields()); StringBuffer sql = new StringBuffer(); @@ -73,9 +82,107 @@ public class HiveSqlService { if (!StringUtil.isEmpty(bean)) { Class clazz = bean.getClass(); Map filedsType = null; - if (Constants.ISUSECLICKHOUSE) { - filedsType = getFiledsType(bean); + filedsType = getFiledsType(bean); + for (; clazz != Object.class; clazz = clazz.getSuperclass()) { + Field[] fields = clazz.getDeclaredFields(); + for (int i = 0; i < fields.length; i++) { + // 现在gwall日志表结构中只有数值和字符串两种类型,数值都是int类型没有bigint所以不需要加L, + Field f = fields[i]; + String key = f.getName();// 获取字段名 + if (f.getType().getName().equals("java.lang.String") && key.startsWith("search")) { + Object value = getFieldValue(bean, key); + if (!StringUtil.isEmpty(value)) { + setFieldValue(bean, key, value.toString().trim()); + if (key.endsWith("Time")) {// 日期开始或结束的字段 + if (col2col.containsKey(key)) { + Long partition = Long.parseLong(sdf2.format(sdf.parse(value.toString().trim()))); + value = sdf.parse(value.toString().trim()).getTime() / 1000; + if (col2col.get(key).get("start") != null) { + whereSB.append(" and " + + filedAndColumnMap.get(col2col.get(key).get("start")).toLowerCase() + + ">=" + value); + } else { + whereSB.append(" and " + + filedAndColumnMap.get(col2col.get(key).get("end")).toLowerCase() + "<" + + value); + } + } + } else { + if (key.toLowerCase().startsWith("search")) { + key = key.replace("search", ""); + key = key.substring(0, 1).toLowerCase() + key.substring(1); + } + + // clickhouse写法 + String type = filedsType.get(key).trim(); + if (type.equals("java.lang.String")) { + String field = filedAndColumnMap.get(key).toLowerCase(); + if (field.equals("url")) { + whereSB.append(" and " + field + " like '" + value.toString().trim() + "%'"); + } else { + whereSB.append(" and " + field + "='" + value.toString().trim() + "'"); + } + } else if (type.equals("java.lang.Integer") || type.equals("int") + || type.equals("java.lang.Long") || type.equals("long")) { + whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "=" + + value.toString().trim()); + } + + } + } + + } + } + } + } + if (whereSB.length() > 0) { + int indexOf = whereSB.indexOf("and") + "and".length(); + sql.append(" where " + whereSB.substring(indexOf)); + } + Integer startNum = (page.getPageNo() - 1) * page.getPageSize(); + if (orderBy.toLowerCase().contains("asc") || orderBy.toLowerCase().contains("desc")) { + sql.append(" order by " + orderBy.toLowerCase()); + } else { + sql.append(" order by " + orderBy.toLowerCase() + " desc"); + } + sql.append(" limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 + + return sql.toString(); + } + + public static String getSqlByHive(Page page, Object bean, String tableName, + Map> col2col, String orderBy, String searchActiveSys) throws Exception { + tableName = tableName.toLowerCase(); + String showColmun = getFiledsSql(bean.getClass().getSimpleName(), page.getFields()); + StringBuffer sql = new StringBuffer(); + Map filedAndColumnMap = getFiledAndColumnMap(bean.getClass()); + if (null == showColmun || showColmun.equals("")) { + for (String key : filedAndColumnMap.keySet()) { + if (!filedAndColumnMap.get(key).toLowerCase().equals("id")) { + sql.append(filedAndColumnMap.get(key) + ","); + } + } + } else { + sql.append(showColmun); + } + String sqlTrim = sql.toString().trim(); + if (sqlTrim.endsWith(",")) { + sqlTrim = sqlTrim.substring(0, sqlTrim.length() - 1); + } + if (orderBy.toLowerCase().contains("asc") || orderBy.toLowerCase().contains("desc")) { + orderBy = " order by " + orderBy; + } else { + orderBy = " order by " + orderBy + " desc "; + } + sql.setLength(0); + sql.append(" select " + sqlTrim.toLowerCase() + " from (select " + sqlTrim.toLowerCase() + + ",row_number() over(partition by found_time_partition " + orderBy + ") as row_num from " + + tableName.toLowerCase() + " "); + StringBuffer whereSB = new StringBuffer(); + if (!StringUtil.isEmpty(bean)) { + Class clazz = bean.getClass(); + Map filedsType = null; for (; clazz != Object.class; clazz = clazz.getSuperclass()) { // 获取所有的字段包括public,private,protected,private // Field[] fields = bean.getClass().getDeclaredFields(); @@ -126,53 +233,35 @@ public class HiveSqlService { key = key.replace("search", ""); key = key.substring(0, 1).toLowerCase() + key.substring(1); } - if (!Constants.ISUSECLICKHOUSE) {// hive写法 - if (typeName.equals("java.lang.String")) { - String field = filedAndColumnMap.get(key); - if (field.equals("url")) { - whereSB.append( - " and " + field + " like '" + value.toString().trim() + "%'"); - } else { - whereSB.append(" and " + field + "='" + value.toString().trim() + "'"); - } - } else if (typeName.equals("java.lang.Integer") || typeName.equals("int")) { - whereSB.append( - " and " + filedAndColumnMap.get(key) + "=" + value.toString().trim()); - } else if (typeName.equals("java.lang.Long") || typeName.equals("long")) { - whereSB.append(" and " + filedAndColumnMap.get(key) + "=" - + value.toString().trim() + "L"); - } - } else {// clickhouse写法 - String type = filedsType.get(key).trim(); - if (type.equals("java.lang.String")) { - String field = filedAndColumnMap.get(key).toLowerCase(); - if (field.equals("url")) { - whereSB.append( - " and " + field + " like '" + value.toString().trim() + "%'"); - } else { - whereSB.append(" and " + field + "='" + value.toString().trim() + "'"); - } - } else if (type.equals("java.lang.Integer") || type.equals("int") - || type.equals("java.lang.Long") || type.equals("long")) { - whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "=" - + value.toString().trim()); + if (typeName.equals("java.lang.String")) { + String field = filedAndColumnMap.get(key); + if (field.equals("url")) { + whereSB.append(" and " + field + " like '" + value.toString().trim() + "%'"); + } else { + whereSB.append(" and " + field + "='" + value.toString().trim() + "'"); } + } else if (typeName.equals("java.lang.Integer") || typeName.equals("int")) { + whereSB.append( + " and " + filedAndColumnMap.get(key) + "=" + value.toString().trim()); + + } else if (typeName.equals("java.lang.Long") || typeName.equals("long")) { + whereSB.append( + " and " + filedAndColumnMap.get(key) + "=" + value.toString().trim() + "L"); } + } } } } - if (!Constants.ISUSECLICKHOUSE) {// hive需要加这个字段 - if (null != foundTimePartStart) { - // sql.append(" and found_time_partition>=" + foundTimePartStart + "L"); - whereSB.append(" and found_time_partition>=" + foundTimePartStart); - } - if (null != foundTimePartEnd) { - // sql.append(" and found_time_partition<" + foundTimePartEnd + "L"); - whereSB.append(" and found_time_partition<=" + foundTimePartEnd); - } + if (null != foundTimePartStart) { + // sql.append(" and found_time_partition>=" + foundTimePartStart + "L"); + whereSB.append(" and found_time_partition>=" + foundTimePartStart); + } + if (null != foundTimePartEnd) { + // sql.append(" and found_time_partition<" + foundTimePartEnd + "L"); + whereSB.append(" and found_time_partition<=" + foundTimePartEnd); } } @@ -182,32 +271,101 @@ public class HiveSqlService { int indexOf = whereSB.indexOf("and") + "and".length(); sql.append(" where " + whereSB.substring(indexOf)); } - if (Constants.ISUSECLICKHOUSE) { - // Integer startNum = (page.getPageNo() - 1) * page.getPageSize() + 1; - Integer startNum = (page.getPageNo() - 1) * page.getPageSize(); - // Integer endNum = startNum - 1 + page.getPageSize(); - if (orderBy.toLowerCase().contains("asc") || orderBy.toLowerCase().contains("desc")) { - sql.append(" order by " + orderBy.toLowerCase()); - } else { - sql.append(" order by " + orderBy.toLowerCase() + " desc"); - } - sql.append(" limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 - - } else { - // sql.append(" order by " + orderBy + " limit 10000) t1) t2 where - // row_Num between " + startNum + " and " + endNum); - sql.append(" limit " + Constants.EVERY_GETHIVEDATANUM); - logger.info("获取数据中心日志sql==================={}", sql); - } + Integer startNum = (page.getPageNo() - 1) * page.getPageSize() + 1; + Integer endNum = startNum - 1 + page.getPageSize(); + sql.append(" ) t where row_Num between " + startNum + " and " + endNum); + logger.info("获取数据中心日志sql==================={}", sql); return sql.toString(); } - public static Long getHivePageCount(Object bean, String countKey, String tableName, - Map> col2col, String searchActiveSys) throws Exception { + public static Long getLogCount(Object bean, String tableName, Map> col2col) throws Exception { + if (Constants.ISUSECLICKHOUSE) { + return getLogCountFromClickHouse(bean, tableName, col2col); + } else { + return getLogCountFromHive(bean, tableName, col2col); + } + } + + public static Long getLogCountFromClickHouse(Object bean, String tableName, + Map> col2col) throws Exception { tableName = tableName.toLowerCase(); StringBuffer sql = new StringBuffer(); Map filedAndColumnMap = getFiledAndColumnMap(bean.getClass()); - sql.append("select count(1) from " + tableName + " where 1=1 "); + sql.append("select count(1) from " + tableName + " "); + StringBuffer whereSB = new StringBuffer(); + if (!StringUtil.isEmpty(bean)) { + Class clazz = bean.getClass(); + Map filedsType = null; + filedsType = getFiledsType(bean); + for (; clazz != Object.class; clazz = clazz.getSuperclass()) { + Field[] fields = clazz.getDeclaredFields(); + for (int i = 0; i < fields.length; i++) { + // 现在gwall日志表结构中只有数值和字符串两种类型,数值都是int类型没有bigint所以不需要加L, + Field f = fields[i]; + String key = f.getName();// 获取字段名 + if (f.getType().getName().equals("java.lang.String") && key.startsWith("search")) { + Object value = getFieldValue(bean, key); + if (!StringUtil.isEmpty(value)) { + setFieldValue(bean, key, value.toString().trim()); + if (key.endsWith("Time")) {// 日期开始或结束的字段 + if (col2col.containsKey(key)) { + Long partition = Long.parseLong(sdf2.format(sdf.parse(value.toString().trim()))); + value = sdf.parse(value.toString().trim()).getTime() / 1000; + if (col2col.get(key).get("start") != null) { + whereSB.append(" and " + + filedAndColumnMap.get(col2col.get(key).get("start")).toLowerCase() + + ">=" + value); + } else { + whereSB.append(" and " + + filedAndColumnMap.get(col2col.get(key).get("end")).toLowerCase() + "<" + + value); + } + } + } else { + if (key.toLowerCase().startsWith("search")) { + key = key.replace("search", ""); + key = key.substring(0, 1).toLowerCase() + key.substring(1); + } + + // clickhouse写法 + String type = filedsType.get(key).trim(); + if (type.equals("java.lang.String")) { + String field = filedAndColumnMap.get(key).toLowerCase(); + if (field.equals("url")) { + whereSB.append(" and " + field + " like '" + value.toString().trim() + "%'"); + } else { + whereSB.append(" and " + field + "='" + value.toString().trim() + "'"); + } + } else if (type.equals("java.lang.Integer") || type.equals("int") + || type.equals("java.lang.Long") || type.equals("long")) { + whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "=" + + value.toString().trim()); + } + + } + } + + } + } + + } + } + if (whereSB.length() > 0) { + int indexOf = whereSB.indexOf("and") + "and".length(); + sql.append(" where " + whereSB.substring(indexOf)); + } + logger.info("获取数据中心日志总条数sql==================" + sql.toString()); + Long count = new LogJDBCByDruid().getCount(sql.toString()); + return count; + } + + public static Long getLogCountFromHive(Object bean, String tableName, Map> col2col) + throws Exception { + tableName = tableName.toLowerCase(); + StringBuffer sql = new StringBuffer(); + Map filedAndColumnMap = getFiledAndColumnMap(bean.getClass()); + sql.append("select count(1) from " + tableName + " "); + StringBuffer whereSB = new StringBuffer(); if (bean != null) { Class clazz = bean.getClass(); for (; clazz != Object.class; clazz = clazz.getSuperclass()) { @@ -227,12 +385,13 @@ public class HiveSqlService { setFieldValue(bean, key, value.toString().trim()); if (key.endsWith("Time")) {// 日期开始或结束的字段 if (col2col.containsKey(key)) { + Long partition = Long.parseLong(sdf2.format(sdf.parse(value.toString().trim()))); value = sdf.parse(value.toString().trim()).getTime() / 1000; if (key.toLowerCase().equals("searchfoundstarttime")) { - foundTimePartStart = Long.parseLong(value.toString()) / 3600L / 24L; + foundTimePartStart = partition; } if (key.toLowerCase().equals("searchfoundendtime")) { - foundTimePartEnd = Long.parseLong(value.toString()) / 3600L / 24L; + foundTimePartEnd = partition; } if (col2col.get(key).get("start") != null) { // sql.append(" and " + @@ -240,15 +399,17 @@ public class HiveSqlService { // + ">=to_date('" + // value.toString().trim() // + "','yyyy-mm-dd HH24:mi:ss')"); - sql.append(" and " + filedAndColumnMap.get(col2col.get(key).get("start")) + ">=" - + value); + whereSB.append(" and " + + filedAndColumnMap.get(col2col.get(key).get("start")).toLowerCase() + + ">=" + value); } else { // sql.append(" and " + // filedAndColumnMap.get(col2col.get(key).get("end")) // + "<=to_date('" + // value.toString().trim() // + "','yyyy-mm-dd HH24:mi:ss')"); - sql.append(" and " + filedAndColumnMap.get(col2col.get(key).get("end")) + "<" + whereSB.append(" and " + + filedAndColumnMap.get(col2col.get(key).get("end")).toLowerCase() + "<" + value); } } @@ -259,15 +420,21 @@ public class HiveSqlService { } if (typeName.equals("java.lang.String")) { - sql.append(" and " + filedAndColumnMap.get(key) + "='" + value.toString().trim() - + "'"); + String field = filedAndColumnMap.get(key); + if (field.equals("url")) { + whereSB.append(" and " + field + " like '" + value.toString().trim() + "%'"); + } else { + whereSB.append(" and " + field + "='" + value.toString().trim() + "'"); + } } else if (typeName.equals("java.lang.Integer") || typeName.equals("int")) { - sql.append(" and " + filedAndColumnMap.get(key) + "=" + value.toString().trim()); + whereSB.append( + " and " + filedAndColumnMap.get(key) + "=" + value.toString().trim()); } else if (typeName.equals("java.lang.Long") || typeName.equals("long")) { - sql.append( + whereSB.append( " and " + filedAndColumnMap.get(key) + "=" + value.toString().trim() + "L"); } + } } @@ -275,33 +442,21 @@ public class HiveSqlService { } if (null != foundTimePartStart) { // sql.append(" and found_time_partition>=" + foundTimePartStart + "L"); - sql.append(" and found_time_partition>=" + foundTimePartStart); + whereSB.append(" and found_time_partition>=" + foundTimePartStart); } if (null != foundTimePartEnd) { // sql.append(" and found_time_partition<" + foundTimePartEnd + "L"); - sql.append(" and found_time_partition<" + foundTimePartEnd); + whereSB.append(" and found_time_partition<=" + foundTimePartEnd); } } } + if (whereSB.length() > 0) { + int indexOf = whereSB.indexOf("and") + "and".length(); + sql.append(" where " + whereSB.substring(indexOf)); + } logger.info("获取数据中心日志总条数sql==================" + sql.toString()); - // ResultSet countRs = HiveJDBC.query(countSql.toString()); - ResultSet countRs = new LogJDBCByDruid().query(sql.toString()); - String countStr = null; - while (countRs.next()) { - countStr = countRs.getObject(1).toString(); - break; - } - if (countStr == null || countStr.equals("")) { - logger.info("获取数据中心日志总条数成功总共===================0条配置"); - return 0l; - } - Long count = Long.valueOf(countStr); - logger.info("获取数据中心日志总条数成功总共===================" + count + "条配置"); - // HiveJDBC.closeConn(); - if (Constants.IS_OPEN_REDIS && Constants.DATACENTER_OPEN_REDIS) { - new SaveRedisThread(countKey, count, Constants.HIVE_EXPIRE).start(); - } + Long count = new LogJDBCByDruid().getCount(sql.toString()); return count; }