diff --git a/pom.xml b/pom.xml index da62685..9a06bd7 100644 --- a/pom.xml +++ b/pom.xml @@ -637,6 +637,14 @@ fastdfs-client-java 1.27-SNAPSHOT + + + + com.alibaba + druid + 1.1.10 + + diff --git a/src/main/java/com/nis/util/HiveJDBC.java b/src/main/java/com/nis/util/HiveJDBC.java index 81ce4ec..5ee83d0 100644 --- a/src/main/java/com/nis/util/HiveJDBC.java +++ b/src/main/java/com/nis/util/HiveJDBC.java @@ -40,24 +40,16 @@ public class HiveJDBC { static { try { prop.load(Configurations.class.getResourceAsStream("/jdbc.properties")); + driverName = prop.getProperty("jdbc.hive.driver").trim(); + url = prop.getProperty("jdbc.hive.url").trim(); + username = prop.getProperty("jdbc.hive.username").trim(); + password = prop.getProperty("jdbc.hive.password").trim(); } catch (IOException e) { e.printStackTrace(); } } - public static void getConn(String searchActiveSys) throws Exception { - - if (null != searchActiveSys && searchActiveSys.equals("4")) { - driverName = prop.getProperty("jdbc.hiveA.driver").trim(); - url = prop.getProperty("jdbc.hiveA.url").trim(); - username = prop.getProperty("jdbc.hiveA.username").trim(); - password = prop.getProperty("jdbc.hiveA.password").trim(); - } else { - driverName = prop.getProperty("jdbc.hiveB.driver").trim(); - url = prop.getProperty("jdbc.hiveB.url").trim(); - username = prop.getProperty("jdbc.hiveB.username").trim(); - password = prop.getProperty("jdbc.hiveB.password").trim(); - } + public static void getConn() throws Exception { Class.forName(driverName); conn = DriverManager.getConnection(url, username, password); @@ -65,7 +57,7 @@ public class HiveJDBC { public static ResultSet query(String sql, String searchActiveSys) throws Exception { logger.info("开始连接数据中心日志库--------------------------"); - getConn(searchActiveSys); + getConn(); logger.info("连接数据中心日志库成功--------------------------"); st = conn.createStatement(); if (null != searchActiveSys && searchActiveSys.equals("4")) { @@ -135,15 +127,15 @@ public class HiveJDBC { Map filedAndColumnMap = getColumn2FiledMap(entityClass); List listString = new ArrayList(); List listObject = new ArrayList(); - //不从Object... obj中获取需要date类型的字段了,调用的时候容易漏写,改为反射获取date类型的字段 - List columnList =getDateColumn(entityClass); -// List columnList =null; -// if (null != obj && obj.length > 0) { -// columnList = new ArrayList(); -// for (int i = 0; i < obj.length; i++) { -// columnList.add(obj[i].toString().toLowerCase()); -// } -// } + // 不从Object... obj中获取需要date类型的字段了,调用的时候容易漏写,改为反射获取date类型的字段 + List columnList = getDateColumn(entityClass); + // List columnList =null; + // if (null != obj && obj.length > 0) { + // columnList = new ArrayList(); + // for (int i = 0; i < obj.length; i++) { + // columnList.add(obj[i].toString().toLowerCase()); + // } + // } // ResultSet rs = HiveJDBC.query(sql.toString()); ResultSetMetaData metaData = rs.getMetaData(); while (rs.next()) { @@ -152,11 +144,11 @@ public class HiveJDBC { Object value = rs.getObject(i); String filedName = filedAndColumnMap.get(metaData.getColumnName(i).toString().toLowerCase()); if (!StringUtil.isEmpty(value)) { - //如果是日期类型的属性需要把时间戳转换成日期,如果时间戳为0直接把值设置为null + // 如果是日期类型的属性需要把时间戳转换成日期,如果时间戳为0直接把值设置为null if (null != columnList && columnList.contains(filedName.toLowerCase())) { long time = 0l; time = Long.parseLong(value.toString()); - map.put(filedName, time==0l?null:new Date(time * 1000)); + map.put(filedName, time == 0l ? null : new Date(time * 1000)); // map.put(filedName, new // Date(Long.parseLong("1476583810000"))); } else { diff --git a/src/main/java/com/nis/web/controller/restful/LogController.java b/src/main/java/com/nis/web/controller/restful/LogController.java index 8ca3b30..8bf1386 100644 --- a/src/main/java/com/nis/web/controller/restful/LogController.java +++ b/src/main/java/com/nis/web/controller/restful/LogController.java @@ -1,6 +1,5 @@ package com.nis.web.controller.restful; -import java.sql.ResultSet; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; @@ -24,7 +23,7 @@ import com.nis.restful.RestServiceException; import com.nis.util.Configurations; import com.nis.util.Constants; import com.nis.util.DateUtils; -import com.nis.util.HiveJDBC; +import com.nis.util.HiveJDBCByDruid; import com.nis.util.JsonMapper; import com.nis.web.controller.BaseRestController; import com.nis.web.service.AuditLogThread; @@ -74,10 +73,10 @@ public class LogController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, dkBehaviorLog, + String sql = HiveSqlService.getSql(page, dkBehaviorLog, Configurations.getStringProperty(DkBehaviorLog.class.getSimpleName() + "HiveTable", "DK_BEHAVIOR_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, DkBehaviorLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, DkBehaviorLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); @@ -127,10 +126,10 @@ public class LogController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, pxyHttpLog, + String sql = HiveSqlService.getSql(page, pxyHttpLog, Configurations.getStringProperty(PxyHttpLog.class.getSimpleName() + "HiveTable", "PXY_HTTP_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, PxyHttpLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, PxyHttpLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); diff --git a/src/main/java/com/nis/web/controller/restful/MmLogSearchController.java b/src/main/java/com/nis/web/controller/restful/MmLogSearchController.java index 4cc618e..606cf01 100644 --- a/src/main/java/com/nis/web/controller/restful/MmLogSearchController.java +++ b/src/main/java/com/nis/web/controller/restful/MmLogSearchController.java @@ -1,6 +1,5 @@ package com.nis.web.controller.restful; -import java.sql.ResultSet; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; @@ -34,7 +33,7 @@ import com.nis.restful.RestServiceException; import com.nis.util.Configurations; import com.nis.util.Constants; import com.nis.util.DateUtils; -import com.nis.util.HiveJDBC; +import com.nis.util.HiveJDBCByDruid; import com.nis.util.JsonMapper; import com.nis.web.controller.BaseRestController; import com.nis.web.service.AuditLogThread; @@ -83,10 +82,10 @@ public class MmLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, mmAvIpLog, + String sql = HiveSqlService.getSql(page, mmAvIpLog, Configurations.getStringProperty(MmAvIpLog.class.getSimpleName() + "HiveTable", "MM_AV_IP_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, MmAvIpLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, MmAvIpLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); @@ -136,10 +135,10 @@ public class MmLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, mmAvUrlLog, + String sql = HiveSqlService.getSql(page, mmAvUrlLog, Configurations.getStringProperty(MmAvUrlLog.class.getSimpleName() + "HiveTable", "MM_AV_URL_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, MmAvUrlLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, MmAvUrlLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); @@ -189,10 +188,10 @@ public class MmLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, mmPicIpLog, + String sql = HiveSqlService.getSql(page, mmPicIpLog, Configurations.getStringProperty(MmPicIpLog.class.getSimpleName() + "HiveTable", "MM_PIC_IP_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, MmPicIpLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, MmPicIpLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); @@ -242,10 +241,10 @@ public class MmLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, mmPicUrlLog, + String sql = HiveSqlService.getSql(page, mmPicUrlLog, Configurations.getStringProperty(MmPicUrlLog.class.getSimpleName() + "HiveTable", "MM_PIC_URL_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, MmPicUrlLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, MmPicUrlLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); @@ -296,10 +295,10 @@ public class MmLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, mmVoipIpLog, + String sql = HiveSqlService.getSql(page, mmVoipIpLog, Configurations.getStringProperty(MmVoipIpLog.class.getSimpleName() + "HiveTable", "MM_VOIP_IP_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, MmVoipIpLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, MmVoipIpLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); @@ -349,10 +348,10 @@ public class MmLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, mmVoipLog, + String sql = HiveSqlService.getSql(page, mmVoipLog, Configurations.getStringProperty(MmVoipAccountLog.class.getSimpleName() + "HiveTable", "MM_VOIP_ACCOUNT_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, MmVoipAccountLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, MmVoipAccountLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); @@ -403,10 +402,10 @@ public class MmLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, mmSampleAudioLog, + String sql = HiveSqlService.getSql(page, mmSampleAudioLog, Configurations.getStringProperty(MmSampleAudioLog.class.getSimpleName() + "HiveTable", "MM_SAMPLE_AUDIO_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, MmSampleAudioLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, MmSampleAudioLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); @@ -456,10 +455,10 @@ public class MmLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, mmSampleVideoLog, + String sql = HiveSqlService.getSql(page, mmSampleVideoLog, Configurations.getStringProperty(MmSampleVideoLog.class.getSimpleName() + "HiveTable", "MM_SAMPLE_VIDEO_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, MmSampleVideoLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, MmSampleVideoLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); @@ -509,10 +508,10 @@ public class MmLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, mmPornAudioLevelLog, + String sql = HiveSqlService.getSql(page, mmPornAudioLevelLog, Configurations.getStringProperty(MmPornAudioLevelLog.class.getSimpleName() + "HiveTable", "MM_PORN_AUDIO_LEVEL_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, MmPornAudioLevelLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, MmPornAudioLevelLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); @@ -562,10 +561,10 @@ public class MmLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, mmPornVideoLevelLog, + String sql = HiveSqlService.getSql(page, mmPornVideoLevelLog, Configurations.getStringProperty(MmPornVideoLevelLog.class.getSimpleName() + "HiveTable", "MM_PRON_VIDEO_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, MmPornVideoLevelLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, MmPornVideoLevelLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); @@ -616,10 +615,10 @@ public class MmLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, mmSamplePicLog, + String sql = HiveSqlService.getSql(page, mmSamplePicLog, Configurations.getStringProperty(MmSamplePicLog.class.getSimpleName() + "HiveTable", "MM_SAMPLE_PIC_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, MmSamplePicLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, MmSamplePicLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); @@ -669,10 +668,10 @@ public class MmLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, mmSampleVoipLog, + String sql = HiveSqlService.getSql(page, mmSampleVoipLog, Configurations.getStringProperty(MmSampleVoipLog.class.getSimpleName() + "HiveTable", "MM_SAMPLE_VOIP_LOG"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, MmSampleVoipLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, MmSampleVoipLog.class, "foundTime", "recvTime"); if (tableMapping == null) { logPage.setList(new ArrayList()); diff --git a/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java b/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java index 8d044ee..878a4e9 100644 --- a/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java +++ b/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java @@ -36,6 +36,7 @@ import com.nis.util.Configurations; import com.nis.util.Constants; import com.nis.util.DateUtils; import com.nis.util.HiveJDBC; +import com.nis.util.HiveJDBCByDruid; import com.nis.util.JsonMapper; import com.nis.web.controller.BaseRestController; import com.nis.web.service.AuditLogThread; @@ -82,15 +83,20 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, ntcIpLog, +// ResultSet rs = HiveSqlService.getResultSet(page, ntcIpLog, +// Configurations.getStringProperty(NtcIpLog.class.getSimpleName() + "HiveTable", "ntc_ip_log"), +// getCol2Col(), orderBy, null); +// Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcIpLog.class, "foundTime", +// "recvTime"); + String sql = HiveSqlService.getSql(page, ntcIpLog, Configurations.getStringProperty(NtcIpLog.class.getSimpleName() + "HiveTable", "ntc_ip_log"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcIpLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null,sql , NtcIpLog.class, "foundTime", "recvTime"); if (tableMapping == null) { - ntcIpLogPage.setList(new ArrayList()); + ntcIpLogPage.setList(new ArrayList()); } else { - List list = new ArrayList(); + List list = new ArrayList(); list = tableMapping.get("obj"); // if (tableMapping.get("obj").size() > page.getPageSize()) { // list = tableMapping.get("obj").subList(0, page.getPageSize()); @@ -148,10 +154,10 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, ntcHttpLog, + String sql = HiveSqlService.getSql(page, ntcHttpLog, Configurations.getStringProperty(NtcHttpLog.class.getSimpleName() + "HiveTable", "ntc_http_log"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcHttpLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, NtcHttpLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcHttpLogPage.setList(new ArrayList()); @@ -208,10 +214,10 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, ntcDnsLog, + String sql = HiveSqlService.getSql(page, ntcDnsLog, Configurations.getStringProperty(NtcDnsLog.class.getSimpleName() + "HiveTable", "ntc_dns_log"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcDnsLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, NtcDnsLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcDnsLogPage.setList(new ArrayList()); @@ -268,10 +274,10 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, ntcMailLog, + String sql = HiveSqlService.getSql(page, ntcMailLog, Configurations.getStringProperty(NtcMailLog.class.getSimpleName() + "HiveTable", "ntc_mail_log"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcMailLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, NtcMailLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcMailLogPage.setList(new ArrayList()); @@ -328,10 +334,10 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, ntcSslLog, + String sql = HiveSqlService.getSql(page, ntcSslLog, Configurations.getStringProperty(NtcSslLog.class.getSimpleName() + "HiveTable", "ntc_ssl_log"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcSslLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, NtcSslLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcSslLogPage.setList(new ArrayList()); @@ -388,10 +394,10 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, ntcPptpLog, + String sql = HiveSqlService.getSql(page, ntcPptpLog, Configurations.getStringProperty(NtcPptpLog.class.getSimpleName() + "HiveTable", "ntc_pptp_log"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcPptpLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, NtcPptpLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcPptpLogPage.setList(new ArrayList()); @@ -447,10 +453,10 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, ntcL2tpLog, + String sql = HiveSqlService.getSql(page, ntcL2tpLog, Configurations.getStringProperty(NtcL2tpLog.class.getSimpleName() + "HiveTable", "ntc_l2tp_log"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcL2tpLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, NtcL2tpLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcL2tpLogPage.setList(new ArrayList()); @@ -507,10 +513,10 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, ntcOpenvpnLog, + String sql = HiveSqlService.getSql(page, ntcOpenvpnLog, Configurations.getStringProperty(NtcOpenvpnLog.class.getSimpleName() + "HiveTable", "ntc_openvpn_log"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcOpenvpnLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, NtcOpenvpnLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcOpenvpnLogPage.setList(new ArrayList()); @@ -567,10 +573,10 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, ntcIpsecLog, + String sql = HiveSqlService.getSql(page, ntcIpsecLog, Configurations.getStringProperty(NtcIpsecLog.class.getSimpleName() + "HiveTable", "ntc_ipsec_log"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcIpsecLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, NtcIpsecLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcIpsecLogPage.setList(new ArrayList()); @@ -627,10 +633,10 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, ntcSshLog, + String sql = HiveSqlService.getSql(page, ntcSshLog, Configurations.getStringProperty(NtcSshLog.class.getSimpleName() + "HiveTable", "ntc_ssh_log"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcSshLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, NtcSshLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcSshLogPage.setList(new ArrayList()); @@ -687,10 +693,10 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, ntcFtpLog, + String sql = HiveSqlService.getSql(page, ntcFtpLog, Configurations.getStringProperty(NtcFtpLog.class.getSimpleName() + "HiveTable", "ntc_ftp_log"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcFtpLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, NtcFtpLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcFtpLogPage.setList(new ArrayList()); @@ -745,10 +751,10 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, ntcAppLog, + String sql = HiveSqlService.getSql(page, ntcAppLog, Configurations.getStringProperty(NtcAppLog.class.getSimpleName() + "HiveTable", "ntc_app_log"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcAppLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, NtcAppLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcAppLogPage.setList(new ArrayList()); @@ -803,10 +809,10 @@ public class NtcLogSearchController extends BaseRestController { } else { orderBy = "found_Time"; } - ResultSet rs = HiveSqlService.getResultSet(page, ntcDdosLog, + String sql = HiveSqlService.getSql(page, ntcDdosLog, Configurations.getStringProperty(NtcDdosLog.class.getSimpleName() + "HiveTable", "ntc_ddos_log"), getCol2Col(), orderBy, null); - Map tableMapping = HiveJDBC.tableMapping(page, null, rs, NtcDdosLog.class, "foundTime", + Map> tableMapping = new HiveJDBCByDruid().tableMapping(page, null, sql, NtcDdosLog.class, "foundTime", "recvTime"); if (tableMapping == null) { ntcDdosLogPage.setList(new ArrayList()); diff --git a/src/main/java/com/nis/web/service/HiveSqlService.java b/src/main/java/com/nis/web/service/HiveSqlService.java index e9f4344..24b4fdf 100644 --- a/src/main/java/com/nis/web/service/HiveSqlService.java +++ b/src/main/java/com/nis/web/service/HiveSqlService.java @@ -2,7 +2,10 @@ package com.nis.web.service; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.sql.Connection; import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -17,6 +20,7 @@ import org.apache.ibatis.session.SqlSessionFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.alibaba.druid.pool.DruidDataSource; import com.nis.domain.Page; import com.nis.util.Configurations; import com.nis.util.Constants; @@ -26,9 +30,21 @@ import com.nis.util.redis.SaveRedisThread; public class HiveSqlService { private final static Logger logger = LoggerFactory.getLogger(HiveSqlService.class); + static DruidDataSource datasource = null; + Connection conn = null; + ResultSet rs = null; + Statement st = null; + + public static Connection getConnection() throws SQLException { + if (datasource == null) { + datasource = (DruidDataSource) SpringContextHolder.getBean("HiveDataSourceByDruid"); + } + return datasource.getConnection(); + } private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); private static SimpleDateFormat sdf2 = new SimpleDateFormat("yyyyMMdd"); - public static ResultSet getResultSet(Page page, Object bean, String tableName, + + public static String getSql(Page page, Object bean, String tableName, Map> col2col, String orderBy, String searchActiveSys) throws Exception { tableName = tableName.toLowerCase(); String showColmun = getFiledsSql(bean.getClass().getSimpleName(), page.getFields()); @@ -132,10 +148,7 @@ public class HiveSqlService { // row_Num between " + startNum + " and " + endNum); sql.append(" limit " + Constants.EVERY_GETHIVEDATANUM); logger.info("获取数据中心日志sql==================={}", sql); - // ResultSet query = HiveJDBC.query(sql.toString()); - ResultSet query = HiveDataSource.query(sql.toString()); - logger.info("获取数据中心日志成功"); - return query; + return sql.toString(); } public static Long getHivePageCount(Object bean, String countKey, String tableName, @@ -444,7 +457,6 @@ public class HiveSqlService { public static String getFiledsSql(String mapName, String fileds) throws Exception { String[] fieldsColoumn = null; - String orderByStr = ""; // 所有字段名 List columnList = new ArrayList(); // 所有属性名 @@ -489,7 +501,7 @@ public class HiveSqlService { return fileds; } - public static Map getFiledAndColumnMap(Class clazz) { + public static Map getFiledAndColumnMap(Class clazz) { Map map = new HashMap(); SqlSessionFactory sqlSessionFactory = SpringContextHolder.getBean(SqlSessionFactory.class); ResultMap resultMap = sqlSessionFactory.getConfiguration().getResultMap(clazz.getSimpleName() + "Map"); @@ -605,10 +617,4 @@ public class HiveSqlService { } } - - public static void main(String[] args) { - Long datacenterTime = Constants.DATACENTER_TIME; - double doubleValue = datacenterTime.doubleValue(); - System.out.println(doubleValue); - } } diff --git a/src/main/resources/applicationContext-mybatis.xml b/src/main/resources/applicationContext-mybatis.xml index 391fbec..3320b57 100644 --- a/src/main/resources/applicationContext-mybatis.xml +++ b/src/main/resources/applicationContext-mybatis.xml @@ -79,6 +79,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/jdbc.properties b/src/main/resources/jdbc.properties index 9bceab2..bd4e7e8 100644 --- a/src/main/resources/jdbc.properties +++ b/src/main/resources/jdbc.properties @@ -66,6 +66,45 @@ bonecp.hive.partitionCount=3 bonecp.hive.acquireIncrement=5 bonecp.hive.statementsCacheSize=100 + + +#########################配置hive使用druid连接池######################################### +#配置初始化连接池数量 +druid.hive.initialSize=5 +#配置最小连接池数量 +druid.hive.minIdle=1 +#配置最大连接池数量 +druid.hive.maxActive=200 +# 配置获取连接等待超时的时间 单位毫秒 +druid.hive.maxWait=600000 +#使用非公平锁 +druid.hive.useUnfairLock=true +#配置间隔多久才进行一次检测,检测需要关闭的空闲连接,单位是毫秒 +druid.hive.timeBetweenEvictionRunsMillis=60000 +#配置一个连接在池中最小生存的时间,单位是毫秒 +druid.hive.minEvictableIdleTimeMillis=300000 +#用来检测连接是否有效的sql,要求是一个查询语句 +druid.hive.validationQuery=select unix_timestamp() +#申请连接的时候检测,如果空闲时间大于timeBetweenEvictionRunsMillis,执行validationQuery检测连接是否有效 +druid.hive.testWhileIdle=true +#申请连接时执行validationQuery检测连接是否有效 +druid.hive.testOnBorrow=true +#归还连接时执行validationQuery检测连接是否有效 +druid.hive.testOnReturn=false +#打开PSCache,并且指定每个连接上PSCache的大小 +druid.hive.poolPreparedStatements=true +#要启用PSCache,必须配置大于0,当大于0时,poolPreparedStatements自动触发修改为true。在Druid中,不会存在Oracle下PSCache占用内存过多的问题,可以把这个数值配置大一些,比如说100 +druid.hive.maxOpenPreparedStatements=100 +#配置监控统计拦截的filters,去掉后监控界面sql无法统计 +druid.hive.filters=stat + + + + + + + + ##################################################################################################################################### ##redis连接 ##################################################################################################################################### diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 772a4ab..f99e89e 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -176,6 +176,25 @@ - + + DruidStatView + com.alibaba.druid.support.http.StatViewServlet + + + DruidStatView + /druid/* + + + DruidWebStatFilter + com.alibaba.druid.support.http.WebStatFilter + + exclusions + *.js,*.gif,*.jpg,*.png,*.css,*.ico,/druid/* + + + + DruidWebStatFilter + /* + \ No newline at end of file