gfwleak/k18-ntcs-web-ntc
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(1)导入验证修正bug,多线程引入service调用时发生异常，将service调用dns响应策略，dns组改为从缓存中获取

Browse Source 
(2)DNS IP导入改为批量导入
This commit is contained in:
wangxin
2018-11-16 17:06:36 +08:00
parent 19be05a25d
commit 0ece4f5729
8 changed files with 98 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/com/nis/util/Constants.java
View File

@@ -172,6 +172,14 @@ public final class Constants {
* 功能配置域字典key
*/
public static final String CACHE_FUNCTION_REGION_DICT = "functionRegionDict";
/**
* DNS 响应策略KEY
*/
public static final String CACHE_DNSRESSTRATEGY_DICT = "dnsResStrategyDict";
/**
* DNS 策略组KEY
*/
public static final String CACHE_DNSGROUP_DICT = "dnsGroupDict";
/**
* 词典数据分类
*/

49
src/main/java/com/nis/util/DictUtils.java
View File

@@ -12,11 +12,15 @@ import com.nis.domain.FunctionRegionDict;
import com.nis.domain.FunctionServiceDict;
import com.nis.domain.SysDataDictionaryItem;
import com.nis.domain.SysDataDictionaryName;
import com.nis.domain.basics.PolicyGroupInfo;
import com.nis.domain.basics.SysDictInfo;
import com.nis.domain.configuration.DnsResStrategy;
import com.nis.web.dao.FunctionRegionDictDao;
import com.nis.web.dao.FunctionServiceDictDao;
import com.nis.web.dao.SysDictDao;
import com.nis.web.dao.basics.PolicyGroupInfoDao;
import com.nis.web.dao.basics.SysDictInfoDao;
import com.nis.web.dao.configuration.DnsResStrategyDao;
import com.nis.web.service.SpringContextHolder;
@@ -34,6 +38,8 @@ public class DictUtils {
private final static SysDictInfoDao sysDictInfoDao = SpringContextHolder.getBean(SysDictInfoDao.class);
private final static FunctionRegionDictDao functionRegionDictDao = SpringContextHolder.getBean(FunctionRegionDictDao.class);
private final static FunctionServiceDictDao functionServiceDictDao = SpringContextHolder.getBean(FunctionServiceDictDao.class);
private final static DnsResStrategyDao dnsResStrategyDao = SpringContextHolder.getBean(DnsResStrategyDao.class);
private final static PolicyGroupInfoDao policyGroupInfoDao = SpringContextHolder.getBean(PolicyGroupInfoDao.class);
public static Map<String, List<SysDataDictionaryItem>> getDictData() {
@@ -320,4 +326,47 @@ public class DictUtils {
return allDictList;
}
public static List<DnsResStrategy> getDnsResStrategyList(){
List<DnsResStrategy> allDictList = (List<DnsResStrategy>)CacheUtils.get(Constants.CACHE_DNSRESSTRATEGY_DICT);
if(StringUtil.isEmpty(allDictList)){
allDictList = dnsResStrategyDao.findList(null, Constants.VALID_YES, Constants.AUDIT_YES);
CacheUtils.put(Constants.CACHE_DNSRESSTRATEGY_DICT, allDictList);
}
return allDictList;
}
public static List<DnsResStrategy> getDnsResStrategyList(Long cfgId){
List<DnsResStrategy> dnsResStrategies=Lists.newArrayList();
List<DnsResStrategy> allDictList = getDnsResStrategyList();
if(StringUtil.isEmpty(allDictList)){
}else {
for(DnsResStrategy dnsResStrategy:allDictList) {
if(dnsResStrategy.getCfgId().longValue()==cfgId.longValue()) {
dnsResStrategies.add(dnsResStrategy);
}
}
}
return dnsResStrategies;
}
public static List<PolicyGroupInfo> getDnsGroups(){
List<PolicyGroupInfo> allDictList = (List<PolicyGroupInfo>)CacheUtils.get(Constants.CACHE_DNSGROUP_DICT);
if(StringUtil.isEmpty(allDictList)){
allDictList = policyGroupInfoDao.findPolicyGroupInfosByType(1);
CacheUtils.put(Constants.CACHE_DNSGROUP_DICT, allDictList);
}
return allDictList;
}
public static PolicyGroupInfo getDnsGroups(int groupId){
List<PolicyGroupInfo> allDictList = getDnsGroups();
if(StringUtil.isEmpty(allDictList)){
}else {
for(PolicyGroupInfo group:allDictList) {
if(group.getGroupId().intValue()==groupId) {
return group;
}
}
}
return null;
}
}

14
src/main/java/com/nis/util/excel/thread/CheckComplexStringFormatThread.java
View File

@@ -21,22 +21,23 @@ import com.nis.domain.FunctionServiceDict;
import com.nis.domain.configuration.ComplexkeywordCfg;
import com.nis.domain.configuration.DnsResStrategy;
import com.nis.util.Constants;
import com.nis.web.service.configuration.DnsResStrategyService;
import com.nis.util.DictUtils;
public class CheckComplexStringFormatThread implements Callable<String>{
private Logger logger=Logger.getLogger(CheckComplexStringFormatThread.class);
private BlockingQueue<? extends Object> srcQueue;
private BlockingQueue<ComplexkeywordCfg> destQueue;
private Properties prop;
private DnsResStrategyService dnsResStrategyService;
private FunctionServiceDict serviceDict;
private FunctionRegionDict regionDict;
// private DnsResStrategyDao dnsResStrategyDao;
public CheckComplexStringFormatThread(FunctionServiceDict serviceDict,FunctionRegionDict regionDict,Properties prop,BlockingQueue<? extends Object> srcQueue,BlockingQueue<ComplexkeywordCfg> destQueue) {
this.serviceDict=serviceDict;
this.regionDict=regionDict;
this.srcQueue=srcQueue;
this.destQueue=destQueue;
this.prop=prop;
// this.dnsResStrategyDao=SpringContextHolder.getBean(DnsResStrategyDao.class);
}
@Override
public String call() throws Exception {
@@ -90,8 +91,7 @@ public class CheckComplexStringFormatThread implements Callable<String>{
if (regionDict.getFunctionId().equals(7)) {
Long dnsStrategyId = baseStringCfg.getDnsStrategyId();
if (dnsStrategyId != null&&dnsStrategyId>0) {
List<DnsResStrategy> dnsStrategys = dnsResStrategyService.findDnsResStrategys(dnsStrategyId,
Constants.VALID_YES, Constants.AUDIT_YES);
List<DnsResStrategy> dnsStrategys = DictUtils.getDnsResStrategyList(dnsStrategyId);
if (dnsStrategys == null || dnsStrategys.size() == 0) {
errInfo.append(String.format(prop.getProperty("is_incorrect"),
prop.getProperty("dns_res_strategy")) + ";");
@@ -309,10 +309,4 @@ public class CheckComplexStringFormatThread implements Callable<String>{
logger.warn("validate complexString data finish,cost:"+(end-start));
return stringList;
}
public DnsResStrategyService getDnsResStrategyService() {
return dnsResStrategyService;
}
public void setDnsResStrategyService(DnsResStrategyService dnsResStrategyService) {
this.dnsResStrategyService = dnsResStrategyService;
}
}

28
src/main/java/com/nis/util/excel/thread/CheckDnsResStrategyFormatThread.java
View File

@@ -1,9 +1,7 @@
package com.nis.util.excel.thread;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.regex.Matcher;
@@ -15,31 +13,30 @@ import org.jets3t.service.ServiceException;
import org.springframework.beans.BeanUtils;
import com.beust.jcommander.internal.Lists;
import com.beust.jcommander.internal.Sets;
import com.nis.domain.FunctionRegionDict;
import com.nis.domain.FunctionServiceDict;
import com.nis.domain.basics.PolicyGroupInfo;
import com.nis.domain.configuration.ComplexkeywordCfg;
import com.nis.domain.configuration.DnsResStrategy;
import com.nis.util.Constants;
import com.nis.web.service.basics.PolicyGroupInfoService;
import com.nis.web.service.configuration.DnsResStrategyService;
import com.nis.util.DictUtils;
public class CheckDnsResStrategyFormatThread implements Callable<String>{
private Logger logger=Logger.getLogger(CheckDnsResStrategyFormatThread.class);
private BlockingQueue<? extends Object> srcQueue;
private BlockingQueue<DnsResStrategy> destQueue;
private Properties prop;
private DnsResStrategyService dnsResStrategyService;
private FunctionServiceDict serviceDict;
private FunctionRegionDict regionDict;
private PolicyGroupInfoService policyGroupInfoService;
// private PolicyGroupInfoDao policyGroupInfoDao;
// private DnsResStrategyDao dnsResStrategyDao;
public CheckDnsResStrategyFormatThread(FunctionServiceDict serviceDict,FunctionRegionDict regionDict,Properties prop,BlockingQueue<? extends Object> srcQueue,BlockingQueue<DnsResStrategy> destQueue) {
this.serviceDict=serviceDict;
this.regionDict=regionDict;
this.srcQueue=srcQueue;
this.destQueue=destQueue;
this.prop=prop;
// this.dnsResStrategyDao=SpringContextHolder.getBean(DnsResStrategyDao.class);
// this.policyGroupInfoDao=SpringContextHolder.getBean(PolicyGroupInfoDao.class);
}
@Override
public String call() throws Exception {
@@ -79,7 +76,8 @@ public class CheckDnsResStrategyFormatThread implements Callable<String>{
errInfo.append(String.format(prop.getProperty("can_not_null"),
prop.getProperty("group")) + ";");
}else {
PolicyGroupInfo info=policyGroupInfoService.getById(resGroup1Id);
PolicyGroupInfo info=DictUtils.getDnsGroups(resGroup1Id);
// PolicyGroupInfo info=policyGroupInfoDao.getById(resGroup1Id);
if(info==null) {
errInfo.append(String.format(prop.getProperty("is_incorrect"),
prop.getProperty("group")) + ";");
@@ -139,17 +137,5 @@ public class CheckDnsResStrategyFormatThread implements Callable<String>{
}
return dnsResStrategies;
}
public DnsResStrategyService getDnsResStrategyService() {
return dnsResStrategyService;
}
public void setDnsResStrategyService(DnsResStrategyService dnsResStrategyService) {
this.dnsResStrategyService = dnsResStrategyService;
}
public PolicyGroupInfoService getPolicyGroupInfoService() {
return policyGroupInfoService;
}
public void setPolicyGroupInfoService(PolicyGroupInfoService policyGroupInfoService) {
this.policyGroupInfoService = policyGroupInfoService;
}
}

17
src/main/java/com/nis/util/excel/thread/CheckIpFormatThread.java
View File

@@ -26,23 +26,24 @@ import com.nis.domain.specific.ConfigGroupInfo;
import com.nis.util.AsnCacheUtils;
import com.nis.util.Constants;
import com.nis.util.DictUtils;
import com.nis.web.service.configuration.DnsResStrategyService;
public class CheckIpFormatThread implements Callable<String>{
private Logger logger=Logger.getLogger(CheckIpFormatThread.class);
private BlockingQueue<? extends Object> srcQueue;
private BlockingQueue<BaseIpCfg> destQueue;
private Properties prop;
private DnsResStrategyService dnsResStrategyService;
// private DnsResStrategyService dnsResStrategyService;
private FunctionServiceDict serviceDict;
private FunctionRegionDict regionDict;
private List<Map<Long,Integer>> asnNoMaps;
// private DnsResStrategyDao dnsResStrategyDao;
public CheckIpFormatThread(FunctionServiceDict serviceDict,FunctionRegionDict regionDict,Properties prop,BlockingQueue<? extends Object> srcQueue,BlockingQueue<BaseIpCfg> destQueue) {
this.serviceDict=serviceDict;
this.regionDict=regionDict;
this.srcQueue=srcQueue;
this.destQueue=destQueue;
this.prop=prop;
// this.dnsResStrategyDao=SpringContextHolder.getBean(DnsResStrategyDao.class);
}
@Override
public String call() throws Exception {
@@ -170,8 +171,7 @@ public class CheckIpFormatThread implements Callable<String>{
if (regionDict.getFunctionId().equals(7)) {
Integer dnsStrategyId = baseIpCfg.getDnsStrategyId();
if (dnsStrategyId != null&&dnsStrategyId>0) {
List<DnsResStrategy> dnsStrategys = dnsResStrategyService
.findDnsResStrategys(Long.valueOf(dnsStrategyId), Constants.VALID_YES, Constants.AUDIT_YES);
List<DnsResStrategy> dnsStrategys = DictUtils.getDnsResStrategyList(Long.valueOf(dnsStrategyId));
if (dnsStrategys == null || dnsStrategys.size() == 0) {
errInfo.append(
String.format(prop.getProperty("is_incorrect"), prop.getProperty("dns_res_strategy"))
@@ -184,8 +184,7 @@ public class CheckIpFormatThread implements Callable<String>{
Integer dnsStrategyId = baseIpCfg.getDnsStrategyId();
if (dnsStrategyId != null) {
if(dnsStrategyId>0) {
List<DnsResStrategy> dnsStrategys = dnsResStrategyService
.findDnsResStrategys(Long.valueOf(dnsStrategyId), Constants.VALID_YES, Constants.AUDIT_YES);
List<DnsResStrategy> dnsStrategys = DictUtils.getDnsResStrategyList(Long.valueOf(dnsStrategyId));
if (dnsStrategys == null || dnsStrategys.size() == 0) {
errInfo.append(
String.format(prop.getProperty("is_incorrect"), prop.getProperty("dns_res_strategy"))
@@ -1032,12 +1031,6 @@ public class CheckIpFormatThread implements Callable<String>{
return matchType;
}
public DnsResStrategyService getDnsResStrategyService() {
return dnsResStrategyService;
}
public void setDnsResStrategyService(DnsResStrategyService dnsResStrategyService) {
this.dnsResStrategyService = dnsResStrategyService;
}
public List<Map<Long, Integer>> getAsnNoMaps() {
return asnNoMaps;
}

11
src/main/java/com/nis/util/excel/thread/CheckStringFormatThread.java
View File

@@ -22,14 +22,13 @@ import com.nis.domain.SysDataDictionaryItem;
import com.nis.domain.configuration.BaseStringCfg;
import com.nis.util.Constants;
import com.nis.util.DictUtils;
import com.nis.web.service.configuration.DnsResStrategyService;
public class CheckStringFormatThread implements Callable<String>{
private Logger logger=Logger.getLogger(CheckStringFormatThread.class);
private BlockingQueue<? extends Object> srcQueue;
private BlockingQueue<BaseStringCfg<?>> destQueue;
private Properties prop;
private DnsResStrategyService dnsResStrategyService;
// private DnsResStrategyDao dnsResStrategyDao;
private FunctionServiceDict serviceDict;
private FunctionRegionDict regionDict;
public CheckStringFormatThread(FunctionServiceDict serviceDict,FunctionRegionDict regionDict,Properties prop,BlockingQueue<? extends Object> srcQueue,BlockingQueue<BaseStringCfg<?>> destQueue) {
@@ -38,6 +37,7 @@ public class CheckStringFormatThread implements Callable<String>{
this.srcQueue=srcQueue;
this.destQueue=destQueue;
this.prop=prop;
// this.dnsResStrategyDao=SpringContextHolder.getBean(DnsResStrategyDao.class);
}
@Override
public String call() throws Exception {
@@ -320,11 +320,4 @@ public class CheckStringFormatThread implements Callable<String>{
logger.warn("validate stringCfg data finish,cost:"+(end-start));
return stringList;
}
public DnsResStrategyService getDnsResStrategyService() {
return dnsResStrategyService;
}
public void setDnsResStrategyService(DnsResStrategyService dnsResStrategyService) {
this.dnsResStrategyService = dnsResStrategyService;
}
}

5
src/main/java/com/nis/web/controller/BaseController.java
View File

@@ -869,7 +869,6 @@ public class BaseController {
List<Future<String>> futures=new ArrayList<>();
for(int i=0;i<Constants.MULITY_THREAD_SIZE;i++) {
CheckIpFormatThread t=new CheckIpFormatThread(serviceDict,regionDict, this.getMsgProp(), list, queue);
t.setDnsResStrategyService(dnsResStrategyService);
t.setAsnNoMaps(asnNos);
futures.add(service.submit(t));
}
@@ -910,7 +909,6 @@ public class BaseController {
List<Future<String>> futures=new ArrayList<>();
for(int i=0;i<Constants.MULITY_THREAD_SIZE;i++) {
CheckStringFormatThread t=new CheckStringFormatThread(serviceDict,regionDict, this.getMsgProp(), list, queue);
t.setDnsResStrategyService(dnsResStrategyService);
futures.add(service.submit(t));
}
service.shutdown();
@@ -950,7 +948,6 @@ public class BaseController {
List<Future<String>> futures=new ArrayList<>();
for(int i=0;i<Constants.MULITY_THREAD_SIZE;i++) {
CheckComplexStringFormatThread t=new CheckComplexStringFormatThread(serviceDict,regionDict, this.getMsgProp(), list, queue);
t.setDnsResStrategyService(dnsResStrategyService);
futures.add(service.submit(t));
}
service.shutdown();
@@ -990,8 +987,6 @@ public class BaseController {
List<Future<String>> futures=new ArrayList<>();
for(int i=0;i<Constants.MULITY_THREAD_SIZE;i++) {
CheckDnsResStrategyFormatThread t=new CheckDnsResStrategyFormatThread(serviceDict,regionDict, this.getMsgProp(), list, queue);
t.setDnsResStrategyService(dnsResStrategyService);
t.setPolicyGroupInfoService(policyGroupInfoService);
futures.add(service.submit(t));
}
service.shutdown();

24
src/main/java/com/nis/web/service/configuration/DnsIpCfgService.java
View File

@@ -5,11 +5,15 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.ibatis.session.ExecutorType;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.beust.jcommander.internal.Lists;
import com.nis.domain.Page;
import com.nis.domain.callback.InlineIp;
import com.nis.domain.configuration.BaseIpCfg;
@@ -20,6 +24,7 @@ import com.nis.util.ConfigServiceUtil;
import com.nis.web.dao.configuration.DnsIpCfgDao;
import com.nis.web.security.UserUtils;
import com.nis.web.service.BaseService;
import com.nis.web.service.SpringContextHolder;
@@ -45,11 +50,28 @@ public class DnsIpCfgService extends BaseService{
public DnsIpCfg getDnsIpCfg(Long cfgId) {
return dnsIpCfgDao.getDnsIpCfg(cfgId);
}
@Transactional(readOnly=false,rollbackFor=RuntimeException.class)
public void saveDnsIpCfg(List<BaseIpCfg> cfgs) {
List<DnsIpCfg> dnsIpCfgs=Lists.newArrayList(cfgs.size());
for(BaseIpCfg cfg:cfgs) {
DnsIpCfg _cfg=new DnsIpCfg();
BeanUtils.copyProperties(cfg, _cfg);
saveOrUpdate(_cfg);
dnsIpCfgs.add(_cfg);
}
SqlSessionFactory sqlSessionFactory=SpringContextHolder.getBean(SqlSessionFactory.class);
SqlSession batchSqlSession = null;
try{
batchSqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);
for(int index = 0; index < dnsIpCfgs.size();index++){
DnsIpCfg dnsIpCfg = dnsIpCfgs.get(index);
//如果insert加入了select last_insert_id，会拖慢6~7倍以上的效率
((DnsIpCfgDao) batchSqlSession.getMapper(DnsIpCfgDao.class)).insert(dnsIpCfg);
}
batchSqlSession.commit();
}finally {
if(batchSqlSession != null){
batchSqlSession.close();
}
}
}
@Transactional(readOnly=false,rollbackFor=RuntimeException.class)