(1)asn no放入eCache中

(2)导入验证采用多线程验证,优化验证速度
(3)asn ip导入方式调整(未采用多线程,因为redis承受不了)
(4)asn ip列表展示速度优化
(5)导入方式重写:采用csv模式,限制采用xlsx格式,加载80万数据不会内存溢出.
This commit is contained in:
wangxin
2018-11-11 19:36:53 +08:00
parent 40bcfea683
commit 80dde7d6a0
53 changed files with 4831 additions and 2996 deletions

View File

@@ -1,10 +1,14 @@
package com.nis.web.service.basics;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
@@ -14,9 +18,9 @@ import org.springframework.transaction.annotation.Transactional;
import com.beust.jcommander.internal.Lists;
import com.nis.domain.Page;
import com.nis.domain.SysDataDictionaryItem;
import com.nis.domain.basics.AsnIpCfg;
import com.nis.domain.basics.PolicyGroupInfo;
import com.nis.domain.basics.Varibles;
import com.nis.domain.configuration.CfgIndexInfo;
import com.nis.domain.maat.GroupReuseAddBean;
import com.nis.domain.maat.GroupReuseCfg;
@@ -26,21 +30,23 @@ import com.nis.domain.maat.MaatCfg.StringCfg;
import com.nis.domain.maat.ToMaatResult;
import com.nis.domain.specific.ConfigGroupInfo;
import com.nis.exceptions.MaatConvertException;
import com.nis.util.AsnCacheUtils;
import com.nis.util.ConfigServiceUtil;
import com.nis.util.Constants;
import com.nis.util.DictUtils;
import com.nis.web.dao.CrudDao;
import com.nis.web.dao.basics.AsnIpCfgDao;
import com.nis.web.dao.basics.PolicyGroupInfoDao;
import com.nis.web.dao.specific.SpecificServiceCfgDao;
import com.nis.web.dao.specific.ConfigGroupInfoDao;
import com.nis.web.security.UserUtils;
import com.nis.web.service.CrudService;
@Service
public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
@Autowired
private AsnIpCfgDao asnIpCfgDao;
// @Autowired
// private SpecificServiceCfgDao specificServiceCfgDao;
@Autowired
private SpecificServiceCfgDao specificServiceCfgDao;
private ConfigGroupInfoDao configGroupInfoDao;
@Autowired
private PolicyGroupInfoDao policyGroupInfoDao;
/**
@@ -58,9 +64,38 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
public AsnIpCfg get(Long id ) {
return asnIpCfgDao.get(id);
}
// public void save(List<AsnIpCfg> entitys){
// this.saveBatch(entitys, AsnIpCfgDao.class);
// }
@Transactional(readOnly=false,rollbackFor=RuntimeException.class)
public void save(List<AsnIpCfg> entitys){
this.saveBatch(entitys, AsnIpCfgDao.class);
public void save(List<AsnIpCfg> entitys) {
logger.warn("Start to save IP");
long start=System.currentTimeMillis();
int len=0;
List<AsnIpCfg> tempList=Lists.newArrayList();
Varibles maxPacket=asnIpCfgDao.getVaribles("max_allowed_packet");
for(AsnIpCfg asnIpCfg:entitys) {
int tempLen=asnIpCfg.toString().getBytes(Charset.forName("UTF-8")).length;
if((len+tempLen)<maxPacket.getValue().longValue()) {
tempList.add(asnIpCfg);
len+=tempLen;
}else {
logger.warn("save ip size:"+tempList.size());
asnIpCfgDao.insertBatch(tempList);
tempList.clear();
tempList.add(asnIpCfg);
len=tempLen;
}
}
if(tempList.size()>0) {
logger.warn("save ip size:"+tempList.size());
asnIpCfgDao.insertBatch(tempList);
tempList.clear();
}
// entitys.clear();
long end=System.currentTimeMillis();
logger.warn("Save IP finish,cost:"+(end-start));
}
@Transactional(readOnly=false,rollbackFor=RuntimeException.class)
public void update(AsnIpCfg entity){
@@ -70,7 +105,7 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
if(entity.getAsnIpGroup()==null) {
throw new RuntimeException("asn group name not found!");
}
ConfigGroupInfo groupInfo=specificServiceCfgDao.getConfigGroupInfoByGroupId(entity.getAsnIpGroup());
ConfigGroupInfo groupInfo=configGroupInfoDao.getConfigGroupInfoByGroupId(entity.getAsnIpGroup());
if(groupInfo.getIsIssued()==0) {//未下发,可修改
asnIpCfgDao.update(entity);
}else {
@@ -79,53 +114,170 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
}
@Transactional(readOnly=false,rollbackFor=RuntimeException.class)
public void saveAsnIpBatch(Map<String,List<AsnIpCfg>> cfgs,int total) {
//需要通过删除域接口删除的ip集合
List<AsnIpCfg> toDelAsnIpCfgs=Lists.newArrayList();
//需要本地保存的IP集合
List<AsnIpCfg> toSaveAsnIpCfgs=Lists.newArrayList();
public void processGroup(Map<Long,Integer> asnNoMap){
logger.warn("Start to save group");
long start=System.currentTimeMillis();
List<PolicyGroupInfo> policyGroupInfos=Lists.newArrayList();
List<ConfigGroupInfo> configGroupInfoS=Lists.newArrayList();
Date createTime=new Date();
int len=0,len1=0;
Varibles maxPacket=asnIpCfgDao.getVaribles("max_allowed_packet");
for(Entry<Long, Integer> e:asnNoMap.entrySet()) {
PolicyGroupInfo info=new PolicyGroupInfo();
info.setAsnNo(e.getKey());
info.setDescription("Create by import excel");
info.setGroupName("Import"+e.getKey());
info.setGroupType(4);
info.setServiceGroupId(asnNoMap.get(e.getKey()));
info.setIsValid(1);
info.setCreatorId(UserUtils.getUser().getId());
info.setCreateTime(createTime);
int tempLen=info.toString().getBytes(Charset.forName("UTF-8")).length;
if((len+tempLen)<maxPacket.getValue().longValue()) {
policyGroupInfos.add(info);
len+=tempLen;
}else {
policyGroupInfoDao.insertBatch(policyGroupInfos);
policyGroupInfos.clear();
policyGroupInfos.add(info);
len=tempLen;
}
ConfigGroupInfo group=new ConfigGroupInfo();
group.setGroupId(info.getServiceGroupId());
group.setGroupName(info.getGroupName());
group.setIsIssued(0);
group.setGroupType(info.getGroupType());
group.setAsnId(info.getAsnNo());
int tempLen1=group.toString().getBytes(Charset.forName("UTF-8")).length;
if((len1+tempLen1)<maxPacket.getValue().longValue()) {
configGroupInfoS.add(group);
len1+=tempLen1;
}else {
configGroupInfoDao.insertBatch(configGroupInfoS);
configGroupInfoS.clear();
configGroupInfoS.add(group);
len1=tempLen1;
}
}
if(policyGroupInfos.size()>0) {
policyGroupInfoDao.insertBatch(policyGroupInfos);
policyGroupInfos.clear();
}
if(configGroupInfoS.size()>0) {
configGroupInfoDao.insertBatch(configGroupInfoS);
configGroupInfoS.clear();
}
//刷新缓存
AsnCacheUtils.init();
long end=System.currentTimeMillis();
logger.warn("Save group finish,cost:"+(end-start));
}
public void deleteIps(Map<Long,Integer> asnNoMap){
List<AsnIpCfg> toDelAndSendAsnIpCfgs=Lists.newArrayList();
List<Long> asnIds=Lists.newArrayList(asnNoMap.size());
for(Entry<Long, Integer> e:asnNoMap.entrySet()) {
ConfigGroupInfo configGroupInfo=AsnCacheUtils.get(e.getKey());
if(configGroupInfo==null) {
configGroupInfo=this.getConfigGroupInfoByAsnNo(e.getKey());
}
if(configGroupInfo.getIsIssued()==1) {//已下发
List<AsnIpCfg> _toDelAsnIpCfgs=this.getByAsnNo(configGroupInfo.getAsnId());
toDelAndSendAsnIpCfgs.addAll(_toDelAsnIpCfgs);
}else {
asnIds.add(e.getKey());
}
}
if(asnIds.size()>0) {
this.deleteByAsnNo(asnIds);
}
if(toDelAndSendAsnIpCfgs.size()>0) {
int pointsDataLimit = Constants.MAAT_JSON_SEND_SIZE;//限制条数
Integer size = toDelAndSendAsnIpCfgs.size();
//判断是否有必要分批
if(pointsDataLimit<size){
int part = size/pointsDataLimit;//分批数
for (int i = 0; i < part; i++) {
//pointsDataLimit条
List<AsnIpCfg> listPage = toDelAndSendAsnIpCfgs.subList(0, pointsDataLimit);
delAndSend(listPage);
//剔除
toDelAndSendAsnIpCfgs.subList(0, pointsDataLimit).clear();
}
//最后剩下的
if(!toDelAndSendAsnIpCfgs.isEmpty()){
delAndSend(toDelAndSendAsnIpCfgs);
}
}else {
delAndSend(toDelAndSendAsnIpCfgs);
}
}
}
/**
* 根据Id 批量删除
* @param entities
*/
public void deleteByIdsBatch(List<AsnIpCfg> entities) {
StringBuilder sb=new StringBuilder();
for(AsnIpCfg ip:entities) {
sb.append(ip.getCfgId());
sb.append(",");
}
sb.deleteCharAt(sb.toString().lastIndexOf(","));
asnIpCfgDao.delete(sb.toString());
}
@Transactional(readOnly=false,rollbackFor=RuntimeException.class)
public void delAndSend(List<AsnIpCfg> entities) {
deleteByIdsBatch(entities);
asnIPRegionSendToMaat(entities,Constants.VALID_NO);
}
@Transactional(readOnly=false,rollbackFor=RuntimeException.class)
public void saveAsnIpBatch(List<AsnIpCfg> cfgs){
//需要通过新增域接口新增的ip集合
List<AsnIpCfg> toAddRegionAsnIpCfgs=Lists.newArrayList();
List<SysDataDictionaryItem> isImportAll=DictUtils.getDictList("IS_ASN_IP_IMPORT_ALL");
List<Integer> idList=ConfigServiceUtil.getId(3, total);
int ind=0;
for(Entry<String,List<AsnIpCfg>> entry:cfgs.entrySet()) {
Long asnNo=Long.parseLong(entry.getKey());
ConfigGroupInfo configGroupInfo=this.getConfigGroupInfoByAsnNo(asnNo);
if(configGroupInfo==null) {//不存在则新增
PolicyGroupInfo info=new PolicyGroupInfo();
info.setAsnNo(Long.parseLong(entry.getKey()));
info.setDescription("Create by import excel");
info.setGroupName("Import"+asnNo);
info.setGroupType(4);
configGroupInfo=this.saveAsnGroup(info);
logger.warn("process configGroupInfo and delete IP start");
long start=System.currentTimeMillis();
for(AsnIpCfg cfg:cfgs) {
if(Constants.VALID_YES==cfg.getIsValid().intValue()) {
toAddRegionAsnIpCfgs.add(cfg);
}
for(AsnIpCfg cfg:entry.getValue()) {
cfg.setAsnIpGroup(configGroupInfo.getGroupId());
cfg.setRegionId(idList.get(ind));
if(configGroupInfo.getIsIssued()==0)
cfg.setIsValid(Constants.VALID_NO);
else {
cfg.setIsValid(Constants.VALID_YES);
toAddRegionAsnIpCfgs.add(cfg);
}
ind++;
}
if(isImportAll.get(0).getItemCode().equals("1")) {//全量下发,删除已有的
if(configGroupInfo.getIsIssued()==1) {//已下发
List<AsnIpCfg> _toDelAsnIpCfgs=this.getByAsnGroup(configGroupInfo.getGroupId(), configGroupInfo.getAsnId());
toDelAsnIpCfgs.addAll(_toDelAsnIpCfgs);
}
this.deleteByAsnGroup(configGroupInfo.getGroupId(), configGroupInfo.getAsnId());
}
toSaveAsnIpCfgs.addAll(entry.getValue());
}
this.save(toSaveAsnIpCfgs);
if(toDelAsnIpCfgs.size()>0) {
asnIPRegionSendToMaat(toDelAsnIpCfgs,Constants.VALID_NO);
}
if(toAddRegionAsnIpCfgs.size()>0) {
asnIPRegionSendToMaat(toAddRegionAsnIpCfgs,Constants.VALID_YES);
long end=System.currentTimeMillis();
logger.warn("process configGroupInfo and delete IP finish,cost:"+(end-start));
this.save(cfgs);
cfgs.clear();
splitAndSend(toAddRegionAsnIpCfgs,Constants.VALID_YES);
toAddRegionAsnIpCfgs.clear();
}
/**
* 分割IP region分次下发
* @param asnIpCfgs
* @param valid
*/
public void splitAndSend(List<AsnIpCfg> asnIpCfgs,Integer valid){
if(asnIpCfgs!=null&&asnIpCfgs.size()>0) {
// List<AsnIpCfg> asnIpCfgs=Lists.newArrayList(list.size());
// asnIpCfgs.addAll(list);
int pointsDataLimit = Constants.MAAT_JSON_SEND_SIZE;//限制条数
Integer size = asnIpCfgs.size();
//判断是否有必要分批
if(pointsDataLimit<size){
int part = size/pointsDataLimit;//分批数
for (int i = 0; i < part; i++) {
//pointsDataLimit条
List<AsnIpCfg> listPage = asnIpCfgs.subList(0, pointsDataLimit);
asnIPRegionSendToMaat(listPage,valid);
//剔除
asnIpCfgs.subList(0, pointsDataLimit).clear();
}
//最后剩下的
if(!asnIpCfgs.isEmpty()){
asnIPRegionSendToMaat(asnIpCfgs,valid);
}
}else {
asnIPRegionSendToMaat(asnIpCfgs,valid);
}
}
}
/**
@@ -151,7 +303,11 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
maatBean.setOpAction(Constants.INSERT_ACTION);
//调用服务接口下发配置数据
String json=gsonToJson(maatBean);
logger.info("asn ip复用域新增配置下发配置参数"+json);
if(asnIpCfgs.size()<=100) {
logger.info("asn ip复用域新增配置下发配置参数"+json);
}else {
logger.info("asn ip复用域新增配置下发region条数"+asnIpCfgs.size());
}
//调用服务接口下发配置
ToMaatResult result =ConfigServiceUtil.postGroupReuseSources(json);
logger.info("asn ip复用域新增配置响应信息"+result.getMsg());
@@ -159,45 +315,63 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
maatBean.setOpAction(Constants.UPDATE_ACTION);
//调用服务接口下发配置数据
String json=gsonToJson(maatBean);
logger.info("asn ip复用域删除配置下发配置参数"+json);
if(asnIpCfgs.size()<=100) {
logger.info("asn ip复用域删除配置下发配置参数"+json);
}else {
logger.info("asn ip复用域删除配置下发region条数"+asnIpCfgs.size());
}
//调用服务接口下发配置
ToMaatResult result = ConfigServiceUtil.put(json,3);
logger.info("asn ip复用域删除配置响应信息"+result.getMsg());
}
}
/**
* 批量保存asn PolicyGroupInfo
* @param entities
* @return
* @throws MaatConvertException
*/
@Transactional(readOnly=false,rollbackFor=RuntimeException.class)
public ConfigGroupInfo saveAsnGroup(PolicyGroupInfo entity) throws MaatConvertException{
entity.setIsValid(1);
ConfigGroupInfo group=new ConfigGroupInfo();
//新增
Date createTime=new Date();
entity.setCreatorId(UserUtils.getUser().getId());
entity.setCreateTime(createTime);
//asn类型从综合服务获取groupId
if(4==entity.getGroupType().intValue()) {
int serviceGroupId=0;
List<Integer> groupIds= ConfigServiceUtil.getId(2,1);
if(groupIds.size()>0) {
serviceGroupId=groupIds.get(0).intValue();
}else {
throw new MaatConvertException("Get asn group id failed");
public Map<Long,ConfigGroupInfo> saveAsnGroupBatch(List<PolicyGroupInfo> entities) throws MaatConvertException{
Map<Long,ConfigGroupInfo> configGroupInfoMap=new HashMap<>(entities.size());
List<Integer> groupIds= ConfigServiceUtil.getId(2,entities.size());
if(groupIds.size()!=entities.size()) {
throw new MaatConvertException("Get asn group id failed");
}
int ind=0;
for(PolicyGroupInfo entity:entities) {
entity.setIsValid(1);
ConfigGroupInfo group=new ConfigGroupInfo();
//新增
Date createTime=new Date();
entity.setCreatorId(UserUtils.getUser().getId());
entity.setCreateTime(createTime);
//asn类型从综合服务获取groupId
if(4!=entity.getGroupType().intValue()) {
throw new RuntimeException("ConfigGroupInfo is not asn type 4!");
}
entity.setServiceGroupId(serviceGroupId);
entity.setServiceGroupId(groupIds.get(ind).intValue());
//新增协议分组
group.setGroupId(serviceGroupId);
group.setGroupId(groupIds.get(ind).intValue());
group.setGroupName(entity.getGroupName());
group.setIsIssued(0);
group.setGroupType(entity.getGroupType());
group.setAsnId(entity.getAsnNo());
specificServiceCfgDao.insertConfigGroupInfo(group);
configGroupInfoDao.insertConfigGroupInfo(group);
if(group.getGroupType().intValue()==4) {
AsnCacheUtils.put(entity.getAsnNo(), group);
}
policyGroupInfoDao.insert(entity);
configGroupInfoMap.put(group.getAsnId(), group);
ind++;
}
return group;
return configGroupInfoMap;
}
@Transactional(readOnly=false,rollbackFor=RuntimeException.class)
public void saveAsnIpCfg(CfgIndexInfo entity) {
Date createTime=new Date();
ConfigGroupInfo groupInfo=specificServiceCfgDao.getConfigGroupInfoByGroupId(entity.getAsnIpGroup().intValue());
ConfigGroupInfo groupInfo=configGroupInfoDao.getConfigGroupInfoByGroupId(entity.getAsnIpGroup().intValue());
if(groupInfo==null) {
throw new RuntimeException("ConfigGroupInfo is null!");
}
@@ -221,7 +395,7 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
}
}
this.save(entity.getAsnIpCfgs());
asnIPRegionSendToMaat(entity.getAsnIpCfgs(),Constants.VALID_YES);
splitAndSend(entity.getAsnIpCfgs(),Constants.VALID_YES);
}else {
if(entity.getAsnIpCfgs()!=null) {
for(AsnIpCfg cfg:entity.getAsnIpCfgs()) {
@@ -238,7 +412,7 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
}
}
@Transactional(readOnly=false,rollbackFor=RuntimeException.class)
public void delete(Integer functionId,Integer isValid,String ids) {
public void delete(Integer isValid,String ids) {
for(String id:ids.split(",")) {
Long.parseLong(id);
}
@@ -256,11 +430,10 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
}
}
}
if(issuedList.size()>0) {
//已经下发过的,调用分组复用配置删除接口
asnIPRegionSendToMaat(issuedList,Constants.VALID_NO);
}
asnIpCfgDao.updateValid(isValid, ids);
asnIpCfgDao.delete(ids);
//已经下发过的,调用分组复用配置删除接口
splitAndSend(issuedList,Constants.VALID_NO);
}
// public List<ConfigGroupInfo> findPolicyGroupInfosByType(Integer groupId) {
// // TODO Auto-generated method stub
@@ -288,18 +461,65 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
return false;
}
public ConfigGroupInfo getConfigGroupInfoByAsnNo(Long asnNo) {
return asnIpCfgDao.getInfoByAsnNo(asnNo);
ConfigGroupInfo configGroupInfo=configGroupInfoDao.getInfoByAsnNo(asnNo);
AsnCacheUtils.put(asnNo, configGroupInfo);
return configGroupInfo;
}
public List<AsnIpCfg> getByAsnGroup(Integer groupId,Long asnNo) {
if(groupId==null||asnNo==null) {
public List<AsnIpCfg> getByAsnNo(Long asnNo) {
if(asnNo==null) {
throw new RuntimeException("groupId or asnNo is null!");
}
return asnIpCfgDao.getByAsnGroup(groupId.longValue(),asnNo.longValue());
return asnIpCfgDao.getByAsnNo(asnNo.longValue());
}
public void deleteByAsnGroup(Integer groupId,Long asnNo) {
if(groupId==null||asnNo==null) {
throw new RuntimeException("groupId or asnNo is null!");
// public void deleteByAsnGroup(Long asnNo) {
// if(asnNo==null) {
// throw new RuntimeException("asnNo is null!");
// }
// asnIpCfgDao.deleteByAsnGroup(asnNo.longValue());
// }
@Transactional(readOnly=false,rollbackFor=RuntimeException.class)
public void deleteByAsnNo(List<Long> asnNoList) {
int pointsDataLimit = 1000;//限制条数
Integer size = asnNoList.size();
if(pointsDataLimit<size){
int part = size/pointsDataLimit;//分批数
for (int i = 0; i < part; i++) {
List<Long> listPage = asnNoList.subList(0, pointsDataLimit);
StringBuilder asnNoStr=new StringBuilder();
for(Long asnNo:asnNoList) {
asnNoStr.append(asnNo);
asnNoStr.append(",");
}
int result=0;
do {
result=asnIpCfgDao.deleteByAsnGroup(asnNoStr.toString().substring(0,asnNoStr.toString().length()-1));
}while(result>0);
//剔除
asnNoList.subList(0, pointsDataLimit).clear();
}
if(!asnNoList.isEmpty()){
StringBuilder asnNoStr=new StringBuilder();
for(Long asnNo:asnNoList) {
asnNoStr.append(asnNo);
asnNoStr.append(",");
}
int result=0;
do {
result=asnIpCfgDao.deleteByAsnGroup(asnNoStr.toString().substring(0,asnNoStr.toString().length()-1));
}while(result>0);
asnNoList.clear();
}
}else {
StringBuilder asnNoStr=new StringBuilder();
for(Long asnNo:asnNoList) {
asnNoStr.append(asnNo);
asnNoStr.append(",");
}
int result=0;
do {
result=asnIpCfgDao.deleteByAsnGroup(asnNoStr.toString().substring(0,asnNoStr.toString().length()-1));
}while(result>0);
asnNoList.clear();
}
asnIpCfgDao.deleteByAsnGroup(groupId.longValue(),asnNo.longValue());
}
}