gfwleak/k18-ntcs-web-ntc
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

将max_allow_packet参数提取到配置文件中，不用每次从数据库中读取

Browse Source
This commit is contained in:
wangxin
2018-11-18 18:28:53 +08:00
parent 97882ad05f
commit 21dcd430e3
5 changed files with 20 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/nis/util/Constants.java
View File

@@ -734,4 +734,6 @@ public final class Constants {
public static final Integer MAAT_JSON_SEND_SIZE=Configurations.getIntProperty("maat_json_send_size", 1000); public static final Integer MAAT_JSON_SEND_SIZE=Configurations.getIntProperty("maat_json_send_size", 1000);
public static final Integer MULITY_THREAD_SIZE=Configurations.getIntProperty("mulity_thread_size", 5); public static final Integer MULITY_THREAD_SIZE=Configurations.getIntProperty("mulity_thread_size", 5);
public static final Integer SAVE_AND_DEL_THREAD_SIZE=Configurations.getIntProperty("save_and_del_thread_size", 5); public static final Integer SAVE_AND_DEL_THREAD_SIZE=Configurations.getIntProperty("save_and_del_thread_size", 5);
public static final Integer MAX_ALLOWED_PACKET=Configurations.getIntProperty("max_allowed_packet", 1048576);
} }

4
src/main/java/com/nis/util/excel/thread/SaveAsnIpThread.java
View File

@@ -175,10 +175,10 @@ public class SaveAsnIpThread implements Callable<Throwable>{
long start=System.currentTimeMillis(); long start=System.currentTimeMillis();
int len=0; int len=0;
List<AsnIpCfg> tempList=Lists.newArrayList(); List<AsnIpCfg> tempList=Lists.newArrayList();
Varibles maxPacket=asnIpCfgDao.getVaribles("max_allowed_packet"); // Varibles maxPacket=asnIpCfgDao.getVaribles("max_allowed_packet");
for(AsnIpCfg asnIpCfg:entitys) { for(AsnIpCfg asnIpCfg:entitys) {
int tempLen=asnIpCfg.toString().getBytes(Charset.forName("UTF-8")).length; int tempLen=asnIpCfg.toString().getBytes(Charset.forName("UTF-8")).length;
if((len+tempLen)<maxPacket.getValue().longValue()) { if((len+tempLen)<Constants.MAX_ALLOWED_PACKET) {
tempList.add(asnIpCfg); tempList.add(asnIpCfg);
len+=tempLen; len+=tempLen;
}else { }else {

20
src/main/java/com/nis/web/service/basics/AsnIpCfgService.java
View File

@@ -70,18 +70,18 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
// } // }
@Transactional(readOnly=false,rollbackFor=RuntimeException.class) @Transactional(readOnly=false,rollbackFor=RuntimeException.class)
public void save(List<AsnIpCfg> entitys) { public void save(List<AsnIpCfg> entitys) {
logger.warn("Start to save IP"); logger.warn("Start to save IP,size:"+entitys.size());
long start=System.currentTimeMillis(); long start=System.currentTimeMillis();
int len=0; int len=0;
List<AsnIpCfg> tempList=Lists.newArrayList(); List<AsnIpCfg> tempList=Lists.newArrayList();
Varibles maxPacket=asnIpCfgDao.getVaribles("max_allowed_packet"); // Varibles maxPacket=asnIpCfgDao.getVaribles("max_allowed_packet");
for(AsnIpCfg asnIpCfg:entitys) { for(AsnIpCfg asnIpCfg:entitys) {
int tempLen=asnIpCfg.toString().getBytes(Charset.forName("UTF-8")).length; int tempLen=asnIpCfg.toString().getBytes(Charset.forName("UTF-8")).length;
if((len+tempLen)<maxPacket.getValue().longValue()) { if((len+tempLen)<Constants.MAX_ALLOWED_PACKET) {
tempList.add(asnIpCfg); tempList.add(asnIpCfg);
len+=tempLen; len+=tempLen;
}else { }else {
logger.warn("save ip size:"+tempList.size()); // logger.warn("save ip size:"+tempList.size());
asnIpCfgDao.insertBatch(tempList); asnIpCfgDao.insertBatch(tempList);
tempList.clear(); tempList.clear();
tempList.add(asnIpCfg); tempList.add(asnIpCfg);
@@ -89,7 +89,7 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
} }
} }
if(tempList.size()>0) { if(tempList.size()>0) {
logger.warn("save ip size:"+tempList.size()); // logger.warn("save ip size:"+tempList.size());
asnIpCfgDao.insertBatch(tempList); asnIpCfgDao.insertBatch(tempList);
tempList.clear(); tempList.clear();
} }
@@ -121,7 +121,7 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
List<ConfigGroupInfo> configGroupInfoS=Lists.newArrayList(); List<ConfigGroupInfo> configGroupInfoS=Lists.newArrayList();
Date createTime=new Date(); Date createTime=new Date();
int len=0,len1=0; int len=0,len1=0;
Varibles maxPacket=asnIpCfgDao.getVaribles("max_allowed_packet"); // Varibles maxPacket=asnIpCfgDao.getVaribles("max_allowed_packet");
for(Entry<Long, Integer> e:asnNoMap.entrySet()) { for(Entry<Long, Integer> e:asnNoMap.entrySet()) {
PolicyGroupInfo info=new PolicyGroupInfo(); PolicyGroupInfo info=new PolicyGroupInfo();
info.setAsnNo(e.getKey()); info.setAsnNo(e.getKey());
@@ -133,7 +133,7 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
info.setCreatorId(UserUtils.getUser().getId()); info.setCreatorId(UserUtils.getUser().getId());
info.setCreateTime(createTime); info.setCreateTime(createTime);
int tempLen=info.toString().getBytes(Charset.forName("UTF-8")).length; int tempLen=info.toString().getBytes(Charset.forName("UTF-8")).length;
if((len+tempLen)<maxPacket.getValue().longValue()) { if((len+tempLen)<Constants.MAX_ALLOWED_PACKET) {
policyGroupInfos.add(info); policyGroupInfos.add(info);
len+=tempLen; len+=tempLen;
}else { }else {
@@ -149,7 +149,7 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
group.setGroupType(info.getGroupType()); group.setGroupType(info.getGroupType());
group.setAsnId(info.getAsnNo()); group.setAsnId(info.getAsnNo());
int tempLen1=group.toString().getBytes(Charset.forName("UTF-8")).length; int tempLen1=group.toString().getBytes(Charset.forName("UTF-8")).length;
if((len1+tempLen1)<maxPacket.getValue().longValue()) { if((len1+tempLen1)<Constants.MAX_ALLOWED_PACKET) {
configGroupInfoS.add(group); configGroupInfoS.add(group);
len1+=tempLen1; len1+=tempLen1;
}else { }else {
@@ -235,13 +235,11 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
public void saveAsnIpBatch(List<AsnIpCfg> cfgs){ public void saveAsnIpBatch(List<AsnIpCfg> cfgs){
//需要通过新增域接口新增的ip集合 //需要通过新增域接口新增的ip集合
List<AsnIpCfg> toAddRegionAsnIpCfgs=Lists.newArrayList(); List<AsnIpCfg> toAddRegionAsnIpCfgs=Lists.newArrayList();
long start=System.currentTimeMillis();
for(AsnIpCfg cfg:cfgs) { for(AsnIpCfg cfg:cfgs) {
if(Constants.VALID_YES==cfg.getIsValid().intValue()) { if(Constants.VALID_YES==cfg.getIsValid().intValue()) {
toAddRegionAsnIpCfgs.add(cfg); toAddRegionAsnIpCfgs.add(cfg);
} }
} }
long end=System.currentTimeMillis();
this.save(cfgs); this.save(cfgs);
// splitAndSend(toAddRegionAsnIpCfgs,Constants.VALID_YES); // splitAndSend(toAddRegionAsnIpCfgs,Constants.VALID_YES);
if(toAddRegionAsnIpCfgs.size()>0) { if(toAddRegionAsnIpCfgs.size()>0) {
@@ -249,6 +247,8 @@ public class AsnIpCfgService extends CrudService<CrudDao<AsnIpCfg>, AsnIpCfg> {
} }
cfgs.clear(); cfgs.clear();
toAddRegionAsnIpCfgs.clear(); toAddRegionAsnIpCfgs.clear();
cfgs=null;
toAddRegionAsnIpCfgs=null;
} }
/** /**
* 分割IP region分次下发 * 分割IP region分次下发

4
src/main/java/com/nis/web/service/configuration/AppCfgService.java
View File

@@ -352,7 +352,7 @@ public class AppCfgService extends BaseService {
specificServiceCfgDao.update(specificServiceCfg); specificServiceCfgDao.update(specificServiceCfg);
} }
Integer compileId = configGroupInfo.getCompileId(); Integer compileId = configGroupInfo.getCompileId();
Varibles maxPacket=asnIpCfgDao.getVaribles("max_allowed_packet"); // Varibles maxPacket=asnIpCfgDao.getVaribles("max_allowed_packet");
List<AppIpCfg> tempList=Lists.newArrayList(); List<AppIpCfg> tempList=Lists.newArrayList();
int len=0; int len=0;
for(BaseIpCfg cfg:cfgs) { for(BaseIpCfg cfg:cfgs) {
@@ -360,7 +360,7 @@ public class AppCfgService extends BaseService {
BeanUtils.copyProperties(cfg, entity); BeanUtils.copyProperties(cfg, entity);
entity.setCompileId(compileId); entity.setCompileId(compileId);
int tempLen=entity.toString().getBytes(Charset.forName("UTF-8")).length; int tempLen=entity.toString().getBytes(Charset.forName("UTF-8")).length;
if((len+tempLen)<maxPacket.getValue().longValue()) { if((len+tempLen)<Constants.MAX_ALLOWED_PACKET) {
tempList.add(entity); tempList.add(entity);
len+=tempLen; len+=tempLen;
}else { }else {

6
src/main/resources/nis.properties
View File

@@ -544,10 +544,12 @@ ipv4_ip_subnet_regexp_original=^(0|1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|[1-9])\\.(
#\u57df\u540d\u9a8c\u8bc1\u6b63\u5219 #\u57df\u540d\u9a8c\u8bc1\u6b63\u5219
domain_regexp=^(?=^.{3,255}$)[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+$ domain_regexp=^(?=^.{3,255}$)[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+$
#IP复用maat json中的ip region单次send 最大个数 #IP复用maat json中的ip region单次send 最大个数
maat_json_send_size=2000 maat_json_send_size=20000
#导入验证的线程数量 #导入验证的线程数量
mulity_thread_size=5 mulity_thread_size=5
ntcRadiusReport=ntcRadiusReport ntcRadiusReport=ntcRadiusReport
ntcCollectVoipLog=ntcCollectVoipLogs ntcCollectVoipLog=ntcCollectVoipLogs
ntcKeywordsUrlLog=ntcKeywordsUrlLogs ntcKeywordsUrlLog=ntcKeywordsUrlLogs
save_and_del_thread_size=5 save_and_del_thread_size=5
#mysql 单个sql大小限制,根据mariadb相关参数配置
max_allowed_packet=1048576