gfwleak/k18-ntcs-web-ntc
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(1)asn no放入eCache中

Browse Source 
(2)导入验证采用多线程验证，优化验证速度
(3)asn ip导入方式调整(未采用多线程，因为redis承受不了)
(4)asn ip列表展示速度优化
(5)导入方式重写:采用csv模式，限制采用xlsx格式，加载80万数据不会内存溢出.
This commit is contained in:
wangxin
2018-11-11 19:36:53 +08:00
parent 40bcfea683
commit 80dde7d6a0
53 changed files with 4831 additions and 2996 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/com/nis/web/dao/basics/AsnIpCfgDao.java
View File

@@ -5,13 +5,14 @@ import java.util.List;
import org.apache.ibatis.annotations.Param;
import com.nis.domain.basics.AsnIpCfg;
import com.nis.domain.specific.ConfigGroupInfo;
import com.nis.domain.basics.Varibles;
import com.nis.web.dao.CrudDao;
import com.nis.web.dao.MyBatisDao;
@MyBatisDao
public interface AsnIpCfgDao extends CrudDao<AsnIpCfg>{
public List<AsnIpCfg> findPage(AsnIpCfg cfg);
public void updateValid(@Param("isValid")Integer isValid,@Param("ids")String ids);
//public void updateValid(@Param("isValid")Integer isValid,@Param("ids")String ids);
public void delete(@Param("ids")String ids);
public void updateIssued(AsnIpCfg cfg);
public List<AsnIpCfg> getByIds(@Param("ids")String ids);
public List<Integer> hasGroupIds(@Param("ids")String ids);
@@ -19,7 +20,9 @@ public interface AsnIpCfgDao extends CrudDao<AsnIpCfg>{
// public List<ConfigGroupInfo> findPolicyGroupInfosByType(@Param("groupId")Integer groupId);
public List<Integer> findOtherIps(@Param("groupId")Integer groupId,@Param("cfgId")Integer cfgId);
public List<Integer> countValidIPs(@Param("groups")String groups,@Param("ids")String ids);
public ConfigGroupInfo getInfoByAsnNo(@Param("asnId")Long asnNo);
public List<AsnIpCfg> getByAsnGroup(@Param("groupId")long groupId,@Param("asnId")long asnNo);
public void deleteByAsnGroup(@Param("groupId")long groupId,@Param("asnId")long asnNo);
public List<AsnIpCfg> getByAsnNo(@Param("asnId")long asnNo);
public int deleteByAsnGroup(@Param("asnId")String asnNo);
public int insertBatch(List<AsnIpCfg> list);
public Varibles getVaribles(@Param("name")String name);
}

79
src/main/java/com/nis/web/dao/basics/AsnIpCfgDao.xml
View File

@@ -35,14 +35,17 @@
<result column="cfg_region_code" property="cfgRegionCode" jdbcType="INTEGER" />
<result column="cfg_type" property="cfgType" jdbcType="VARCHAR" />
<result column="asn_ip_group" property="asnIpGroup" jdbcType="INTEGER" />
<result column="asn_ip_group_name" property="asnIpGroupName" jdbcType="VARCHAR" />
<!-- <result column="asn_ip_group_name" property="asnIpGroupName" jdbcType="VARCHAR" /> -->
<result column="user_region1" property="userRegion1" jdbcType="VARCHAR" />
<result column="user_region2" property="userRegion2" jdbcType="VARCHAR" />
<result column="user_region3" property="userRegion3" jdbcType="VARCHAR" />
<result column="user_region4" property="userRegion4" jdbcType="VARCHAR" />
<result column="user_region5" property="userRegion5" jdbcType="VARCHAR" />
</resultMap>
<resultMap id="varibleMap" type="com.nis.domain.basics.Varibles">
<result column="variable_name" property="variableName" jdbcType="VARCHAR" />
<result column="value" property="value" jdbcType="VARCHAR" />
</resultMap>
<sql id="columns">
r.cfg_id,r.cfg_desc,r.ip_type,r.src_ip_address,r.ip_pattern,r.port_pattern,r.src_port
,r.protocol,r.protocol_id,r.direction,r.cfg_type,r.action,r.dest_port,r.dest_ip_address
@@ -58,10 +61,10 @@
<include refid="columns"></include>
<trim prefix="," prefixOverrides=",">
,s.name as creator_name,e.name as editor_name,u.name as auditor_name
,asn.group_name as asn_ip_group_name
<!-- ,asn.group_name as asn_ip_group_name -->
</trim>
from asn_ip_cfg r
left join config_group_info asn on asn.group_id=r.asn_ip_group
<!-- left join config_group_info asn on asn.group_id=r.asn_ip_group -->
left join sys_user s on r.creator_id=s.id
left join sys_user e on r.editor_id=e.id
left join sys_user u on r.auditor_id=u.id
@@ -206,10 +209,10 @@
from asn_ip_cfg r
where r.cfg_id in (${ids}) and is_valid !=-1
</select>
<select id="getByAsnGroup" resultMap="asnIpCfgMap" parameterType="java.lang.Long" >
<select id="getByAsnNo" resultMap="asnIpCfgMap" parameterType="java.lang.Long" >
select
<include refid="columns"></include>
from asn_ip_cfg r where r.is_valid!=-1 and r.asn_ip_group=#{groupId} and r.user_region1=#{asnId}
from asn_ip_cfg r where r.is_valid!=-1 and r.user_region1=#{asnId}
</select>
<insert id="insert" parameterType="com.nis.domain.basics.AsnIpCfg" >
insert into asn_ip_cfg (
@@ -393,15 +396,18 @@
and cfg_id = #{cfgId,jdbcType=INTEGER}
</where>
</update>
<update id="updateValid" parameterType="com.nis.domain.basics.AsnIpCfg" >
<!-- <update id="updateValid" parameterType="com.nis.domain.basics.AsnIpCfg" >
update asn_ip_cfg set is_valid=#{isValid} where cfg_id in (${ids})
</update>
</update> -->
<!-- <update id="deleteByAsnGroup" parameterType="java.lang.Integer" >
update asn_ip_cfg set is_valid=-1 where asn_ip_group=#{groupId} and user_region1=#{asnId}
</update> -->
<delete id="deleteByAsnGroup" parameterType="java.lang.Long" >
delete from asn_ip_cfg where asn_ip_group=#{groupId} and user_region1=#{asnId}
<delete id="deleteByAsnGroup" parameterType="java.lang.String" >
delete from asn_ip_cfg where user_region1 in(${asnId}) limit 10000
</delete>
<update id="delete" parameterType="java.lang.String" >
delete from asn_ip_cfg where cfg_id in (${ids})
</update>
<select id="findOtherIps" resultType="java.lang.Integer" parameterType="java.lang.Integer">
select 1 from asn_ip_cfg where is_valid=1 and asn_ip_group=#{groupId} and cfg_id !=#{cfgId} limit 1
</select>
@@ -411,8 +417,55 @@
<select id="countValidIPs" resultType="java.lang.Integer" parameterType="java.lang.String">
select count(1) from asn_ip_cfg where is_valid !=-1 and asn_ip_group in(${groups}) and cfg_id not in(${ids}) GROUP BY asn_ip_group;
</select>
<select id="getInfoByAsnNo" resultType="com.nis.domain.specific.ConfigGroupInfo" parameterType="java.lang.Long">
select id,group_id,group_name,is_issued,insert_time,update_time,group_type,compile_id,asn_id
from config_group_info c where c.asn_id= #{asnId} and c.group_type=4
<select id="getVaribles" resultMap="varibleMap" parameterType="java.lang.String">
show VARIABLES like #{name}
</select>
<insert id="insertBatch">
INSERT INTO asn_ip_cfg
(CFG_DESC,ACTION,IS_VALID,IS_AUDIT,CREATOR_ID,CREATE_TIME,EDITOR_ID,EDIT_TIME,AUDITOR_ID,
AUDIT_TIME,SERVICE_ID,REQUEST_ID,region_id,IS_AREA_EFFECTIVE,CLASSIFY,ATTRIBUTE,LABLE,
AREA_EFFECTIVE_IDS,function_id,ip_type,src_ip_address,ip_pattern,port_pattern,src_port,
protocol,protocol_id,direction,dest_port,dest_ip_address,cfg_type,cfg_region_code,
asn_ip_group,user_region1,user_region2,user_region3,user_region4,user_region5)
VALUES
<foreach collection ="list" item="asnIp" separator =",">
(#{asnIp.cfgDesc,jdbcType=VARCHAR},
#{asnIp.action,jdbcType=INTEGER},
#{asnIp.isValid,jdbcType=INTEGER},
0,
#{asnIp.creatorId,jdbcType=INTEGER},
#{asnIp.createTime,jdbcType=TIMESTAMP},
#{asnIp.editorId,jdbcType=INTEGER},
#{asnIp.editTime,jdbcType=TIMESTAMP},
#{asnIp.auditorId,jdbcType=INTEGER},
#{asnIp.auditTime,jdbcType=TIMESTAMP},
#{asnIp.serviceId,jdbcType=INTEGER},
#{asnIp.requestId,jdbcType=INTEGER},
#{asnIp.regionId,jdbcType=INTEGER},
#{asnIp.isAreaEffective,jdbcType=INTEGER},
#{asnIp.classify,jdbcType=VARCHAR},
#{asnIp.attribute,jdbcType=VARCHAR},
#{asnIp.lable,jdbcType=VARCHAR},
#{asnIp.areaEffectiveIds,jdbcType=VARCHAR},
#{asnIp.functionId,jdbcType=INTEGER},
#{asnIp.ipType,jdbcType=INTEGER},
#{asnIp.srcIpAddress,jdbcType=VARCHAR},
#{asnIp.ipPattern,jdbcType=INTEGER},
#{asnIp.portPattern,jdbcType=INTEGER},
#{asnIp.srcPort,jdbcType=VARCHAR},
#{asnIp.protocol,jdbcType=INTEGER},
#{asnIp.protocolId,jdbcType=INTEGER},
#{asnIp.direction,jdbcType=INTEGER},
#{asnIp.destPort,jdbcType=VARCHAR},
#{asnIp.destIpAddress,jdbcType=VARCHAR},
#{asnIp.cfgType,jdbcType=VARCHAR},
#{asnIp.cfgRegionCode,jdbcType=INTEGER},
#{asnIp.asnIpGroup,jdbcType=INTEGER},
#{asnIp.userRegion1,jdbcType=VARCHAR},
#{asnIp.userRegion2,jdbcType=VARCHAR},
#{asnIp.userRegion3,jdbcType=VARCHAR},
#{asnIp.userRegion4,jdbcType=VARCHAR},
#{asnIp.userRegion5,jdbcType=VARCHAR})
</foreach >
</insert>
</mapper>

1
src/main/java/com/nis/web/dao/basics/PolicyGroupInfoDao.java
View File

@@ -19,4 +19,5 @@ public interface PolicyGroupInfoDao extends CrudDao<PolicyGroupInfo> {
PolicyGroupInfo getInfoByAsnNo(PolicyGroupInfo policyGroupInfo);
List<PolicyGroupInfo> getHasAreaPolicyGroups(int groupType);
int insertBatch(List<PolicyGroupInfo> list);
}

24
src/main/java/com/nis/web/dao/basics/PolicyGroupInfoDao.xml
View File

@@ -113,6 +113,30 @@
#{asnNo,jdbcType=INTEGER},
#{description,jdbcType=VARCHAR}
)
</insert>
<insert id="insertBatch">
insert into policy_group_info(
IS_VALID,
CREATOR_ID,
CREATE_TIME,
GROUP_NAME,
GROUP_TYPE,
SERVICE_GROUP_ID,
ASN_NO,
DESCRIPTION
)values
<foreach collection ="list" item="info" separator =",">
(
1,
#{info.creatorId,jdbcType=INTEGER},
#{info.createTime,jdbcType=TIMESTAMP},
#{info.groupName,jdbcType=VARCHAR},
#{info.groupType,jdbcType=INTEGER},
#{info.serviceGroupId,jdbcType=INTEGER},
#{info.asnNo,jdbcType=INTEGER},
#{info.description,jdbcType=VARCHAR}
)
</foreach>
</insert>
<update id="update" parameterType="com.nis.domain.basics.PolicyGroupInfo" >
update policy_group_info

22
src/main/java/com/nis/web/dao/specific/ConfigGroupInfoDao.java Normal file
View File

@@ -0,0 +1,22 @@
package com.nis.web.dao.specific;
import java.util.List;
import org.apache.ibatis.annotations.Param;
import org.springframework.cache.annotation.Cacheable;
import com.nis.domain.specific.ConfigGroupInfo;
import com.nis.web.dao.CrudDao;
import com.nis.web.dao.MyBatisDao;
@MyBatisDao
public interface ConfigGroupInfoDao extends CrudDao<ConfigGroupInfo>{
List<ConfigGroupInfo> findAllList(int groupType);
void insertConfigGroupInfo(ConfigGroupInfo entity);
int insertBatch(List<ConfigGroupInfo> list);
void updateConfigGroupInfobyGroupId(ConfigGroupInfo entity);
ConfigGroupInfo getConfigGroupInfoByGroupId(Integer groupId);
@Cacheable(value="asnNoCache",key="#asnNo")
ConfigGroupInfo getInfoByAsnNo(@Param("asnId")Long asnNo);
Integer getIssuedConfigGroupInfoByGroupIds(@Param("groupIds")String groupIds);
Long getCountByType(@Param("groupType")Integer groupType);
}

64
src/main/java/com/nis/web/dao/specific/ConfigGroupInfoDao.xml Normal file
View File

@@ -0,0 +1,64 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.nis.web.dao.specific.ConfigGroupInfoDao">
<resultMap id="configGroupInfoMap" type="com.nis.domain.specific.ConfigGroupInfo">
<id column="id" property="id" jdbcType="BIGINT" />
<result column="group_id" property="groupId" jdbcType="BIGINT" />
<result column="group_name" property="groupName" jdbcType="VARCHAR" />
<result column="is_issued" property="isIssued" jdbcType="INTEGER" />
<result column="insert_time" property="insertTime" jdbcType="TIMESTAMP" />
<result column="update_time" property="updateTime" jdbcType="TIMESTAMP" />
<result column="group_type" property="groupType" jdbcType="INTEGER" />
<result column="compile_id" property="compileId" jdbcType="INTEGER" />
<result column="asn_id" property="asnId" jdbcType="BIGINT" />
</resultMap>
<sql id="columns">
id,group_id,group_name,is_issued,insert_time,update_time,group_type,compile_id,asn_id
</sql>
<select id="findAllList" resultMap="configGroupInfoMap">
select
<include refid="columns"></include>
from config_group_info
where group_type =#{groupType}
</select>
<!-- 新增配置分组信息 -->
<insert id="insertConfigGroupInfo" parameterType="com.nis.domain.specific.ConfigGroupInfo" useGeneratedKeys="true">
insert into config_group_info (id,group_id,group_name,is_issued,insert_time,group_type,compile_id,asn_id)
values(#{id},#{groupId},#{groupName},#{isIssued},now(),#{groupType},#{compileId},#{asnId})
</insert>
<insert id="insertBatch">
insert into config_group_info (id,group_id,group_name,is_issued,insert_time,group_type,compile_id,asn_id)
values
<foreach collection ="list" item="info" separator =",">
(#{info.id},#{info.groupId},#{info.groupName},#{info.isIssued},now(),#{info.groupType},#{info.compileId},#{info.asnId})
</foreach>
</insert>
<!-- 修改配置分组状态信息 -->
<update id="updateConfigGroupInfobyGroupId" parameterType="com.nis.domain.specific.ConfigGroupInfo">
UPDATE config_group_info set is_issued = #{isIssued},update_time=now()
<if test="compileId != null and compileId !='' ">
,compile_id= #{compileId}
</if>
where group_id = #{groupId}
</update>
<select id="getInfoByAsnNo" resultType="com.nis.domain.specific.ConfigGroupInfo" parameterType="java.lang.Long">
select id,group_id,group_name,is_issued,insert_time,update_time,group_type,compile_id,asn_id
from config_group_info c where c.asn_id= #{asnId} and c.group_type=4
</select>
<!-- 根据groupId查出配置分组信息 -->
<select id="getConfigGroupInfoByGroupId" resultType="com.nis.domain.specific.ConfigGroupInfo" parameterType="java.lang.Integer">
select id,group_id,group_name,is_issued,insert_time,update_time,group_type,compile_id,asn_id
from config_group_info where group_id= #{groupId}
</select>
<!-- <select id="findPolicyGroupInfosByType" resultType="com.nis.domain.specific.ConfigGroupInfo" parameterType="java.lang.Integer">
select id,group_id,group_name,is_issued,insert_time,update_time,group_type,compile_id,asn_id
from config_group_info c where c.group_type= #{groupType}
</select> -->
<select id="getIssuedConfigGroupInfoByGroupIds" resultType="java.lang.Integer" parameterType="java.lang.String">
select count(1) from config_group_info c where c.group_id in(${groupIds}) and is_issued=1
</select>
<select id="getCountByType" resultType="java.lang.Long" parameterType="java.lang.Integer">
select count(1) from config_group_info c where c.group_type=${groupType}
</select>
</mapper>

10
src/main/java/com/nis/web/dao/specific/SpecificServiceCfgDao.java
View File

@@ -3,6 +3,8 @@ package com.nis.web.dao.specific;
import java.util.List;
import org.apache.ibatis.annotations.Param;
import org.springframework.cache.annotation.CachePut;
import org.springframework.cache.annotation.Cacheable;
import com.nis.domain.specific.ConfigGroupInfo;
import com.nis.domain.specific.SpecificServiceCfg;
@@ -42,14 +44,6 @@ public interface SpecificServiceCfgDao extends CrudDao<SpecificServiceCfg> {
* @return
*/
List<SpecificServiceCfg> getChildrenById(Integer specServiceId);
ConfigGroupInfo getConfigGroupInfoByGroupId(Integer groupId);
ConfigGroupInfo findPolicyGroupInfosByType(@Param("groupType")Integer groupType);
Integer getIssuedConfigGroupInfoByGroupIds(@Param("groupIds")String groupIds);
Integer insertConfigGroupInfo(ConfigGroupInfo entity);
Integer updateConfigGroupInfobyGroupId(ConfigGroupInfo entity);
Integer getParentType(Integer specServiceId);
Integer getParentCode(Integer specServiceId);
SpecificServiceCfg getRepeat(@Param("specServiceCode")Integer code, @Param("cfgType")Integer cfgType,@Param("parentId")Integer parentId);

28
src/main/java/com/nis/web/dao/specific/SpecificServiceCfgDao.xml
View File

@@ -213,32 +213,4 @@
<select id="getChildrenById" resultMap="CFGResultMap" parameterType="java.lang.Integer">
SELECT * FROM specific_service_cfg s WHERE s.is_valid = 1 and s.parent_id = #{specServiceId}
</select>
<!-- 根据groupId查出配置分组信息 -->
<select id="getConfigGroupInfoByGroupId" resultType="com.nis.domain.specific.ConfigGroupInfo" parameterType="java.lang.Integer">
select id,group_id,group_name,is_issued,insert_time,update_time,group_type,compile_id,asn_id
from config_group_info c where c.group_id= #{groupId}
</select>
<select id="getIssuedConfigGroupInfoByGroupIds" resultType="java.lang.Integer" parameterType="java.lang.String">
select count(1) from config_group_info c where c.group_id in(${groupIds}) and is_issued=1
</select>
<!-- 修改配置分组状态信息 -->
<update id="updateConfigGroupInfobyGroupId" parameterType="com.nis.domain.specific.ConfigGroupInfo">
UPDATE config_group_info set is_issued = #{isIssued},update_time=now()
<if test="compileId != null and compileId !='' ">
,compile_id= #{compileId}
</if>
where group_id = #{groupId}
</update>
<!-- 新增配置分组信息 -->
<insert id="insertConfigGroupInfo" parameterType="com.nis.domain.specific.ConfigGroupInfo" useGeneratedKeys="true">
insert into config_group_info (id,group_id,group_name,is_issued,insert_time,group_type,compile_id,asn_id)
values(#{id},#{groupId},#{groupName},#{isIssued},now(),#{groupType},#{compileId},#{asnId})
</insert>
<select id="findPolicyGroupInfosByType" resultType="com.nis.domain.specific.ConfigGroupInfo" parameterType="java.lang.Integer">
select id,group_id,group_name,is_issued,insert_time,update_time,group_type,compile_id,asn_id
from config_group_info c where c.group_type= #{groupType}
</select>
</mapper>