(1)asn no放入eCache中
(2)导入验证采用多线程验证，优化验证速度 (3)asn ip导入方式调整(未采用多线程，因为redis承受不了) (4)asn ip列表展示速度优化 (5)导入方式重写:采用csv模式，限制采用xlsx格式，加载80万数据不会内存溢出.
This commit is contained in:
wangxin
147
src/main/java/com/nis/util/AsnCacheUtils.java
Normal file
147
src/main/java/com/nis/util/AsnCacheUtils.java
Normal file
@@ -0,0 +1,147 @@
|
||||
package com.nis.util;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import com.nis.domain.specific.ConfigGroupInfo;
|
||||
import com.nis.web.dao.specific.ConfigGroupInfoDao;
|
||||
import com.nis.web.service.SpringContextHolder;
|
||||
|
||||
import jersey.repackaged.com.google.common.collect.Maps;
|
||||
import net.sf.ehcache.Cache;
|
||||
import net.sf.ehcache.Element;
|
||||
|
||||
/**
|
||||
* asn no缓存工具类
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
public class AsnCacheUtils{
|
||||
private static Logger logger=Logger.getLogger(AsnCacheUtils.class);
|
||||
/**
|
||||
* ASN号缓存
|
||||
*/
|
||||
private static final String ASN_NO_CACHE = "asnNoCache";
|
||||
private static final int cache_rage = 1000;
|
||||
private final static ConfigGroupInfoDao configGroupInfoDao = SpringContextHolder.getBean(ConfigGroupInfoDao.class);
|
||||
/**
|
||||
* 获取缓存
|
||||
* @param cacheName
|
||||
* @param key
|
||||
* @return
|
||||
*/
|
||||
public static ConfigGroupInfo get(Long key) {
|
||||
Element element = getCache(ASN_NO_CACHE).get(key/cache_rage);
|
||||
if(element!=null) {
|
||||
Map<Long,ConfigGroupInfo> map=(Map<Long,ConfigGroupInfo>)element.getObjectValue();
|
||||
if(map.containsKey(key)) {
|
||||
return map.get(key);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
public static Map<Long,ConfigGroupInfo> getMap(Object key) {
|
||||
Element element = getCache(ASN_NO_CACHE).get(key);
|
||||
return (Map<Long,ConfigGroupInfo>)element.getObjectValue();
|
||||
}
|
||||
/**
|
||||
* 初始化缓存
|
||||
*/
|
||||
//@PostConstruct
|
||||
public synchronized static void init() {
|
||||
long start=System.currentTimeMillis();
|
||||
logger.warn("AsnCacheUtils init start...");
|
||||
Cache cache=getCache(ASN_NO_CACHE);
|
||||
//查询总量
|
||||
Long count=configGroupInfoDao.getCountByType(4);
|
||||
//缓存key
|
||||
boolean loadDatabase=false;
|
||||
if(cache.getKeys().size()==0) {
|
||||
loadDatabase=true;
|
||||
}else {
|
||||
long c=0l;
|
||||
for(Object key:cache.getKeys()) {
|
||||
c+=getMap(key).size();
|
||||
}
|
||||
if(c!=count) {
|
||||
loadDatabase=true;
|
||||
}
|
||||
}
|
||||
if(loadDatabase) {
|
||||
List<ConfigGroupInfo> list=configGroupInfoDao.findAllList(4);
|
||||
Map<Long,Map<Long,ConfigGroupInfo>> groupMap=Maps.newHashMap();
|
||||
for(ConfigGroupInfo configGroupInfo:list) {
|
||||
if(groupMap.containsKey(configGroupInfo.getAsnId()/cache_rage)) {
|
||||
groupMap.get(configGroupInfo.getAsnId()/cache_rage)
|
||||
.put(configGroupInfo.getAsnId(), configGroupInfo);
|
||||
}else {
|
||||
Map<Long,ConfigGroupInfo> m=Maps.newHashMap();
|
||||
m.put(configGroupInfo.getAsnId(), configGroupInfo);
|
||||
groupMap.put(configGroupInfo.getAsnId()/cache_rage, m);
|
||||
}
|
||||
}
|
||||
for(Entry<Long, Map<Long, ConfigGroupInfo>> e:groupMap.entrySet()) {
|
||||
Element element = new Element(e.getKey(), e.getValue());
|
||||
cache.put(element);
|
||||
}
|
||||
}
|
||||
long end=System.currentTimeMillis();
|
||||
logger.warn("AsnCacheUtils init finish,cost:"+(end-start));
|
||||
}
|
||||
/**
|
||||
* 写入ASN_NO_CACHE缓存
|
||||
* @param key
|
||||
* @return
|
||||
*/
|
||||
public static void put(Long key, ConfigGroupInfo value) {
|
||||
Long _key=key/cache_rage;
|
||||
Element element = getCache(ASN_NO_CACHE).get(_key);
|
||||
if(element==null) {
|
||||
Map<Long,ConfigGroupInfo> map=Maps.newHashMap();
|
||||
map.put(key, value);
|
||||
element = new Element(_key, map);
|
||||
}else {
|
||||
Map<Long,ConfigGroupInfo> map=(Map<Long,ConfigGroupInfo>)element.getObjectValue();
|
||||
map.put(key, value);
|
||||
element = new Element(_key, map);
|
||||
}
|
||||
getCache(ASN_NO_CACHE).put(element);
|
||||
}
|
||||
/**
|
||||
* 从缓存中移除
|
||||
* @param cacheName
|
||||
* @param key
|
||||
*/
|
||||
public static void remove(String key) {
|
||||
getCache(ASN_NO_CACHE).remove(key);
|
||||
}
|
||||
public static void remove(Long key) {
|
||||
Long _key=key/cache_rage;
|
||||
Element element = getCache(ASN_NO_CACHE).get(_key);
|
||||
if(element!=null) {
|
||||
Map<Long,ConfigGroupInfo> map=(Map<Long,ConfigGroupInfo>)element.getObjectValue();
|
||||
if(map.containsKey(key)) {
|
||||
map.remove(key);
|
||||
}
|
||||
if(map.isEmpty()) {
|
||||
getCache(ASN_NO_CACHE).remove(_key);
|
||||
}else {
|
||||
element=new Element(_key,map);
|
||||
getCache(ASN_NO_CACHE).put(element);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
private static Cache getCache(String cacheName){
|
||||
Cache cache = CacheUtils.getCacheManager().getCache(cacheName);
|
||||
if (cache == null){
|
||||
CacheUtils.getCacheManager().addCache(cacheName);
|
||||
cache = CacheUtils.getCacheManager().getCache(cacheName);
|
||||
cache.getCacheConfiguration().setEternal(true);
|
||||
}
|
||||
return cache;
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package com.nis.util;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.google.gson.GsonBuilder;
|
||||
|
||||
@@ -707,4 +708,21 @@ public final class Constants {
|
||||
public static final String CERT_VALIDATE_FILE=Configurations.getStringProperty("cert_validate_file", "x509");
|
||||
//证书校验成功关键字
|
||||
public static final String CERT_VALIDATE_SUCCESS_INFO=Configurations.getStringProperty("cert_validate_success_info", "x509");
|
||||
/**
|
||||
* 一些正则的pattern，预先编译,避免多次编译
|
||||
*/
|
||||
public static final Pattern IPV4_IP_RANGE_REGEXP_NEW_PATTERN = Pattern.compile(IPV4_IP_RANGE_REGEXP_NEW);
|
||||
public static final Pattern RANGE_PATTERN = Pattern.compile("\\d+\\-\\d+");
|
||||
public static final Pattern PORT_MASK_PATTERN = Pattern.compile("^[0-9]+/[0-9]+$");
|
||||
public static final Pattern PORT_PATTERN = Pattern.compile("^[0-9]+$");
|
||||
public static final Pattern IPV4_IP_SUBNET_ORIGINAL_PATTERN = Pattern.compile(IPV4_IP_SUBNET_REGEXP_ORIGINAL);
|
||||
public static final Pattern IPV4_IP_SUBNET_PATTERN = Pattern.compile(IPV4_IP_SUBNET_REGEXP);
|
||||
public static final Pattern IPV6_IP_SUBNET_PATTERN = Pattern.compile(IPV6_IP_SUBNET_REGEXP);
|
||||
public static final Pattern IPV4_IP_RANGE_PATTERN = Pattern.compile(IPV4_IP_RANGE_REGEXP);
|
||||
public static final Pattern IPV6_IP_RANGE_PATTERN = Pattern.compile(IPV6_IP_RANGE_REGEXP);
|
||||
public static final Pattern IPV4_IP_PATTERN = Pattern.compile(IPV4_IP_REGEXP);
|
||||
public static final Pattern IPV6_IP_PATTERN = Pattern.compile(IPV6_IP_REGEXP);
|
||||
//IP复用maat json中的ip region单次send 最大个数
|
||||
public static final Integer MAAT_JSON_SEND_SIZE=Configurations.getIntProperty("maat_json_send_size", 1000);
|
||||
public static final Integer MULITY_THREAD_SIZE=Configurations.getIntProperty("mulity_thread_size", 5);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
package com.nis.util.excel;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.jets3t.service.ServiceException;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
|
||||
import com.beust.jcommander.internal.Lists;
|
||||
import com.beust.jcommander.internal.Sets;
|
||||
import com.nis.domain.FunctionRegionDict;
|
||||
import com.nis.domain.FunctionServiceDict;
|
||||
import com.nis.domain.configuration.ComplexkeywordCfg;
|
||||
import com.nis.domain.configuration.DnsResStrategy;
|
||||
import com.nis.util.Constants;
|
||||
import com.nis.web.service.configuration.DnsResStrategyService;
|
||||
|
||||
public class CheckComplexStringFormatThread implements Callable<String>{
|
||||
private Logger logger=Logger.getLogger(CheckComplexStringFormatThread.class);
|
||||
private BlockingQueue<? extends Object> srcQueue;
|
||||
private BlockingQueue<ComplexkeywordCfg> destQueue;
|
||||
private Properties prop;
|
||||
private DnsResStrategyService dnsResStrategyService;
|
||||
private FunctionServiceDict serviceDict;
|
||||
private FunctionRegionDict regionDict;
|
||||
public CheckComplexStringFormatThread(FunctionServiceDict serviceDict,FunctionRegionDict regionDict,Properties prop,BlockingQueue<? extends Object> srcQueue,BlockingQueue<ComplexkeywordCfg> destQueue) {
|
||||
this.serviceDict=serviceDict;
|
||||
this.regionDict=regionDict;
|
||||
this.srcQueue=srcQueue;
|
||||
this.destQueue=destQueue;
|
||||
this.prop=prop;
|
||||
}
|
||||
@Override
|
||||
public String call() throws Exception {
|
||||
List<Object> dataList=Lists.newArrayList(Constants.MAAT_JSON_SEND_SIZE);
|
||||
String msg=null;
|
||||
while(!srcQueue.isEmpty()) {
|
||||
int size=srcQueue.drainTo(dataList,Constants.MAAT_JSON_SEND_SIZE);
|
||||
if(regionDict.getRegionType().intValue()==3) {
|
||||
try {
|
||||
List<ComplexkeywordCfg> cfgs=this.checkComplexStringCfg(dataList);
|
||||
destQueue.addAll(cfgs);
|
||||
}catch (Exception e) {
|
||||
// TODO: handle exception
|
||||
msg=e.getMessage();
|
||||
break;
|
||||
}
|
||||
}
|
||||
dataList.clear();
|
||||
}
|
||||
return msg;
|
||||
}
|
||||
public List<ComplexkeywordCfg> checkComplexStringCfg(
|
||||
List<?> list) throws ServiceException {
|
||||
logger.warn("start to validate complexString data...");
|
||||
long start=System.currentTimeMillis();
|
||||
List<ComplexkeywordCfg> stringList = new ArrayList<ComplexkeywordCfg>();
|
||||
String exprTypeP = regionDict.getConfigExprType();
|
||||
if (StringUtils.isBlank(exprTypeP)) {
|
||||
throw new RuntimeException("Found String region,but exprType is Empty");
|
||||
}
|
||||
String matchMethodP = regionDict.getConfigMatchMethod();
|
||||
if (StringUtils.isBlank(matchMethodP)) {
|
||||
throw new RuntimeException("Found String region,but matchMethod is Empty");
|
||||
}
|
||||
String hexP = regionDict.getConfigHex();
|
||||
if (StringUtils.isBlank(hexP)) {
|
||||
throw new RuntimeException("Found String region,but hex is Empty");
|
||||
}
|
||||
String mulityKeywordsP = regionDict.getConfigMultiKeywords();
|
||||
if (StringUtils.isBlank(mulityKeywordsP)) {
|
||||
throw new RuntimeException("Found String region,but mulityKeywords is Empty");
|
||||
}
|
||||
String dirtrictP = regionDict.getConfigDistrict();
|
||||
StringBuffer errTip = new StringBuffer();
|
||||
Pattern pattern = Pattern.compile("\t|\r|\n|\b|\f");
|
||||
for (int i = 0; i < list.size(); i++) {
|
||||
StringBuffer errInfo = new StringBuffer();
|
||||
ComplexkeywordCfg baseStringCfg = new ComplexkeywordCfg();
|
||||
BeanUtils.copyProperties(list.get(i), baseStringCfg);
|
||||
if (regionDict.getRegionType().equals(3)) {
|
||||
if (regionDict.getFunctionId().equals(7)) {
|
||||
Long dnsStrategyId = baseStringCfg.getDnsStrategyId();
|
||||
if (dnsStrategyId != null&&dnsStrategyId>0) {
|
||||
List<DnsResStrategy> dnsStrategys = dnsResStrategyService.findDnsResStrategys(dnsStrategyId,
|
||||
Constants.VALID_YES, Constants.AUDIT_YES);
|
||||
if (dnsStrategys == null || dnsStrategys.size() == 0) {
|
||||
errInfo.append(String.format(prop.getProperty("is_incorrect"),
|
||||
prop.getProperty("dns_res_strategy")) + ";");
|
||||
}
|
||||
}
|
||||
}
|
||||
String keyword = baseStringCfg.getCfgKeywords();
|
||||
String district = baseStringCfg.getDistrict();
|
||||
if (StringUtils.isBlank(keyword)) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("can_not_null"), prop.getProperty("key_word") + " ") + ";");
|
||||
}
|
||||
if (StringUtils.isNotBlank(dirtrictP)) {
|
||||
if (StringUtils.isBlank(district)) {
|
||||
if (dirtrictP.indexOf(",") == -1) {
|
||||
baseStringCfg.setDistrict(dirtrictP);
|
||||
} else {
|
||||
// baseStringCfg.setDistrict(dirtrictP.split(",")[0]);
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("can_not_null"), prop.getProperty("district") + " ")
|
||||
+ ";");
|
||||
}
|
||||
} else if (dirtrictP.indexOf(district) == -1) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("district") + " ")
|
||||
+ ";");
|
||||
} else if (dirtrictP.indexOf("others")>-1&&district.equals("others")) {
|
||||
//不允许自定义匹配区域导入
|
||||
errInfo.append(prop.getProperty("district")+" "+
|
||||
String.format(prop.getProperty("can_not_be"), " 'others'")+ ";");
|
||||
}
|
||||
}
|
||||
if (mulityKeywordsP.equals("0")) {
|
||||
if (keyword.indexOf("\n") > -1) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("not_multiple"), prop.getProperty("key_word")) + ";");
|
||||
}
|
||||
Matcher m = pattern.matcher(keyword);
|
||||
if (m.find()) {
|
||||
errInfo.append(String.format(prop.getProperty("has_invisible_char"),
|
||||
prop.getProperty("key_word") + " '" + keyword + "'") + ";");
|
||||
}
|
||||
} else {
|
||||
boolean has = false;
|
||||
Set<String> keywordSet=Sets.newHashSet();
|
||||
|
||||
for (String key : keyword.split("\n")) {
|
||||
Matcher m = pattern.matcher(key);
|
||||
if (m.find()) {
|
||||
has = true;
|
||||
errInfo.append(String.format(prop.getProperty("has_invisible_char"),
|
||||
prop.getProperty("key_word") + " '" + key + "'") + ";");
|
||||
break;
|
||||
}
|
||||
if(!keywordSet.contains(key)) {
|
||||
keywordSet.add(key);
|
||||
}else {
|
||||
errInfo.append(prop.getProperty("key_word") + " '" + key + "'"+" "+prop.getProperty("repeat") + ";");
|
||||
}
|
||||
}
|
||||
if (!has) {
|
||||
if(keyword.replaceAll("\n","").length()>1024) {
|
||||
errInfo.append(String.format(prop.getProperty("most_keywords"),
|
||||
prop.getProperty("key_word")) + ";");
|
||||
}else {
|
||||
String reWord = keyword.replaceAll("\n", Constants.KEYWORD_EXPR);
|
||||
baseStringCfg.setCfgKeywords(reWord);
|
||||
}
|
||||
}
|
||||
}
|
||||
Integer exprType = baseStringCfg.getExprType();
|
||||
boolean has = false;
|
||||
if (exprType == null) {
|
||||
if (exprTypeP.indexOf(",") == -1) {
|
||||
if (mulityKeywordsP.equals("0") && exprTypeP.equals("1")) {
|
||||
throw new RuntimeException("region dict config error,dict id is " + regionDict.getDictId());
|
||||
}
|
||||
baseStringCfg.setExprType(Integer.parseInt(exprTypeP));
|
||||
} else if (exprTypeP.indexOf("0") > -1 && mulityKeywordsP.equals("0")) {
|
||||
baseStringCfg.setExprType(0);
|
||||
} else if (exprTypeP.indexOf("1") > -1 && mulityKeywordsP.equals("1")
|
||||
&& keyword.indexOf("\n") > -1) {
|
||||
baseStringCfg.setExprType(1);
|
||||
} else if (exprTypeP.indexOf("0") > -1 && mulityKeywordsP.equals("1")
|
||||
&& keyword.indexOf("\n") == -1) {
|
||||
baseStringCfg.setExprType(0);
|
||||
} else {
|
||||
baseStringCfg.setExprType(Integer.parseInt(exprTypeP.split(",")[0]));
|
||||
}
|
||||
// errInfo.append(String.format(prop.getProperty("can_not_null"),
|
||||
// prop.getProperty("expression_type"))+";");
|
||||
} else {
|
||||
for (String exp : exprTypeP.split(",")) {
|
||||
if (exp.equals(exprType.toString())) {
|
||||
has = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!has) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("expression_type"))
|
||||
+ ";");
|
||||
}
|
||||
has = false;
|
||||
}
|
||||
exprType = baseStringCfg.getExprType();
|
||||
Integer matchMethod = baseStringCfg.getMatchMethod();
|
||||
if (matchMethod == null) {
|
||||
if (matchMethodP.indexOf(",") == -1) {
|
||||
if (exprTypeP.equals("1") && !matchMethodP.equals("0")) {
|
||||
throw new RuntimeException("region dict config error,dict id is " + regionDict.getDictId());
|
||||
}
|
||||
baseStringCfg.setMatchMethod(Integer.parseInt(matchMethodP));
|
||||
} else if (exprType != null && exprType.intValue() == 1) {
|
||||
if (matchMethodP.indexOf("0") > -1) {
|
||||
baseStringCfg.setMatchMethod(0);
|
||||
} else {
|
||||
throw new RuntimeException("region dict config error,dict id is " + regionDict.getDictId());
|
||||
}
|
||||
|
||||
} else {
|
||||
baseStringCfg.setMatchMethod(Integer.parseInt(matchMethodP.split(",")[0]));
|
||||
}
|
||||
// errInfo.append(String.format(prop.getProperty("can_not_null"),
|
||||
// prop.getProperty("match_method"))+";");
|
||||
} else {
|
||||
for (String exp : matchMethodP.split(",")) {
|
||||
if (exp.equals(matchMethod.toString())) {
|
||||
has = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!has) {
|
||||
errInfo.append(String.format(prop.getProperty("is_incorrect"), prop.getProperty("match_method"))
|
||||
+ ";");
|
||||
}
|
||||
}
|
||||
Integer isHex = baseStringCfg.getIsHex();
|
||||
Integer isCaseInsenstive = baseStringCfg.getIsCaseInsenstive();
|
||||
if (isHex == null || isCaseInsenstive == null) {
|
||||
if (isHex == null) {
|
||||
if (hexP.equals("0") || hexP.equals("2")) {
|
||||
baseStringCfg.setIsHex(0);
|
||||
} else if (hexP.equals("1")) {
|
||||
baseStringCfg.setIsHex(1);
|
||||
} else {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("can_not_null"), prop.getProperty("is_hex")) + ";");
|
||||
}
|
||||
}
|
||||
if (isCaseInsenstive == null) {
|
||||
if (hexP.equals("0") || hexP.equals("1")) {
|
||||
baseStringCfg.setIsCaseInsenstive(0);
|
||||
} else if (hexP.equals("2")) {
|
||||
baseStringCfg.setIsCaseInsenstive(1);
|
||||
} else {
|
||||
errInfo.append(String.format(prop.getProperty("can_not_null"),
|
||||
prop.getProperty("is_case_insenstive")) + ";");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (isHex.intValue() != 0 && isHex.intValue() != 1) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("is_hex")) + ";");
|
||||
}
|
||||
if (isCaseInsenstive.intValue() != 0 && isCaseInsenstive.intValue() != 1) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("is_case_insenstive"))
|
||||
+ ";");
|
||||
}
|
||||
if (hexP.indexOf("1") == -1 && isHex.intValue() == 1) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("is_hex")) + ";");
|
||||
}
|
||||
if (hexP.equals("1") && isHex.intValue() == 0) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("is_hex")) + ";");
|
||||
}
|
||||
if (hexP.indexOf("2") == -1 && isCaseInsenstive.intValue() == 1) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("is_case_insenstive"))
|
||||
+ ";");
|
||||
}
|
||||
if (hexP.equals("2") && isCaseInsenstive.intValue() == 0) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("is_case_insenstive"))
|
||||
+ ";");
|
||||
}
|
||||
}
|
||||
isHex = baseStringCfg.getIsHex();
|
||||
isCaseInsenstive = baseStringCfg.getIsCaseInsenstive();
|
||||
if (isHex != null && isCaseInsenstive != null) {
|
||||
if (isHex.intValue() == 0 && isCaseInsenstive.intValue() == 0) {
|
||||
baseStringCfg.setIsHexbin(0);
|
||||
} else if (isHex.intValue() == 1 && isCaseInsenstive.intValue() == 0) {
|
||||
baseStringCfg.setIsHexbin(1);
|
||||
} else if (isHex.intValue() == 0 && isCaseInsenstive.intValue() == 1) {
|
||||
baseStringCfg.setIsHexbin(2);
|
||||
}else {
|
||||
errInfo.append(prop.getProperty("hex_case_insensitive")+ ";");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (errInfo.toString().length() > 0) {//
|
||||
errTip.append(String.format(prop.getProperty("line"), baseStringCfg.getIndex()) + ",");
|
||||
errTip.append(errInfo);
|
||||
errTip.append("<br>");
|
||||
}
|
||||
stringList.add(baseStringCfg);
|
||||
}
|
||||
if (errTip.toString().length() > 0) {
|
||||
throw new ServiceException(errTip.toString());
|
||||
}
|
||||
long end=System.currentTimeMillis();
|
||||
logger.warn("validate complexString data finish,cost:"+(end-start));
|
||||
return stringList;
|
||||
}
|
||||
public DnsResStrategyService getDnsResStrategyService() {
|
||||
return dnsResStrategyService;
|
||||
}
|
||||
public void setDnsResStrategyService(DnsResStrategyService dnsResStrategyService) {
|
||||
this.dnsResStrategyService = dnsResStrategyService;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package com.nis.util.excel;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.jets3t.service.ServiceException;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
|
||||
import com.beust.jcommander.internal.Lists;
|
||||
import com.beust.jcommander.internal.Sets;
|
||||
import com.nis.domain.FunctionRegionDict;
|
||||
import com.nis.domain.FunctionServiceDict;
|
||||
import com.nis.domain.basics.PolicyGroupInfo;
|
||||
import com.nis.domain.configuration.ComplexkeywordCfg;
|
||||
import com.nis.domain.configuration.DnsResStrategy;
|
||||
import com.nis.util.Constants;
|
||||
import com.nis.web.service.basics.PolicyGroupInfoService;
|
||||
import com.nis.web.service.configuration.DnsResStrategyService;
|
||||
|
||||
public class CheckDnsResStrategyFormatThread implements Callable<String>{
|
||||
private Logger logger=Logger.getLogger(CheckDnsResStrategyFormatThread.class);
|
||||
private BlockingQueue<? extends Object> srcQueue;
|
||||
private BlockingQueue<DnsResStrategy> destQueue;
|
||||
private Properties prop;
|
||||
private DnsResStrategyService dnsResStrategyService;
|
||||
private FunctionServiceDict serviceDict;
|
||||
private FunctionRegionDict regionDict;
|
||||
private PolicyGroupInfoService policyGroupInfoService;
|
||||
public CheckDnsResStrategyFormatThread(FunctionServiceDict serviceDict,FunctionRegionDict regionDict,Properties prop,BlockingQueue<? extends Object> srcQueue,BlockingQueue<DnsResStrategy> destQueue) {
|
||||
this.serviceDict=serviceDict;
|
||||
this.regionDict=regionDict;
|
||||
this.srcQueue=srcQueue;
|
||||
this.destQueue=destQueue;
|
||||
this.prop=prop;
|
||||
}
|
||||
@Override
|
||||
public String call() throws Exception {
|
||||
List<Object> dataList=Lists.newArrayList(Constants.MAAT_JSON_SEND_SIZE);
|
||||
String msg=null;
|
||||
while(!srcQueue.isEmpty()) {
|
||||
int size=srcQueue.drainTo(dataList,Constants.MAAT_JSON_SEND_SIZE);
|
||||
if(regionDict.getRegionType().intValue()==6) {
|
||||
try {
|
||||
List<DnsResStrategy> cfgs=this.checkDnsResStrategyCfg(dataList);
|
||||
destQueue.addAll(cfgs);
|
||||
}catch (Exception e) {
|
||||
// TODO: handle exception
|
||||
msg=e.getMessage();
|
||||
break;
|
||||
}
|
||||
}
|
||||
dataList.clear();
|
||||
}
|
||||
return msg;
|
||||
}
|
||||
public List<DnsResStrategy> checkDnsResStrategyCfg(List<?> list)
|
||||
throws ServiceException {
|
||||
List<DnsResStrategy> dnsResStrategies=Lists.newArrayList();
|
||||
StringBuffer errTip = new StringBuffer();
|
||||
for (int i = 0; i < list.size(); i++) {
|
||||
StringBuffer errInfo = new StringBuffer();
|
||||
DnsResStrategy dnsResStrategy=new DnsResStrategy();
|
||||
BeanUtils.copyProperties(list.get(i), dnsResStrategy);
|
||||
String groupName=dnsResStrategy.getCfgDesc();
|
||||
if(StringUtils.isBlank(groupName)) {
|
||||
errInfo.append(String.format(prop.getProperty("can_not_null"),
|
||||
prop.getProperty("policy_name")) + ";");
|
||||
}
|
||||
Integer resGroup1Id=dnsResStrategy.getResGroup1Id();
|
||||
if(resGroup1Id==null) {
|
||||
errInfo.append(String.format(prop.getProperty("can_not_null"),
|
||||
prop.getProperty("group")) + ";");
|
||||
}else {
|
||||
PolicyGroupInfo info=policyGroupInfoService.getById(resGroup1Id);
|
||||
if(info==null) {
|
||||
errInfo.append(String.format(prop.getProperty("is_incorrect"),
|
||||
prop.getProperty("group")) + ";");
|
||||
}
|
||||
}
|
||||
Integer resGroup1Num=dnsResStrategy.getResGroup1Num();
|
||||
if(resGroup1Num==null) {
|
||||
errInfo.append(String.format(prop.getProperty("can_not_null"),
|
||||
prop.getProperty("res_group_num")) + ";");
|
||||
}
|
||||
String ttl=dnsResStrategy.getTtl();
|
||||
if(StringUtils.isBlank(ttl)) {
|
||||
errInfo.append(String.format(prop.getProperty("can_not_null"),
|
||||
prop.getProperty("min_ttl")) + ";");
|
||||
}else {
|
||||
Pattern p=Constants.RANGE_PATTERN;
|
||||
Matcher m=p.matcher(ttl);
|
||||
if(!m.matches()) {
|
||||
errInfo.append(String.format(prop.getProperty("is_incorrect"),
|
||||
prop.getProperty("min_ttl")) + ";");
|
||||
}else {
|
||||
String minTtl=ttl.split("-")[0];
|
||||
String maxTtl=ttl.split("-")[1];
|
||||
Integer min=null,max=null;
|
||||
try {
|
||||
min=Integer.parseInt(minTtl);
|
||||
} catch (Exception e) {
|
||||
// TODO: handle exception
|
||||
errInfo.append(String.format(prop.getProperty("is_incorrect"),
|
||||
prop.getProperty("min_ttl")) + ";");
|
||||
}
|
||||
try {
|
||||
max=Integer.parseInt(maxTtl);
|
||||
} catch (Exception e) {
|
||||
// TODO: handle exception
|
||||
errInfo.append(String.format(prop.getProperty("is_incorrect"),
|
||||
prop.getProperty("min_ttl")) + ";");
|
||||
}
|
||||
if(min.intValue()>max.intValue())
|
||||
{errInfo.append(String.format(prop.getProperty("is_incorrect"),
|
||||
prop.getProperty("min_ttl")) + ";");
|
||||
}else {
|
||||
dnsResStrategy.setMinTtl(min);
|
||||
dnsResStrategy.setMaxTtl(max);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (errInfo.toString().length() > 0) {//
|
||||
errTip.append(String.format(prop.getProperty("line"), dnsResStrategy.getIndex()) + ",");
|
||||
errTip.append(errInfo);
|
||||
errTip.append("<br>");
|
||||
}
|
||||
dnsResStrategies.add(dnsResStrategy);
|
||||
}
|
||||
if (errTip.toString().length() > 0) {
|
||||
throw new ServiceException(errTip.toString());
|
||||
}
|
||||
return dnsResStrategies;
|
||||
}
|
||||
public DnsResStrategyService getDnsResStrategyService() {
|
||||
return dnsResStrategyService;
|
||||
}
|
||||
public void setDnsResStrategyService(DnsResStrategyService dnsResStrategyService) {
|
||||
this.dnsResStrategyService = dnsResStrategyService;
|
||||
}
|
||||
public PolicyGroupInfoService getPolicyGroupInfoService() {
|
||||
return policyGroupInfoService;
|
||||
}
|
||||
public void setPolicyGroupInfoService(PolicyGroupInfoService policyGroupInfoService) {
|
||||
this.policyGroupInfoService = policyGroupInfoService;
|
||||
}
|
||||
|
||||
}
|
||||
1046
src/main/java/com/nis/util/excel/CheckIpFormatThread.java
Normal file
1046
src/main/java/com/nis/util/excel/CheckIpFormatThread.java
Normal file
File diff suppressed because it is too large
Load Diff
330
src/main/java/com/nis/util/excel/CheckStringFormatThread.java
Normal file
330
src/main/java/com/nis/util/excel/CheckStringFormatThread.java
Normal file
@@ -0,0 +1,330 @@
|
||||
package com.nis.util.excel;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.jets3t.service.ServiceException;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
|
||||
import com.beust.jcommander.internal.Lists;
|
||||
import com.beust.jcommander.internal.Sets;
|
||||
import com.nis.domain.FunctionRegionDict;
|
||||
import com.nis.domain.FunctionServiceDict;
|
||||
import com.nis.domain.SysDataDictionaryItem;
|
||||
import com.nis.domain.configuration.BaseStringCfg;
|
||||
import com.nis.util.Constants;
|
||||
import com.nis.util.DictUtils;
|
||||
import com.nis.web.service.configuration.DnsResStrategyService;
|
||||
|
||||
public class CheckStringFormatThread implements Callable<String>{
|
||||
private Logger logger=Logger.getLogger(CheckStringFormatThread.class);
|
||||
private BlockingQueue<? extends Object> srcQueue;
|
||||
private BlockingQueue<BaseStringCfg<?>> destQueue;
|
||||
private Properties prop;
|
||||
private DnsResStrategyService dnsResStrategyService;
|
||||
private FunctionServiceDict serviceDict;
|
||||
private FunctionRegionDict regionDict;
|
||||
public CheckStringFormatThread(FunctionServiceDict serviceDict,FunctionRegionDict regionDict,Properties prop,BlockingQueue<? extends Object> srcQueue,BlockingQueue<BaseStringCfg<?>> destQueue) {
|
||||
this.serviceDict=serviceDict;
|
||||
this.regionDict=regionDict;
|
||||
this.srcQueue=srcQueue;
|
||||
this.destQueue=destQueue;
|
||||
this.prop=prop;
|
||||
}
|
||||
@Override
|
||||
public String call() throws Exception {
|
||||
List<Object> dataList=Lists.newArrayList(Constants.MAAT_JSON_SEND_SIZE);
|
||||
String msg=null;
|
||||
while(!srcQueue.isEmpty()) {
|
||||
int size=srcQueue.drainTo(dataList,Constants.MAAT_JSON_SEND_SIZE);
|
||||
if(regionDict.getRegionType().intValue()==2) {
|
||||
try {
|
||||
List<BaseStringCfg<?>> cfgs=this.checkStringCfg(dataList);
|
||||
destQueue.addAll(cfgs);
|
||||
}catch (Exception e) {
|
||||
// TODO: handle exception
|
||||
msg=e.getMessage();
|
||||
break;
|
||||
}
|
||||
}
|
||||
dataList.clear();
|
||||
}
|
||||
return msg;
|
||||
}
|
||||
|
||||
public List<BaseStringCfg<?>> checkStringCfg(List<?> list) throws ServiceException {
|
||||
logger.warn("start to validate stringCfg data...");
|
||||
long start=System.currentTimeMillis();
|
||||
List<BaseStringCfg<?>> stringList = new ArrayList<BaseStringCfg<?>>();
|
||||
String exprTypeP = regionDict.getConfigExprType();
|
||||
if (StringUtils.isBlank(exprTypeP)) {
|
||||
throw new RuntimeException("Found String region,but exprType is Empty");
|
||||
}
|
||||
String matchMethodP = regionDict.getConfigMatchMethod();
|
||||
if (StringUtils.isBlank(matchMethodP)) {
|
||||
throw new RuntimeException("Found String region,but matchMethod is Empty");
|
||||
}
|
||||
String hexP = regionDict.getConfigHex();
|
||||
if (StringUtils.isBlank(hexP)) {
|
||||
throw new RuntimeException("Found String region,but hex is Empty");
|
||||
}
|
||||
String mulityKeywordsP = regionDict.getConfigMultiKeywords();
|
||||
if (StringUtils.isBlank(mulityKeywordsP)) {
|
||||
throw new RuntimeException("Found String region,but mulityKeywords is Empty");
|
||||
}
|
||||
StringBuffer errTip = new StringBuffer();
|
||||
Pattern pattern = Pattern.compile("\t|\r|\n|\b|\f");
|
||||
Pattern domainPattern = Pattern.compile(Constants.DOMAIN_REGEXP);
|
||||
for (int i = 0; i < list.size(); i++) {
|
||||
StringBuffer errInfo = new StringBuffer();
|
||||
BaseStringCfg baseStringCfg = new BaseStringCfg();
|
||||
BeanUtils.copyProperties(list.get(i), baseStringCfg);
|
||||
if (regionDict.getRegionType().equals(2)) {
|
||||
if (regionDict.getFunctionId().equals(510) && "p2p_hash".equals(regionDict.getConfigServiceType())) {
|
||||
String userRegion1 = baseStringCfg.getUserRegion1();
|
||||
if (StringUtils.isNotBlank(userRegion1)) {
|
||||
List<SysDataDictionaryItem> hashs = DictUtils.getDictList("P2P_HASH_TYPE");
|
||||
boolean has = false;
|
||||
for (SysDataDictionaryItem hash : hashs) {
|
||||
if (hash.getItemCode().equals(userRegion1)) {
|
||||
has = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!has) {
|
||||
errInfo.append(String.format(prop.getProperty("is_incorrect"),
|
||||
prop.getProperty("p2p_hash_type") + " ") + ";");
|
||||
}
|
||||
}
|
||||
}
|
||||
String keyword = baseStringCfg.getCfgKeywords();
|
||||
if (!regionDict.getFunctionId().equals(403)) {
|
||||
if (StringUtils.isBlank(keyword)) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("can_not_null"), prop.getProperty("key_word") + " ") + ";");
|
||||
}
|
||||
if (mulityKeywordsP.equals("0")) {
|
||||
if (keyword.indexOf("\n") > -1) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("not_multiple"), prop.getProperty("key_word")) + ";");
|
||||
}
|
||||
Matcher m = pattern.matcher(keyword);
|
||||
if (m.find()) {
|
||||
errInfo.append(String.format(prop.getProperty("has_invisible_char"),
|
||||
prop.getProperty("key_word") + " '" + keyword + "'") + ";");
|
||||
}
|
||||
} else {
|
||||
boolean has = false;
|
||||
Set<String> keywordSet=Sets.newHashSet();
|
||||
for (String key : keyword.split("\n")) {
|
||||
Matcher m = pattern.matcher(key);
|
||||
if (m.find()) {
|
||||
has = true;
|
||||
errInfo.append(String.format(prop.getProperty("has_invisible_char"),
|
||||
prop.getProperty("key_word") + " '" + key + "'") + ";");
|
||||
break;
|
||||
}
|
||||
if(!keywordSet.contains(key)) {
|
||||
keywordSet.add(key);
|
||||
}else {
|
||||
errInfo.append(prop.getProperty("key_word") + " '" + key + "'"+" "+prop.getProperty("repeat") + ";");
|
||||
}
|
||||
}
|
||||
if (!has) {
|
||||
if(keyword.replaceAll("\n","").length()>1024) {
|
||||
errInfo.append(String.format(prop.getProperty("most_keywords"),
|
||||
prop.getProperty("key_word")) + ";");
|
||||
}else {
|
||||
String reWord = keyword.replaceAll("\n", Constants.KEYWORD_EXPR);
|
||||
baseStringCfg.setCfgKeywords(reWord);
|
||||
}
|
||||
}
|
||||
}
|
||||
}else {
|
||||
if (StringUtils.isBlank(keyword)) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("can_not_null"), prop.getProperty("domain_name") + " ") + ";");
|
||||
}else {
|
||||
Matcher m = pattern.matcher(keyword);
|
||||
if (m.find()) {
|
||||
errInfo.append(String.format(prop.getProperty("has_invisible_char"),
|
||||
prop.getProperty("domain_name") + " '" + keyword + "'") + ";");
|
||||
}else {
|
||||
m = domainPattern.matcher(keyword);
|
||||
if(!m.matches()) {
|
||||
errInfo.append(String.format(prop.getProperty("not_valid_domain"),
|
||||
prop.getProperty("domain_name") + " '" + keyword + "'") + ";");
|
||||
}else {
|
||||
baseStringCfg.setDomain(keyword);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Integer exprType = baseStringCfg.getExprType();
|
||||
boolean has = false;
|
||||
if (exprType == null) {
|
||||
if (exprTypeP.indexOf(",") == -1) {
|
||||
if (mulityKeywordsP.equals("0") && exprTypeP.equals("1")) {
|
||||
throw new RuntimeException("region dict config error,dict id is " + regionDict.getDictId());
|
||||
}
|
||||
baseStringCfg.setExprType(Integer.parseInt(exprTypeP));
|
||||
} else if (exprTypeP.indexOf("0") > -1 && mulityKeywordsP.equals("0")) {
|
||||
baseStringCfg.setExprType(0);
|
||||
} else if (exprTypeP.indexOf("1") > -1 && mulityKeywordsP.equals("1")
|
||||
&& keyword.indexOf("\n") > -1) {
|
||||
baseStringCfg.setExprType(1);
|
||||
} else if (exprTypeP.indexOf("0") > -1 && mulityKeywordsP.equals("1")
|
||||
&& keyword.indexOf("\n") == -1) {
|
||||
baseStringCfg.setExprType(0);
|
||||
} else {
|
||||
baseStringCfg.setExprType(Integer.parseInt(exprTypeP.split(",")[0]));
|
||||
}
|
||||
// errInfo.append(String.format(prop.getProperty("can_not_null"),
|
||||
// prop.getProperty("expression_type"))+";");
|
||||
} else {
|
||||
for (String exp : exprTypeP.split(",")) {
|
||||
if (exp.equals(exprType.toString())) {
|
||||
has = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!has) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("expression_type"))
|
||||
+ ";");
|
||||
}
|
||||
has = false;
|
||||
}
|
||||
exprType = baseStringCfg.getExprType();
|
||||
Integer matchMethod = baseStringCfg.getMatchMethod();
|
||||
if (matchMethod == null) {
|
||||
if (matchMethodP.indexOf(",") == -1) {
|
||||
if (exprTypeP.equals("1") && !matchMethodP.equals("0")) {
|
||||
throw new RuntimeException("region dict config error,dict id is " + regionDict.getDictId());
|
||||
}
|
||||
baseStringCfg.setMatchMethod(Integer.parseInt(matchMethodP));
|
||||
} else if (exprType != null && exprType.intValue() == 1) {
|
||||
if (matchMethodP.indexOf("0") > -1) {
|
||||
baseStringCfg.setMatchMethod(0);
|
||||
} else {
|
||||
throw new RuntimeException("region dict config error,dict id is " + regionDict.getDictId());
|
||||
}
|
||||
|
||||
} else {
|
||||
baseStringCfg.setMatchMethod(Integer.parseInt(matchMethodP.split(",")[0]));
|
||||
}
|
||||
// errInfo.append(String.format(prop.getProperty("can_not_null"),
|
||||
// prop.getProperty("match_method"))+";");
|
||||
} else {
|
||||
for (String exp : matchMethodP.split(",")) {
|
||||
if (exp.equals(matchMethod.toString())) {
|
||||
has = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!has) {
|
||||
errInfo.append(String.format(prop.getProperty("is_incorrect"), prop.getProperty("match_method"))
|
||||
+ ";");
|
||||
}
|
||||
}
|
||||
|
||||
Integer isHex = baseStringCfg.getIsHex();
|
||||
Integer isCaseInsenstive = baseStringCfg.getIsCaseInsenstive();
|
||||
if (isHex == null || isCaseInsenstive == null) {
|
||||
if (isHex == null) {
|
||||
if (hexP.equals("0") || hexP.equals("2")) {
|
||||
baseStringCfg.setIsHex(0);
|
||||
} else if (hexP.equals("1")) {
|
||||
baseStringCfg.setIsHex(1);
|
||||
} else {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("can_not_null"), prop.getProperty("is_hex")) + ";");
|
||||
}
|
||||
}
|
||||
if (isCaseInsenstive == null) {
|
||||
if (hexP.equals("0") || hexP.equals("1")) {
|
||||
baseStringCfg.setIsCaseInsenstive(0);
|
||||
} else if (hexP.equals("2")) {
|
||||
baseStringCfg.setIsCaseInsenstive(1);
|
||||
} else {
|
||||
errInfo.append(String.format(prop.getProperty("can_not_null"),
|
||||
prop.getProperty("is_case_insenstive")) + ";");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (isHex.intValue() != 0 && isHex.intValue() != 1) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("is_hex")) + ";");
|
||||
}
|
||||
if (isCaseInsenstive.intValue() != 0 && isCaseInsenstive.intValue() != 1) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("is_case_insenstive"))
|
||||
+ ";");
|
||||
}
|
||||
if (hexP.indexOf("1") == -1 && isHex.intValue() == 1) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("is_hex")) + ";");
|
||||
}
|
||||
if (hexP.equals("1") && isHex.intValue() == 0) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("is_hex")) + ";");
|
||||
}
|
||||
if (hexP.indexOf("2") == -1 && isCaseInsenstive.intValue() == 1) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("is_case_insenstive"))
|
||||
+ ";");
|
||||
}
|
||||
if (hexP.equals("2") && isCaseInsenstive.intValue() == 0) {
|
||||
errInfo.append(
|
||||
String.format(prop.getProperty("is_incorrect"), prop.getProperty("is_case_insenstive"))
|
||||
+ ";");
|
||||
}
|
||||
}
|
||||
isHex = baseStringCfg.getIsHex();
|
||||
isCaseInsenstive = baseStringCfg.getIsCaseInsenstive();
|
||||
if (isHex != null && isCaseInsenstive != null) {
|
||||
if (isHex.intValue() == 0 && isCaseInsenstive.intValue() == 0) {
|
||||
baseStringCfg.setIsHexbin(0);
|
||||
} else if (isHex.intValue() == 1 && isCaseInsenstive.intValue() == 0) {
|
||||
baseStringCfg.setIsHexbin(1);
|
||||
} else if (isHex.intValue() == 0 && isCaseInsenstive.intValue() == 1) {
|
||||
baseStringCfg.setIsHexbin(2);
|
||||
}else {
|
||||
errInfo.append(prop.getProperty("hex_case_insensitive")+ ";");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if (errInfo.toString().length() > 0) {//
|
||||
errTip.append(String.format(prop.getProperty("line"), baseStringCfg.getIndex()) + ",");
|
||||
errTip.append(errInfo);
|
||||
errTip.append("<br>");
|
||||
}
|
||||
stringList.add(baseStringCfg);
|
||||
}
|
||||
if (errTip.toString().length() > 0) {
|
||||
throw new ServiceException(errTip.toString());
|
||||
}
|
||||
long end=System.currentTimeMillis();
|
||||
logger.warn("validate stringCfg data finish,cost:"+(end-start));
|
||||
return stringList;
|
||||
}
|
||||
|
||||
public DnsResStrategyService getDnsResStrategyService() {
|
||||
return dnsResStrategyService;
|
||||
}
|
||||
public void setDnsResStrategyService(DnsResStrategyService dnsResStrategyService) {
|
||||
this.dnsResStrategyService = dnsResStrategyService;
|
||||
}
|
||||
}
|
||||
525
src/main/java/com/nis/util/excel/ImportBigExcel.java
Normal file
525
src/main/java/com/nis/util/excel/ImportBigExcel.java
Normal file
@@ -0,0 +1,525 @@
|
||||
/**
|
||||
* Copyright © 2012-2014 <a href="https://github.com/thinkgem/jeesite">JeeSite</a> All rights reserved.
|
||||
*/
|
||||
package com.nis.util.excel;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Method;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
|
||||
import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
|
||||
import org.apache.poi.ss.usermodel.DateUtil;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.nis.domain.FunctionRegionDict;
|
||||
import com.nis.domain.FunctionServiceDict;
|
||||
import com.nis.util.DictUtils;
|
||||
import com.nis.util.Reflections;
|
||||
|
||||
/**
|
||||
* 导入Excel文件（支持“XLS”和“XLSX”格式）
|
||||
* @author ThinkGem
|
||||
* @version 2013-03-10
|
||||
*/
|
||||
public class ImportBigExcel extends XLSXCovertCSVReader{
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(ImportBigExcel.class);
|
||||
/**
|
||||
* 标题行号
|
||||
*/
|
||||
private Integer headerNum;
|
||||
private int sheetIndex;
|
||||
private Properties props;
|
||||
private File uploadFile;
|
||||
private FunctionRegionDict regionDict;
|
||||
private FunctionServiceDict serviceDict;
|
||||
//ip相关
|
||||
private boolean srcIpShow=false;
|
||||
private boolean destIpShow=false;
|
||||
private boolean srcPortShow=false;
|
||||
private boolean destPortShow=false;
|
||||
private boolean directionShow=false;
|
||||
private boolean protocolShow=false;
|
||||
//字符串公共属性
|
||||
private boolean matchMethodShow=false;
|
||||
private boolean hexShow=false;
|
||||
private boolean isCaseSensitiveShow=false;
|
||||
//增强字符串相关
|
||||
private boolean districtShow=false;
|
||||
private List<Object[]> annotationList;
|
||||
private List<List<Object>> dataList=Lists.newArrayList();
|
||||
/**
|
||||
* 构造函数
|
||||
* @param file 导入文件对象
|
||||
* @param headerNum 标题行号，数据行号=标题行号+1
|
||||
* @param sheetIndex 工作表编号
|
||||
* @throws InvalidFormatException
|
||||
* @throws IOException
|
||||
*/
|
||||
public ImportBigExcel(MultipartFile multipartFile, Integer headerNum, int sheetIndex)
|
||||
throws InvalidFormatException, IOException {
|
||||
File upFile=new File(System.currentTimeMillis()+multipartFile.getOriginalFilename());
|
||||
upFile.mkdirs();
|
||||
multipartFile.transferTo(upFile);
|
||||
if (!upFile.exists()){
|
||||
throw new RuntimeException("导入文档为空!");
|
||||
}else if(upFile.getName().toLowerCase().endsWith("xls")){
|
||||
throw new RuntimeException("xls not supported");
|
||||
}else if(upFile.getName().toLowerCase().endsWith("xlsx")){
|
||||
}else{
|
||||
throw new RuntimeException("文档格式不正确!");
|
||||
}
|
||||
this.sheetIndex = sheetIndex;
|
||||
this.headerNum = headerNum;
|
||||
this.uploadFile=upFile;
|
||||
log.debug("Initialize success.");
|
||||
}
|
||||
/**
|
||||
* //递归获取cls实体对象及父级对象的属性
|
||||
* wx:修改，子类覆盖父类的同名方法
|
||||
* @param list
|
||||
* @param cls
|
||||
*/
|
||||
public void getFields(List<Field> list,Class<?> cls) {
|
||||
Field[] fields=cls.getDeclaredFields();
|
||||
if(fields != null && fields.length > 0){
|
||||
List<Field> tempList=new ArrayList<>();
|
||||
for (Field field : fields) {
|
||||
if(list.size()==0) {
|
||||
tempList.add(field);
|
||||
}else {
|
||||
boolean has=false;
|
||||
for(Field checkF:list) {
|
||||
if(checkF.getName().equals(field.getName())) {
|
||||
has=true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(!has) {
|
||||
tempList.add(field);
|
||||
}
|
||||
}
|
||||
}
|
||||
list.addAll(tempList);
|
||||
}
|
||||
if(cls.getSuperclass() != null){
|
||||
getFields(list,cls.getSuperclass());
|
||||
}
|
||||
}
|
||||
/**
|
||||
* //递归获取cls实体对象及父级对象的method
|
||||
* @param list
|
||||
* @param cls
|
||||
*/
|
||||
public void getMethods(List<Method> list,Class<?> cls) {
|
||||
Method[] methods=cls.getDeclaredMethods();
|
||||
if(methods != null && methods.length > 0){
|
||||
List<Method> tempList=new ArrayList<>();
|
||||
for (Method method : methods) {
|
||||
if(list.size()==0) {
|
||||
tempList.add(method);
|
||||
}else {
|
||||
boolean has=false;
|
||||
for(Method checkM:list) {
|
||||
if(checkM.getName().equals(method.getName())) {
|
||||
has=true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(!has) {
|
||||
tempList.add(method);
|
||||
}
|
||||
}
|
||||
}
|
||||
list.addAll(tempList);
|
||||
}
|
||||
if(cls.getSuperclass() != null){
|
||||
getMethods(list,cls.getSuperclass());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载region,service,注解
|
||||
* @param cls
|
||||
* @param props
|
||||
* @param regionDict
|
||||
* @param serviceDict
|
||||
* @param groups
|
||||
*/
|
||||
public void loadInitParams(Class cls,Properties props,FunctionRegionDict regionDict,FunctionServiceDict serviceDict, int... groups) {
|
||||
this.props=props;
|
||||
this.regionDict=regionDict;
|
||||
this.serviceDict=serviceDict;
|
||||
//从region中获取配置信息
|
||||
Integer regionType=regionDict.getRegionType();
|
||||
|
||||
if(regionType==1) {
|
||||
String ipPortShow=regionDict.getConfigIpPortShow();
|
||||
if(StringUtils.isNotBlank(ipPortShow)) {
|
||||
if(ipPortShow.indexOf("1")>-1) {
|
||||
srcIpShow=true;
|
||||
}
|
||||
if(ipPortShow.indexOf("2")>-1) {
|
||||
srcPortShow=true;
|
||||
}
|
||||
if(ipPortShow.indexOf("3")>-1) {
|
||||
destIpShow=true;
|
||||
}
|
||||
if(ipPortShow.indexOf("4")>-1) {
|
||||
destPortShow=true;
|
||||
}
|
||||
};
|
||||
//协议方向等，如果只有一个值，就 不需要输入了
|
||||
String direction=regionDict.getConfigDirection();
|
||||
if(StringUtils.isNotBlank(direction)&&direction.indexOf(",")>-1) {
|
||||
directionShow=true;
|
||||
}
|
||||
String protocol=regionDict.getConfigProtocol();
|
||||
if(StringUtils.isNotBlank(protocol)&&protocol.indexOf(",")>-1) {
|
||||
protocolShow=true;
|
||||
}
|
||||
//packet ip reject
|
||||
if(regionDict.getFunctionId().equals(5)&&serviceDict!=null&&serviceDict.getServiceId().equals(16)) {
|
||||
protocolShow=false;
|
||||
}
|
||||
}else if(regionType==2||regionType==3){
|
||||
String matchMethod= regionDict.getConfigMatchMethod();
|
||||
if(StringUtils.isNotBlank(matchMethod)&&matchMethod.indexOf(",")>-1) {
|
||||
matchMethodShow=true;
|
||||
}
|
||||
String hex= regionDict.getConfigHex();
|
||||
if(StringUtils.isNotBlank(hex)&&hex.indexOf(",")>-1) {
|
||||
hexShow=true;
|
||||
isCaseSensitiveShow=true;
|
||||
}
|
||||
String district=regionDict.getConfigDistrict();
|
||||
if(StringUtils.isNotBlank(district)&&district.indexOf(",")>-1) {
|
||||
districtShow=true;
|
||||
}
|
||||
}
|
||||
List<Object[]> annotationList = Lists.newArrayList();
|
||||
List<Field> fs=new ArrayList<Field>();
|
||||
// Get annotation field
|
||||
//递归获取cls实体对象及父级对象的属性
|
||||
getFields(fs, cls);
|
||||
for (Field f : fs){
|
||||
//排除不需要导出的字段
|
||||
if(!srcIpShow) {
|
||||
if(f.getName().equalsIgnoreCase("srcIpAddress")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!destIpShow) {
|
||||
if(f.getName().equalsIgnoreCase("destIpAddress")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!srcPortShow) {
|
||||
if(f.getName().equalsIgnoreCase("srcPort")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!destPortShow) {
|
||||
if(f.getName().equalsIgnoreCase("destPort")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!directionShow) {
|
||||
if(f.getName().equalsIgnoreCase("direction")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!protocolShow) {
|
||||
if(f.getName().equalsIgnoreCase("protocol")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!matchMethodShow) {
|
||||
if(f.getName().equalsIgnoreCase("matchMethod")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!hexShow) {
|
||||
if(f.getName().equalsIgnoreCase("isHex")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!isCaseSensitiveShow) {
|
||||
if(f.getName().equalsIgnoreCase("isCaseInsenstive")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!districtShow) {
|
||||
if(f.getName().equalsIgnoreCase("district")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
ExcelField ef = f.getAnnotation(ExcelField.class);
|
||||
if (ef != null && (ef.type()==0 || ef.type()==2)){
|
||||
if (groups!=null && groups.length>0){
|
||||
boolean inGroup = false;
|
||||
for (int g : groups){
|
||||
if (inGroup){
|
||||
break;
|
||||
}
|
||||
for (int efg : ef.groups()){
|
||||
if (g == efg){
|
||||
inGroup = true;
|
||||
annotationList.add(new Object[]{ef, f});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
annotationList.add(new Object[]{ef, f});
|
||||
}
|
||||
}
|
||||
}
|
||||
// Get annotation method
|
||||
// Method[] ms = cls.getDeclaredMethods();
|
||||
List<Method> ms=new ArrayList<Method>();
|
||||
// Get annotation method
|
||||
//递归获取cls实体对象及父级对象的属性
|
||||
getMethods(ms, cls);
|
||||
for (Method m : ms){
|
||||
//排除不需要导出的字段
|
||||
if(!srcIpShow) {
|
||||
if(m.getName().equalsIgnoreCase("get"+"srcIpAddress")||m.getName().equalsIgnoreCase("set"+"srcIpAddress")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!destIpShow) {
|
||||
if(m.getName().equalsIgnoreCase("get"+"destIpAddress")||m.getName().equalsIgnoreCase("set"+"destIpAddress")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!srcPortShow) {
|
||||
if(m.getName().equalsIgnoreCase("get"+"srcPort")||m.getName().equalsIgnoreCase("set"+"srcPort")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!destPortShow) {
|
||||
if(m.getName().equalsIgnoreCase("get"+"destPort")||m.getName().equalsIgnoreCase("set"+"destPort")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!directionShow) {
|
||||
if(m.getName().equalsIgnoreCase("get"+"direction")||m.getName().equalsIgnoreCase("set"+"direction")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!protocolShow) {
|
||||
if(m.getName().equalsIgnoreCase("get"+"protocol")||m.getName().equalsIgnoreCase("set"+"protocol")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!matchMethodShow) {
|
||||
if(m.getName().equalsIgnoreCase("get"+"matchMethod")||m.getName().equalsIgnoreCase("set"+"matchMethod")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!hexShow) {
|
||||
if(m.getName().equalsIgnoreCase("get"+"isHex")||m.getName().equalsIgnoreCase("set"+"isHex")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!isCaseSensitiveShow) {
|
||||
if(m.getName().equalsIgnoreCase("get"+"isCaseInsenstive")||m.getName().equalsIgnoreCase("set"+"isCaseInsenstive")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(!districtShow) {
|
||||
if(m.getName().equalsIgnoreCase("get"+"district")||m.getName().equalsIgnoreCase("set"+"district")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
ExcelField ef = m.getAnnotation(ExcelField.class);
|
||||
if (ef != null && (ef.type()==0 || ef.type()==2)){
|
||||
if (groups!=null && groups.length>0){
|
||||
boolean inGroup = false;
|
||||
for (int g : groups){
|
||||
if (inGroup){
|
||||
break;
|
||||
}
|
||||
for (int efg : ef.groups()){
|
||||
if (g == efg){
|
||||
inGroup = true;
|
||||
annotationList.add(new Object[]{ef, m});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
annotationList.add(new Object[]{ef, m});
|
||||
}
|
||||
}
|
||||
}
|
||||
// Field sorting
|
||||
Collections.sort(annotationList, new Comparator<Object[]>() {
|
||||
public int compare(Object[] o1, Object[] o2) {
|
||||
return new Integer(((ExcelField)o1[0]).sort()).compareTo(
|
||||
new Integer(((ExcelField)o2[0]).sort()));
|
||||
};
|
||||
});
|
||||
this.annotationList=annotationList;
|
||||
}
|
||||
@Override
|
||||
public List<Object> optRows(int sheetIndex, int curRow, List<Object> rowlist) {
|
||||
// TODO Auto-generated method stub
|
||||
List<Object> rowlistCopy=Lists.newArrayList();
|
||||
rowlistCopy.addAll(rowlist);
|
||||
dataList.add(rowlistCopy);
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* 获取导入数据列表
|
||||
* @param cls 导入对象类型
|
||||
* @param groups 导入分组
|
||||
* @throws SQLException
|
||||
* @throws SAXException
|
||||
* @throws ParserConfigurationException
|
||||
* @throws OpenXML4JException
|
||||
* @throws IOException
|
||||
*/
|
||||
public <E> BlockingQueue<E> getDataList(Class<E> cls) throws InstantiationException, IllegalAccessException, IOException, OpenXML4JException, ParserConfigurationException, SAXException, SQLException{
|
||||
log.warn("start to load data...");
|
||||
this.processOneSheet(uploadFile, this.sheetIndex);
|
||||
long start=System.currentTimeMillis();
|
||||
if(regionDict==null) {
|
||||
throw new RuntimeException("regionDict is null!");
|
||||
}
|
||||
// Get excel data
|
||||
BlockingQueue<E> _dataList =new ArrayBlockingQueue(dataList.size(),true);
|
||||
for (int i = 0; i < dataList.size(); i++) {
|
||||
E e = (E)cls.newInstance();
|
||||
if(i<=headerNum) {
|
||||
continue;
|
||||
}
|
||||
int column = 0;
|
||||
List<Object> row=dataList.get(i);
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (Object[] os : annotationList){
|
||||
Object val=row.get(column);
|
||||
column++;
|
||||
if (val != null){
|
||||
ExcelField ef = (ExcelField)os[0];
|
||||
// If is dict type, get dict value
|
||||
if (StringUtils.isNotBlank(ef.dictType())){
|
||||
Object val1 = DictUtils.getDictCode(ef.dictType(), val.toString(), "");
|
||||
//没有获取到字典值的话会影响验证判断
|
||||
if(val1!=null&&StringUtils.isNotBlank(val1.toString())) {
|
||||
val=val1;
|
||||
}
|
||||
//log.debug("Dictionary type value: ["+i+","+colunm+"] " + val);
|
||||
}
|
||||
// Get param type and type cast
|
||||
Class<?> valType = Class.class;
|
||||
if (os[1] instanceof Field){
|
||||
valType = ((Field)os[1]).getType();
|
||||
}else if (os[1] instanceof Method){
|
||||
Method method = ((Method)os[1]);
|
||||
if ("get".equals(method.getName().substring(0, 3))){
|
||||
valType = method.getReturnType();
|
||||
}else if("set".equals(method.getName().substring(0, 3))){
|
||||
valType = ((Method)os[1]).getParameterTypes()[0];
|
||||
}
|
||||
}
|
||||
//log.debug("Import value type: ["+i+","+column+"] " + valType);
|
||||
try {
|
||||
if (valType == String.class){
|
||||
String s = String.valueOf(val.toString());
|
||||
//0.0.0.0表示任意IP的含义
|
||||
if(StringUtils.endsWith(s, ".0") && !s.endsWith("0.0.0.0")){
|
||||
val = StringUtils.substringBefore(s, ".0");
|
||||
}else{
|
||||
val = String.valueOf(val.toString());
|
||||
}
|
||||
}else if (valType == Integer.class){
|
||||
val = Double.valueOf(val.toString()).intValue();
|
||||
}else if (valType == Long.class){
|
||||
val = Double.valueOf(val.toString()).longValue();
|
||||
}else if (valType == Double.class){
|
||||
val = Double.valueOf(val.toString());
|
||||
}else if (valType == Float.class){
|
||||
val = Float.valueOf(val.toString());
|
||||
}else if (valType == Date.class){
|
||||
val = DateUtil.getJavaDate((Double)val);
|
||||
}else{
|
||||
if (ef.fieldType() != Class.class){
|
||||
val = ef.fieldType().getMethod("getValue", String.class).invoke(null, val.toString());
|
||||
}else{
|
||||
val = Class.forName(this.getClass().getName().replaceAll(this.getClass().getSimpleName(),
|
||||
"fieldtype."+valType.getSimpleName()+"Type")).getMethod("getValue", String.class).invoke(null, val.toString());
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.info("Get cell value ["+i+","+column+"] error: " + ex.toString());
|
||||
val = null;
|
||||
}
|
||||
// set entity value
|
||||
if (os[1] instanceof Field){
|
||||
Reflections.invokeSetter(e, ((Field)os[1]).getName(), val);
|
||||
}else if (os[1] instanceof Method){
|
||||
String mthodName = ((Method)os[1]).getName();
|
||||
if ("get".equals(mthodName.substring(0, 3))){
|
||||
mthodName = "set"+StringUtils.substringAfter(mthodName, "get");
|
||||
}
|
||||
Reflections.invokeMethod(e, mthodName, new Class[] {valType}, new Object[] {val});
|
||||
}
|
||||
//设置索引值
|
||||
Reflections.invokeSetter(e, "index", (i+1));
|
||||
}
|
||||
sb.append(val+", ");
|
||||
}
|
||||
_dataList.offer(e);
|
||||
row.clear();
|
||||
log.debug("Read success: ["+i+"] "+sb.toString());
|
||||
}
|
||||
long end=System.currentTimeMillis();
|
||||
log.warn(" load data finish,cost:"+(end-start));
|
||||
return _dataList;
|
||||
}
|
||||
public File getUploadFile() {
|
||||
return uploadFile;
|
||||
}
|
||||
public void setUploadFile(File uploadFile) {
|
||||
this.uploadFile = uploadFile;
|
||||
}
|
||||
|
||||
// /**
|
||||
// * 导入测试
|
||||
// */
|
||||
// public static void main(String[] args) throws Throwable {
|
||||
//
|
||||
// ImportExcel ei = new ImportExcel("target/export.xlsx", 1);
|
||||
//
|
||||
// for (int i = ei.getDataRowNum(); i < ei.getLastDataRowNum(); i++) {
|
||||
// Row row = ei.getRow(i);
|
||||
// for (int j = 0; j < ei.getLastCellNum(); j++) {
|
||||
// Object val = ei.getCellValue(row, j);
|
||||
// System.out.print(val+", ");
|
||||
// }
|
||||
// System.out.print("\n");
|
||||
// }
|
||||
//
|
||||
// }
|
||||
|
||||
}
|
||||
593
src/main/java/com/nis/util/excel/XLSXCovertCSVReader.java
Normal file
593
src/main/java/com/nis/util/excel/XLSXCovertCSVReader.java
Normal file
@@ -0,0 +1,593 @@
|
||||
package com.nis.util.excel;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.SQLException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.parsers.SAXParser;
|
||||
import javax.xml.parsers.SAXParserFactory;
|
||||
|
||||
import org.apache.poi.hssf.usermodel.HSSFDataFormatter;
|
||||
import org.apache.poi.hssf.usermodel.HSSFDateUtil;
|
||||
import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
|
||||
import org.apache.poi.openxml4j.opc.OPCPackage;
|
||||
import org.apache.poi.openxml4j.opc.PackageAccess;
|
||||
import org.apache.poi.ss.usermodel.BuiltinFormats;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;
|
||||
import org.apache.poi.xssf.eventusermodel.XSSFReader;
|
||||
import org.apache.poi.xssf.model.StylesTable;
|
||||
import org.apache.poi.xssf.usermodel.XSSFCellStyle;
|
||||
import org.apache.poi.xssf.usermodel.XSSFRichTextString;
|
||||
import org.xml.sax.Attributes;
|
||||
import org.xml.sax.InputSource;
|
||||
import org.xml.sax.SAXException;
|
||||
import org.xml.sax.XMLReader;
|
||||
import org.xml.sax.helpers.DefaultHandler;
|
||||
|
||||
|
||||
/**
|
||||
* 使用CVS模式解决XLSX文件，可以有效解决用户模式内存溢出的问题
|
||||
* 该模式是POI官方推荐的读取大数据的模式，在用户模式下，数据量较大、Sheet较多、或者是有很多无用的空行的情况
|
||||
* ，容易出现内存溢出,用户模式读取Excel的典型代码如下： FileInputStream file=new
|
||||
* FileInputStream("c:\\test.xlsx"); Workbook wb=new XSSFWorkbook(file);
|
||||
*
|
||||
*
|
||||
* @author 山人
|
||||
*/
|
||||
public abstract class XLSXCovertCSVReader {
|
||||
|
||||
/**
|
||||
* The type of the data value is indicated by an attribute on the cell. The
|
||||
* value is usually in a "v" element within the cell.
|
||||
*/
|
||||
enum xssfDataType {
|
||||
BOOL, ERROR, FORMULA, INLINESTR, SSTINDEX, NUMBER,
|
||||
}
|
||||
/**
|
||||
* 使用xssf_sax_API处理Excel,请参考： http://poi.apache.org/spreadsheet/how-to.html#xssf_sax_api
|
||||
* <p/>
|
||||
* Also see Standard ECMA-376, 1st edition, part 4, pages 1928ff, at
|
||||
* http://www.ecma-international.org/publications/standards/Ecma-376.htm
|
||||
* <p/>
|
||||
* A web-friendly version is http://openiso.org/Ecma/376/Part4
|
||||
*/
|
||||
//定义前一个元素和当前元素的位置，用来计算其中空的单元格数量，如A6和A8等
|
||||
private String preRefnum = null, refnum = null;
|
||||
//定义该文档一行最大的单元格数，用来补全一行最后可能缺失的单元格
|
||||
private String maxRefnum = null;
|
||||
private String lastContents;
|
||||
private int curCol = 0;
|
||||
// private Integer type;
|
||||
private List<IndexValue> rowData= new ArrayList<IndexValue>();
|
||||
//excel记录行操作方法，以sheet索引，行索引和行元素列表为参数，对sheet的一行元素进行操作，元素为String类型
|
||||
public abstract List<Object> optRows(int sheetIndex,int curRow, List<Object> rowlist) ;
|
||||
class MyXSSFSheetHandler extends DefaultHandler {
|
||||
|
||||
/**
|
||||
* Table with styles
|
||||
*/
|
||||
private StylesTable stylesTable;
|
||||
|
||||
/**
|
||||
* Table with unique strings
|
||||
*/
|
||||
private ReadOnlySharedStringsTable sharedStringsTable;
|
||||
|
||||
|
||||
/**
|
||||
* Number of columns to read starting with leftmost
|
||||
*/
|
||||
|
||||
// Set when V start element is seen
|
||||
private boolean vIsOpen;
|
||||
|
||||
// Set when cell start element is seen;
|
||||
// used when cell close element is seen.
|
||||
private xssfDataType nextDataType;
|
||||
private int sheetIndex = -1;
|
||||
// Used to format numeric cell values.
|
||||
private short formatIndex;
|
||||
private String formatString;
|
||||
private final DataFormatter formatter;
|
||||
|
||||
private int thisColumn = -1;
|
||||
// The last column printed to the output stream
|
||||
private int lastColumnNumber = -1;
|
||||
private List<Object> rowlist = new ArrayList<Object>();
|
||||
// Gathers characters as they are seen.
|
||||
private StringBuffer value;
|
||||
// private String[] record;
|
||||
// private List<String[]> rows = new ArrayList<String[]>();
|
||||
private boolean isCellNull = false;
|
||||
|
||||
/**
|
||||
* Accepts objects needed while parsing.
|
||||
*
|
||||
* @param styles
|
||||
* Table of styles
|
||||
* @param strings
|
||||
* Table of shared strings
|
||||
* @param cols
|
||||
* Minimum number of columns to show
|
||||
* @param target
|
||||
* Sink for output
|
||||
*/
|
||||
public MyXSSFSheetHandler(StylesTable styles,
|
||||
ReadOnlySharedStringsTable strings) {
|
||||
this.stylesTable = styles;
|
||||
this.sharedStringsTable = strings;
|
||||
this.value = new StringBuffer();
|
||||
this.nextDataType = xssfDataType.NUMBER;
|
||||
this.formatter = new DataFormatter();
|
||||
rowlist.clear();// 每次读取都清空行集合
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
*
|
||||
* @see
|
||||
* org.xml.sax.helpers.DefaultHandler#startElement(java.lang.String,
|
||||
* java.lang.String, java.lang.String, org.xml.sax.Attributes)
|
||||
*/
|
||||
public void startElement(String uri, String localName, String name,
|
||||
Attributes attributes) throws SAXException {
|
||||
|
||||
if ("inlineStr".equals(name) || "v".equals(name)) {
|
||||
vIsOpen = true;
|
||||
// Clear contents cache
|
||||
value.setLength(0);
|
||||
}
|
||||
// c => cell
|
||||
else if ("c".equals(name)) {
|
||||
// Get the cell reference
|
||||
String r = attributes.getValue("r");
|
||||
int firstDigit = -1;
|
||||
for (int c = 0; c < r.length(); ++c) {
|
||||
if (Character.isDigit(r.charAt(c))) {
|
||||
firstDigit = c;
|
||||
break;
|
||||
}
|
||||
}
|
||||
thisColumn = nameToColumn(r.substring(0, firstDigit));
|
||||
String cellType = attributes.getValue("t");
|
||||
|
||||
if(preRefnum == null){
|
||||
preRefnum = attributes.getValue("r");
|
||||
}
|
||||
// else{
|
||||
// preRefnum = refnum;
|
||||
// }
|
||||
//当前单元格的位置
|
||||
refnum = attributes.getValue("r");
|
||||
|
||||
|
||||
// Figure out if the value is an index in the SST
|
||||
|
||||
// Set up defaults.
|
||||
this.nextDataType = xssfDataType.NUMBER;
|
||||
this.formatIndex = -1;
|
||||
this.formatString = null;
|
||||
|
||||
String cellStyleStr = attributes.getValue("s");
|
||||
if ("b".equals(cellType))
|
||||
nextDataType = xssfDataType.BOOL;
|
||||
else if ("e".equals(cellType))
|
||||
nextDataType = xssfDataType.ERROR;
|
||||
else if ("inlineStr".equals(cellType))
|
||||
nextDataType = xssfDataType.INLINESTR;
|
||||
else if ("s".equals(cellType))
|
||||
nextDataType = xssfDataType.SSTINDEX;
|
||||
else if ("str".equals(cellType))
|
||||
nextDataType = xssfDataType.FORMULA;
|
||||
else if (cellStyleStr != null) {
|
||||
// It's a number, but almost certainly one
|
||||
// with a special style or format
|
||||
int styleIndex = Integer.parseInt(cellStyleStr);
|
||||
XSSFCellStyle style = stylesTable.getStyleAt(styleIndex);
|
||||
this.formatIndex = style.getDataFormat();
|
||||
this.formatString = style.getDataFormatString();
|
||||
if (this.formatString == null)
|
||||
this.formatString = BuiltinFormats
|
||||
.getBuiltinFormat(this.formatIndex);
|
||||
}
|
||||
}
|
||||
lastContents = "";
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
*
|
||||
* @see org.xml.sax.helpers.DefaultHandler#endElement(java.lang.String,
|
||||
* java.lang.String, java.lang.String)
|
||||
*/
|
||||
public void endElement(String uri, String localName, String name)
|
||||
throws SAXException {
|
||||
|
||||
Object thisVal = null;
|
||||
|
||||
// v => contents of a cell
|
||||
if ("v".equals(name)) {
|
||||
// Process the value contents as required.
|
||||
// Do now, as characters() may be called more than once
|
||||
switch (nextDataType) {
|
||||
|
||||
case BOOL:
|
||||
char first = value.charAt(0);
|
||||
thisVal = first == '0' ? false : true;
|
||||
break;
|
||||
|
||||
case ERROR:
|
||||
thisVal = "\"ERROR:" + value.toString() + '"';
|
||||
break;
|
||||
|
||||
case FORMULA:
|
||||
// A formula could result in a string value,
|
||||
// so always add double-quote characters.
|
||||
thisVal = value.toString() ;
|
||||
break;
|
||||
|
||||
case INLINESTR:
|
||||
// TODO: have seen an example of this, so it's untested.
|
||||
XSSFRichTextString rtsi = new XSSFRichTextString(
|
||||
value.toString());
|
||||
thisVal = rtsi.toString() ;
|
||||
break;
|
||||
|
||||
case SSTINDEX:
|
||||
String sstIndex = value.toString();
|
||||
try {
|
||||
int idx = Integer.parseInt(sstIndex);
|
||||
XSSFRichTextString rtss = new XSSFRichTextString(
|
||||
sharedStringsTable.getEntryAt(idx));
|
||||
thisVal = rtss.toString();
|
||||
} catch (NumberFormatException ex) {
|
||||
System.out.println("Failed to parse SST index '" + sstIndex
|
||||
+ "': " + ex.toString());
|
||||
}
|
||||
break;
|
||||
|
||||
case NUMBER:
|
||||
String n = value.toString();
|
||||
// 判断是否是日期格式
|
||||
if (HSSFDateUtil.isADateFormat(this.formatIndex, n)) {
|
||||
Double d = Double.parseDouble(n);
|
||||
Date date=HSSFDateUtil.getJavaDate(d);
|
||||
thisVal=date;
|
||||
} else if (this.formatString != null)
|
||||
thisVal = formatter.formatRawCellContents(
|
||||
Double.parseDouble(n), this.formatIndex,
|
||||
this.formatString);
|
||||
else if(n.indexOf("E")!=-1) {//处理科学计数法
|
||||
BigDecimal bd = new BigDecimal(n);
|
||||
thisVal = bd.toPlainString();
|
||||
}else
|
||||
thisVal = n;
|
||||
break;
|
||||
|
||||
default:
|
||||
thisVal = "(TODO: Unexpected type: " + nextDataType + ")";
|
||||
break;
|
||||
}
|
||||
|
||||
// Output after we've seen the string contents
|
||||
// Emit commas for any fields that were missing on this row
|
||||
if (lastColumnNumber == -1) {
|
||||
lastColumnNumber = 0;
|
||||
}
|
||||
|
||||
if(!refnum.equals(preRefnum)){
|
||||
int len = countNullCell(refnum, preRefnum);
|
||||
for(int i=0;i<len;i++){
|
||||
rowlist.add(curCol, "");
|
||||
curCol++;
|
||||
}
|
||||
}
|
||||
//判断单元格的值是否为空
|
||||
if (thisVal == null || "".equals(isCellNull)) {
|
||||
// isCellNull = true;// 设置单元格是否为空值
|
||||
}else{
|
||||
preRefnum = refnum;
|
||||
}
|
||||
// System.out.println("refnum="+refnum+"preRefnum="+preRefnum+"curCol="+curCol);
|
||||
rowlist.add(curCol, thisVal);
|
||||
rowData.add(new IndexValue(refnum,lastContents) );
|
||||
curCol++;
|
||||
// record[thisColumn] = thisStr;
|
||||
// Update column
|
||||
if (thisColumn > -1)
|
||||
lastColumnNumber = thisColumn;
|
||||
|
||||
} else if ("row".equals(name)) {
|
||||
|
||||
// Print out any missing commas if needed
|
||||
if (thisColumn > 0) {
|
||||
sheetIndex++;
|
||||
// Columns are 0 based
|
||||
if (lastColumnNumber == -1) {
|
||||
lastColumnNumber = 0;
|
||||
}
|
||||
//默认第一行为表头，以该行单元格数目为最大数目
|
||||
if(sheetIndex == 0){
|
||||
maxRefnum = refnum;
|
||||
}
|
||||
//补全一行尾部可能缺失的单元格
|
||||
if(maxRefnum != null){
|
||||
int len = countNullCell(maxRefnum, refnum);
|
||||
for(int i=0;i<=len;i++){
|
||||
rowlist.add(curCol, "");
|
||||
curCol++;
|
||||
}
|
||||
}
|
||||
if ( rowlist.get(0) != null
|
||||
&& rowlist.get(1) != null)// 判断是否空行
|
||||
{
|
||||
optRows(sheetIndex,lastColumnNumber,rowlist);
|
||||
|
||||
rowlist.clear();
|
||||
// rows.add(record.clone());
|
||||
isCellNull = false;
|
||||
// for (int i = 0; i < record.length; i++) {
|
||||
// record[i] = null;
|
||||
// }
|
||||
}
|
||||
}
|
||||
rowlist.clear();
|
||||
rowData.clear();
|
||||
curCol = 0;
|
||||
preRefnum = null;
|
||||
refnum = null;
|
||||
lastColumnNumber = -1;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// public List<String[]> getRows() {
|
||||
// return rows;
|
||||
// }
|
||||
//
|
||||
// public void setRows(List<String[]> rows) {
|
||||
// this.rows = rows;
|
||||
// }
|
||||
|
||||
/**
|
||||
* Captures characters only if a suitable element is open. Originally
|
||||
* was just "v"; extended for inlineStr also.
|
||||
*/
|
||||
public void characters(char[] ch, int start, int length)
|
||||
throws SAXException {
|
||||
if (vIsOpen)
|
||||
value.append(ch, start, length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts an Excel column name like "C" to a zero-based index.
|
||||
*
|
||||
* @param name
|
||||
* @return Index corresponding to the specified name
|
||||
*/
|
||||
private int nameToColumn(String name) {
|
||||
int column = -1;
|
||||
for (int i = 0; i < name.length(); ++i) {
|
||||
int c = name.charAt(i);
|
||||
column = (column + 1) * 26 + c - 'A';
|
||||
}
|
||||
return column;
|
||||
}
|
||||
|
||||
private String formateDateToString(Date date) {
|
||||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");//格式化日期
|
||||
return sdf.format(date);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// /////////////////////////////////////
|
||||
|
||||
private String path;
|
||||
|
||||
/**
|
||||
* Creates a new XLSX -> CSV converter
|
||||
*
|
||||
* @param pkg
|
||||
* The XLSX package to process
|
||||
* @param output
|
||||
* The PrintStream to output the CSV to
|
||||
* @param minColumns
|
||||
* The minimum number of columns to output, or -1 for no minimum
|
||||
*/
|
||||
public XLSXCovertCSVReader(
|
||||
String path) {
|
||||
this.path = path;
|
||||
}
|
||||
public XLSXCovertCSVReader(
|
||||
) {
|
||||
}
|
||||
/**
|
||||
* Parses and shows the content of one sheet using the specified styles and
|
||||
* shared-strings tables.
|
||||
*
|
||||
* @param styles
|
||||
* @param strings
|
||||
* @param sheetInputStream
|
||||
*/
|
||||
public void processSheet(StylesTable styles,
|
||||
ReadOnlySharedStringsTable strings, InputStream sheetInputStream)
|
||||
throws IOException, ParserConfigurationException, SAXException {
|
||||
|
||||
InputSource sheetSource = new InputSource(sheetInputStream);
|
||||
SAXParserFactory saxFactory = SAXParserFactory.newInstance();
|
||||
SAXParser saxParser = saxFactory.newSAXParser();
|
||||
XMLReader sheetParser = saxParser.getXMLReader();
|
||||
MyXSSFSheetHandler handler = new MyXSSFSheetHandler(styles, strings);
|
||||
sheetParser.setContentHandler(handler);
|
||||
sheetParser.parse(sheetSource);
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化这个处理程序 将
|
||||
*
|
||||
* @throws IOException
|
||||
* @throws OpenXML4JException
|
||||
* @throws ParserConfigurationException
|
||||
* @throws SAXException
|
||||
*/
|
||||
public void processOneSheet(String path,int sheetId) throws IOException, OpenXML4JException,
|
||||
ParserConfigurationException, SAXException,SQLException {
|
||||
OPCPackage xlsxPackage = OPCPackage.open(path, PackageAccess.READ);
|
||||
ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(
|
||||
xlsxPackage);
|
||||
XSSFReader xssfReader = new XSSFReader(xlsxPackage);
|
||||
List<String[]> list = null;
|
||||
StylesTable styles = xssfReader.getStylesTable();
|
||||
InputStream stream=xssfReader.getSheet("rId"+sheetId);
|
||||
processSheet(styles, strings,stream );
|
||||
stream.close();
|
||||
xlsxPackage.close();
|
||||
}
|
||||
public void processOneSheet(File file,int sheetId) throws IOException, OpenXML4JException,
|
||||
ParserConfigurationException, SAXException,SQLException {
|
||||
OPCPackage xlsxPackage = OPCPackage.open(file, PackageAccess.READ);
|
||||
ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(
|
||||
xlsxPackage);
|
||||
XSSFReader xssfReader = new XSSFReader(xlsxPackage);
|
||||
List<String[]> list = null;
|
||||
StylesTable styles = xssfReader.getStylesTable();
|
||||
InputStream stream=xssfReader.getSheet("rId"+sheetId);
|
||||
processSheet(styles, strings,stream );
|
||||
stream.close();
|
||||
xlsxPackage.close();
|
||||
}
|
||||
/**
|
||||
* 读取Excel
|
||||
*
|
||||
* @param path
|
||||
* 文件路径
|
||||
* @param sheetName
|
||||
* sheet名称
|
||||
* @param minColumns
|
||||
* 列总数
|
||||
* @return
|
||||
* @throws SAXException
|
||||
* @throws ParserConfigurationException
|
||||
* @throws OpenXML4JException
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void readerExcel(String path, String sheetName,
|
||||
int minColumns) throws IOException, OpenXML4JException,
|
||||
ParserConfigurationException, SAXException {
|
||||
OPCPackage p = OPCPackage.open(path, PackageAccess.READ);
|
||||
// XLSXCovertCSVReader xlsx2csv = new XLSXCovertCSVReader(p,
|
||||
// sheetName);
|
||||
// xlsx2csv.process(p, sheetName);
|
||||
p.close();
|
||||
}
|
||||
|
||||
public List<String> getMyDataList(List<IndexValue> dataList) {
|
||||
|
||||
List<String> myDataList = new ArrayList<String>();
|
||||
if(dataList==null||dataList.size()<=0) return myDataList;
|
||||
|
||||
for(int i=0;i<dataList.size()-1;i++){
|
||||
IndexValue current = dataList.get(i);
|
||||
myDataList.add(current .v_value);
|
||||
IndexValue next = dataList.get(i+1);
|
||||
int level = next.getLevel(current);
|
||||
for(int k = 0;k<level-1;k++){
|
||||
myDataList.add(null);
|
||||
}
|
||||
if(i==dataList.size()-2){
|
||||
myDataList.add(next .v_value);
|
||||
}
|
||||
|
||||
}
|
||||
return myDataList;
|
||||
}
|
||||
private class IndexValue{
|
||||
String v_index;
|
||||
String v_value;
|
||||
public IndexValue(String v_index, String v_value) {
|
||||
super();
|
||||
this.v_index = v_index;
|
||||
this.v_value = v_value;
|
||||
}
|
||||
@Override
|
||||
public String toString() {
|
||||
return "IndexValue [v_index=" + v_index + ", v_value="
|
||||
+ v_value + "]";
|
||||
}
|
||||
public int getLevel(IndexValue p){
|
||||
|
||||
char[] other = p.v_index.replaceAll("[0-9]", "").toCharArray();
|
||||
char[] self = this.v_index.replaceAll("[0-9]", "").toCharArray();
|
||||
if(other.length!=self.length) return -1;
|
||||
for(int i=0;i<other.length;i++){
|
||||
if(i==other.length-1){
|
||||
return self[i]-other[i];
|
||||
}else{
|
||||
if(self[i]!=other[i]){
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return -1;
|
||||
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 计算两个单元格之间的单元格数目(同一行)
|
||||
* @param ref
|
||||
* @param preRef
|
||||
* @return
|
||||
*/
|
||||
public int countNullCell(String ref, String preRef){
|
||||
//excel2007最大行数是1048576，最大列数是16384，最后一列列名是XFD
|
||||
String xfd = ref.replaceAll("\\d+", "");
|
||||
String xfd_1 = preRef.replaceAll("\\d+", "");
|
||||
|
||||
xfd = fillChar(xfd, 3, '@', true);
|
||||
xfd_1 = fillChar(xfd_1, 3, '@', true);
|
||||
|
||||
char[] letter = xfd.toCharArray();
|
||||
char[] letter_1 = xfd_1.toCharArray();
|
||||
int res = (letter[0]-letter_1[0])*26*26 + (letter[1]-letter_1[1])*26 + (letter[2]-letter_1[2]);
|
||||
return res-1;
|
||||
}
|
||||
/**
|
||||
* 字符串的填充
|
||||
* @param str
|
||||
* @param len
|
||||
* @param let
|
||||
* @param isPre
|
||||
* @return
|
||||
*/
|
||||
String fillChar(String str, int len, char let, boolean isPre){
|
||||
int len_1 = str.length();
|
||||
if(len_1 <len){
|
||||
if(isPre){
|
||||
for(int i=0;i<(len-len_1);i++){
|
||||
str = let+str;
|
||||
}
|
||||
}else{
|
||||
for(int i=0;i<(len-len_1);i++){
|
||||
str = str+let;
|
||||
}
|
||||
}
|
||||
}
|
||||
return str;
|
||||
}
|
||||
public void characters(char[] ch, int start, int length)
|
||||
throws SAXException {
|
||||
//得到单元格内容的值
|
||||
lastContents += new String(ch, start, length);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user