/** * Copyright © 2012-2014 JeeSite All rights reserved. */ package com.nis.util.excel; import java.io.File; import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.List; import java.util.Properties; import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import javax.xml.parsers.ParserConfigurationException; import org.apache.commons.lang3.StringUtils; import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.apache.poi.openxml4j.exceptions.OpenXML4JException; import org.apache.poi.ss.usermodel.DateUtil; import org.jets3t.service.ServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.multipart.MultipartFile; import org.xml.sax.SAXException; import com.google.common.collect.Lists; import com.nis.domain.FunctionRegionDict; import com.nis.domain.FunctionServiceDict; import com.nis.util.Constants; import com.nis.util.DictUtils; import com.nis.util.Reflections; /** * 导入Excel文件(支持“XLS”和“XLSX”格式) * @author ThinkGem * @version 2013-03-10 */ public class ImportBigExcel extends XLSXCovertCSVReader{ private static Logger log = LoggerFactory.getLogger(ImportBigExcel.class); /** * 标题行号 */ private Integer headerNum; private int sheetIndex; private Properties props; private File uploadFile; private FunctionRegionDict regionDict; private FunctionServiceDict serviceDict; //ip相关 private boolean srcIpShow=false; private boolean destIpShow=false; private boolean srcPortShow=false; private boolean destPortShow=false; private boolean directionShow=false; private boolean protocolShow=false; //字符串公共属性 private boolean matchMethodShow=false; private boolean hexShow=false; private boolean isCaseSensitiveShow=false; //增强字符串相关 private boolean districtShow=false; private List annotationList; private List> dataList=Lists.newArrayList(); /** * 构造函数 * @param file 导入文件对象 * @param headerNum 标题行号,数据行号=标题行号+1 * @param sheetIndex 工作表编号 * @throws InvalidFormatException * @throws IOException */ public ImportBigExcel(MultipartFile multipartFile, Integer headerNum, int sheetIndex) throws InvalidFormatException, IOException { File upFile=new File(UUID.randomUUID()+"_"+multipartFile.getOriginalFilename()); upFile.mkdirs(); multipartFile.transferTo(upFile); if (!upFile.exists()){ throw new RuntimeException("导入文档为空!"); }else if(upFile.getName().toLowerCase().endsWith("xls")){ throw new RuntimeException("xls not supported"); }else if(upFile.getName().toLowerCase().endsWith("xlsx")){ }else{ throw new RuntimeException("文档格式不正确!"); } this.sheetIndex = sheetIndex; this.headerNum = headerNum; this.uploadFile=upFile; log.debug("Initialize success."); } /** * //递归获取cls实体对象及父级对象的属性 * wx:修改,子类覆盖父类的同名方法 * @param list * @param cls */ public void getFields(List list,Class cls) { Field[] fields=cls.getDeclaredFields(); if(fields != null && fields.length > 0){ List tempList=new ArrayList<>(); for (Field field : fields) { if(list.size()==0) { tempList.add(field); }else { boolean has=false; for(Field checkF:list) { if(checkF.getName().equals(field.getName())) { has=true; break; } } if(!has) { tempList.add(field); } } } list.addAll(tempList); } if(cls.getSuperclass() != null){ getFields(list,cls.getSuperclass()); } } /** * //递归获取cls实体对象及父级对象的method * @param list * @param cls */ public void getMethods(List list,Class cls) { Method[] methods=cls.getDeclaredMethods(); if(methods != null && methods.length > 0){ List tempList=new ArrayList<>(); for (Method method : methods) { if(list.size()==0) { tempList.add(method); }else { boolean has=false; for(Method checkM:list) { if(checkM.getName().equals(method.getName())) { has=true; break; } } if(!has) { tempList.add(method); } } } list.addAll(tempList); } if(cls.getSuperclass() != null){ getMethods(list,cls.getSuperclass()); } } /** * 加载region,service,注解 * @param cls * @param props * @param regionDict * @param serviceDict * @param groups */ public void loadInitParams(Class cls,Properties props,FunctionRegionDict regionDict,FunctionServiceDict serviceDict, int... groups) { this.props=props; this.regionDict=regionDict; this.serviceDict=serviceDict; //从region中获取配置信息 Integer regionType=regionDict.getRegionType(); if(regionType==1) { String ipPortShow=regionDict.getConfigIpPortShow(); if(StringUtils.isNotBlank(ipPortShow)) { if(ipPortShow.indexOf("1")>-1) { srcIpShow=true; } if(ipPortShow.indexOf("2")>-1) { srcPortShow=true; } if(ipPortShow.indexOf("3")>-1) { destIpShow=true; } if(ipPortShow.indexOf("4")>-1) { destPortShow=true; } }; //协议方向等,如果只有一个值,就 不需要输入了 String direction=regionDict.getConfigDirection(); if(StringUtils.isNotBlank(direction)&&direction.indexOf(",")>-1) { directionShow=true; } String protocol=regionDict.getConfigProtocol(); if(StringUtils.isNotBlank(protocol)&&protocol.indexOf(",")>-1) { protocolShow=true; } //packet ip reject if(regionDict.getFunctionId().equals(5)&&serviceDict!=null&&serviceDict.getServiceId().equals(16)) { protocolShow=false; } }else if(regionType==2||regionType==3){ String matchMethod= regionDict.getConfigMatchMethod(); if(StringUtils.isNotBlank(matchMethod)&&matchMethod.indexOf(",")>-1) { matchMethodShow=true; } String hex= regionDict.getConfigHex(); if(StringUtils.isNotBlank(hex)&&hex.indexOf(",")>-1) { hexShow=true; isCaseSensitiveShow=true; } String district=regionDict.getConfigDistrict(); if(StringUtils.isNotBlank(district)&&district.indexOf(",")>-1) { districtShow=true; } } List annotationList = Lists.newArrayList(); List fs=new ArrayList(); // Get annotation field //递归获取cls实体对象及父级对象的属性 getFields(fs, cls); for (Field f : fs){ //排除不需要导出的字段 if(!srcIpShow) { if(f.getName().equalsIgnoreCase("srcIpAddress")) { continue; } } if(!destIpShow) { if(f.getName().equalsIgnoreCase("destIpAddress")) { continue; } } if(!srcPortShow) { if(f.getName().equalsIgnoreCase("srcPort")) { continue; } } if(!destPortShow) { if(f.getName().equalsIgnoreCase("destPort")) { continue; } } /*if(!directionShow) { if(f.getName().equalsIgnoreCase("direction")) { continue; } } if(!protocolShow) { if(f.getName().equalsIgnoreCase("protocol")) { continue; } } if(!matchMethodShow) { if(f.getName().equalsIgnoreCase("matchMethod")) { continue; } } if(!hexShow) { if(f.getName().equalsIgnoreCase("isHex")) { continue; } } if(!isCaseSensitiveShow) { if(f.getName().equalsIgnoreCase("isCaseInsenstive")) { continue; } } if(!districtShow) { if(f.getName().equalsIgnoreCase("district")) { continue; } }*/ ExcelField ef = f.getAnnotation(ExcelField.class); if (ef != null && (ef.type()==0 || ef.type()==2)){ if (groups!=null && groups.length>0){ boolean inGroup = false; for (int g : groups){ if (inGroup){ break; } for (int efg : ef.groups()){ if (g == efg){ inGroup = true; annotationList.add(new Object[]{ef, f}); break; } } } }else{ annotationList.add(new Object[]{ef, f}); } } } // Get annotation method // Method[] ms = cls.getDeclaredMethods(); List ms=new ArrayList(); // Get annotation method //递归获取cls实体对象及父级对象的属性 getMethods(ms, cls); for (Method m : ms){ //排除不需要导出的字段 if(!srcIpShow) { if(m.getName().equalsIgnoreCase("get"+"srcIpAddress")||m.getName().equalsIgnoreCase("set"+"srcIpAddress")) { continue; } } if(!destIpShow) { if(m.getName().equalsIgnoreCase("get"+"destIpAddress")||m.getName().equalsIgnoreCase("set"+"destIpAddress")) { continue; } } if(!srcPortShow) { if(m.getName().equalsIgnoreCase("get"+"srcPort")||m.getName().equalsIgnoreCase("set"+"srcPort")) { continue; } } if(!destPortShow) { if(m.getName().equalsIgnoreCase("get"+"destPort")||m.getName().equalsIgnoreCase("set"+"destPort")) { continue; } } /*if(!directionShow) { if(m.getName().equalsIgnoreCase("get"+"direction")||m.getName().equalsIgnoreCase("set"+"direction")) { continue; } } if(!protocolShow) { if(m.getName().equalsIgnoreCase("get"+"protocol")||m.getName().equalsIgnoreCase("set"+"protocol")) { continue; } } if(!matchMethodShow) { if(m.getName().equalsIgnoreCase("get"+"matchMethod")||m.getName().equalsIgnoreCase("set"+"matchMethod")) { continue; } } if(!hexShow) { if(m.getName().equalsIgnoreCase("get"+"isHex")||m.getName().equalsIgnoreCase("set"+"isHex")) { continue; } } if(!isCaseSensitiveShow) { if(m.getName().equalsIgnoreCase("get"+"isCaseInsenstive")||m.getName().equalsIgnoreCase("set"+"isCaseInsenstive")) { continue; } } if(!districtShow) { if(m.getName().equalsIgnoreCase("get"+"district")||m.getName().equalsIgnoreCase("set"+"district")) { continue; } }*/ ExcelField ef = m.getAnnotation(ExcelField.class); if (ef != null && (ef.type()==0 || ef.type()==2)){ if (groups!=null && groups.length>0){ boolean inGroup = false; for (int g : groups){ if (inGroup){ break; } for (int efg : ef.groups()){ if (g == efg){ inGroup = true; annotationList.add(new Object[]{ef, m}); break; } } } }else{ annotationList.add(new Object[]{ef, m}); } } } // Field sorting Collections.sort(annotationList, new Comparator() { public int compare(Object[] o1, Object[] o2) { return new Integer(((ExcelField)o1[0]).sort()).compareTo( new Integer(((ExcelField)o2[0]).sort())); }; }); this.annotationList=annotationList; } @Override public List optRows(int sheetIndex, int curRow, List rowlist) { // TODO Auto-generated method stub List rowlistCopy=Lists.newArrayList(); rowlistCopy.addAll(rowlist); dataList.add(rowlistCopy); return null; } /** * 获取导入数据列表 * @param cls 导入对象类型 * @param groups 导入分组 * @throws SQLException * @throws SAXException * @throws ParserConfigurationException * @throws OpenXML4JException * @throws IOException * @throws ServiceException */ public BlockingQueue getDataList(Class cls) throws InstantiationException, IllegalAccessException, IOException, OpenXML4JException, ParserConfigurationException, SAXException, SQLException, ServiceException{ log.warn("start to load data..."); this.processOneSheet(uploadFile, this.sheetIndex); long start=System.currentTimeMillis(); if(regionDict==null) { throw new RuntimeException("regionDict is null!"); } if(dataList.size()==0) { throw new ServiceException(props.getProperty("noneData", "noneData")); }else if(dataList.size()>(Constants.IMPORT_LIMIT+1)) { throw new ServiceException(props.getProperty("import_limit_is", "The maximum import size is")+" "+Constants.IMPORT_LIMIT); } // Get excel data BlockingQueue _dataList =new ArrayBlockingQueue(dataList.size(),true); for (int i = 0; i < dataList.size(); i++) { E e = (E)cls.newInstance(); if(i<=headerNum) { continue; } int column = 0; List row=dataList.get(i); StringBuilder sb = new StringBuilder(); for (Object[] os : annotationList){ Object val=row.get(column); column++; if (val != null){ ExcelField ef = (ExcelField)os[0]; // If is dict type, get dict value if (StringUtils.isNotBlank(ef.dictType())){ Object val1 = DictUtils.getDictCode(ef.dictType(), val.toString(), ""); //没有获取到字典值的话会影响验证判断 if(val1!=null&&StringUtils.isNotBlank(val1.toString())) { val=val1; } //log.debug("Dictionary type value: ["+i+","+colunm+"] " + val); } // Get param type and type cast Class valType = Class.class; if (os[1] instanceof Field){ valType = ((Field)os[1]).getType(); }else if (os[1] instanceof Method){ Method method = ((Method)os[1]); if ("get".equals(method.getName().substring(0, 3))){ valType = method.getReturnType(); }else if("set".equals(method.getName().substring(0, 3))){ valType = ((Method)os[1]).getParameterTypes()[0]; } } //log.debug("Import value type: ["+i+","+column+"] " + valType); try { if (valType == String.class){ String s = String.valueOf(val.toString().trim()); //0.0.0.0表示任意IP的含义 if(StringUtils.endsWith(s, ".0") && !s.endsWith("0.0.0.0")){ val = StringUtils.substringBefore(s, ".0"); }else{ val = String.valueOf(val.toString().trim()); } }else if (valType == Integer.class){ val = Double.valueOf(val.toString().trim()).intValue(); }else if (valType == Long.class){ val = Double.valueOf(val.toString().trim()).longValue(); }else if (valType == Double.class){ val = Double.valueOf(val.toString().trim()); }else if (valType == Float.class){ val = Float.valueOf(val.toString().trim()); }else if (valType == Date.class){ val = DateUtil.getJavaDate((Double)val); }else{ if (ef.fieldType() != Class.class){ val = ef.fieldType().getMethod("getValue", String.class).invoke(null, val.toString().trim()); }else{ val = Class.forName(this.getClass().getName().replaceAll(this.getClass().getSimpleName(), "fieldtype."+valType.getSimpleName()+"Type")).getMethod("getValue", String.class).invoke(null, val.toString().trim()); } } } catch (Exception ex) { log.info("Get cell value ["+i+","+column+"] error: " + ex.toString()); val = null; } // set entity value if (os[1] instanceof Field){ Reflections.invokeSetter(e, ((Field)os[1]).getName(), val); }else if (os[1] instanceof Method){ String mthodName = ((Method)os[1]).getName(); if ("get".equals(mthodName.substring(0, 3))){ mthodName = "set"+StringUtils.substringAfter(mthodName, "get"); } Reflections.invokeMethod(e, mthodName, new Class[] {valType}, new Object[] {val}); } //设置索引值 Reflections.invokeSetter(e, "index", (i+1)); } sb.append(val+", "); } _dataList.offer(e); row.clear(); log.debug("Read success: ["+i+"] "+sb.toString()); } long end=System.currentTimeMillis(); log.warn(" load data finish,size:"+_dataList.size()+",cost:"+(end-start)); return _dataList; } public File getUploadFile() { return uploadFile; } public void setUploadFile(File uploadFile) { this.uploadFile = uploadFile; } // /** // * 导入测试 // */ // public static void main(String[] args) throws Throwable { // // ImportExcel ei = new ImportExcel("target/export.xlsx", 1); // // for (int i = ei.getDataRowNum(); i < ei.getLastDataRowNum(); i++) { // Row row = ei.getRow(i); // for (int j = 0; j < ei.getLastCellNum(); j++) { // Object val = ei.getCellValue(row, j); // System.out.print(val+", "); // } // System.out.print("\n"); // } // // } }