diff --git a/pom.xml b/pom.xml index 74d7205..4dee80d 100644 --- a/pom.xml +++ b/pom.xml @@ -181,12 +181,12 @@ ${storm.version} - - junit - junit - 4.12 - test - + + + + + + com.alibaba diff --git a/src/main/java/cn/ac/iie/storm/bean/SessionRecordLog.java b/src/main/java/cn/ac/iie/storm/bean/SessionRecordLog.java index 96cbed0..305292b 100644 --- a/src/main/java/cn/ac/iie/storm/bean/SessionRecordLog.java +++ b/src/main/java/cn/ac/iie/storm/bean/SessionRecordLog.java @@ -4,7 +4,9 @@ package cn.ac.iie.storm.bean; * @author wangchengcheng */ public class SessionRecordLog { - + private String ydy; + private String mddy; + private int yys; private String http_request_header; private String gjc; private String common_user_region; @@ -24,7 +26,6 @@ public class SessionRecordLog { private String common_isp; private int common_address_type; private String common_app_label; - private int common_app_id; private int common_c2s_pkt_num; private long common_c2s_byte_num; private int common_s2c_pkt_num; @@ -73,6 +74,8 @@ public class SessionRecordLog { private String yjxy; private String common_protocol_label; + private String lx; + public String getHttp_content_type() { return http_content_type; } @@ -289,13 +292,7 @@ public class SessionRecordLog { this.common_app_label = common_app_label; } - public int getCommon_app_id() { - return common_app_id; - } - public void setCommon_app_id(int common_app_id) { - this.common_app_id = common_app_id; - } public int getCommon_c2s_pkt_num() { return common_c2s_pkt_num; @@ -597,7 +594,40 @@ public class SessionRecordLog { return protocol; } + + public void setProtocol(String protocol) { this.protocol = protocol; } + public int getYys() { + return yys; + } + + public void setYys(int yys) { + this.yys = yys; + } + + public String getYdy() { + return ydy; + } + + public void setYdy(String ydy) { + this.ydy = ydy; + } + + public String getMddy() { + return mddy; + } + + public void setMddy(String mddy) { + this.mddy = mddy; + } + + public String getLx() { + return lx; + } + + public void setLx(String lx) { + this.lx = lx; + } } \ No newline at end of file diff --git a/src/main/java/cn/ac/iie/storm/bean/Traffic_Forward.java b/src/main/java/cn/ac/iie/storm/bean/Traffic_Forward.java index 1e8fed5..53a29ab 100644 --- a/src/main/java/cn/ac/iie/storm/bean/Traffic_Forward.java +++ b/src/main/java/cn/ac/iie/storm/bean/Traffic_Forward.java @@ -16,6 +16,7 @@ public class Traffic_Forward { private String labels; private String zsmc; private String yjxy; + private String lx; public String getYjxy() { return yjxy; @@ -128,4 +129,12 @@ public class Traffic_Forward { public void setGjc(String gjc) { this.gjc = gjc; } + + public String getLx() { + return lx; + } + + public void setLx(String lx) { + this.lx = lx; + } } diff --git a/src/main/java/cn/ac/iie/storm/bolt/conn/ConnHTTPSendBolt.java b/src/main/java/cn/ac/iie/storm/bolt/conn/ConnHTTPSendBolt.java index 7e38739..0afcf3f 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/conn/ConnHTTPSendBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/conn/ConnHTTPSendBolt.java @@ -44,7 +44,7 @@ public class ConnHTTPSendBolt extends BaseBasicBolt { i++; } if (connHttpPost!=null&&!connHttpPost.containsValue("200")){ - logger.error("-------条数为:"+httpcount+"connhttp数据发送到"+postUrl+"成功-----"+connHttpPost); + logger.error("-------条数为:"+httpcount+"connhttp数据发送到"+postUrl+"失败-----"+connHttpPost); conn_http_fail = conn_http_fail + httpcount; } } diff --git a/src/main/java/cn/ac/iie/storm/bolt/conn/ConnIPSendBolt.java b/src/main/java/cn/ac/iie/storm/bolt/conn/ConnIPSendBolt.java index eb7db56..219c04d 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/conn/ConnIPSendBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/conn/ConnIPSendBolt.java @@ -12,7 +12,6 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.TupleUtils; import java.util.HashMap; -import java.util.List; import java.util.Map; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; @@ -46,7 +45,7 @@ public class ConnIPSendBolt extends BaseBasicBolt { i++; } if (connIpPost!=null&&!connIpPost.containsValue("200")){ - logger.error("-------条数为:"+ipcount+"connip数据发送到"+postUrl+"成功-----"+connIpPost); + logger.error("-------条数为:"+ipcount+"connip数据发送到"+postUrl+"失败-----"+connIpPost); conn_ip_fail = conn_ip_fail + ipcount; } } diff --git a/src/main/java/cn/ac/iie/storm/bolt/conn/ConnParseBolt.java b/src/main/java/cn/ac/iie/storm/bolt/conn/ConnParseBolt.java index b18c94d..aef73f3 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/conn/ConnParseBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/conn/ConnParseBolt.java @@ -4,6 +4,7 @@ import cn.ac.iie.storm.bean.SessionRecordLog; import cn.ac.iie.storm.utils.config.StreamAggregateConfig; import com.alibaba.fastjson.JSONObject; +import com.zdjizhi.utils.StringUtil; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; @@ -23,28 +24,27 @@ import java.util.List; import java.util.Map; +import static cn.ac.iie.storm.utils.general.GetMap.getDy; import static cn.ac.iie.storm.utils.general.CONNAssignment.*; import static cn.ac.iie.storm.utils.general.TransFormUtils.*; -import static cn.ac.iie.storm.utils.hos.HosService.httpPostObject; -import static cn.ac.iie.storm.utils.hos.Send2S3.SendObject2S3; -import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost; -import static cn.ac.iie.storm.utils.http.HttpGetUrl.getS3Url; + import static cn.ac.iie.storm.utils.http.HttpGetUrl.getYL5; public class ConnParseBolt extends BaseBasicBolt { private final static Logger logger = Logger.getLogger(ConnParseBolt.class); - - private static final long serialVersionUID = 7483439472002862539L; - private List IPList = new ArrayList(); - private List HTTPList = new ArrayList(); - private List FTPList = new ArrayList(); - private List SSLList = new ArrayList(); - private List VPNList = new ArrayList(); - private List DNSList = new ArrayList(); - private List MAILList = new ArrayList(); - private List JSTXList = new ArrayList(); + + + private List ipList = new ArrayList(); + private List httpList = new ArrayList(); + private List ftpList = new ArrayList(); + private List sslList = new ArrayList(); + private List vpnList = new ArrayList(); + private List dnsList = new ArrayList(); + private List mailList = new ArrayList(); + private List jstxList = new ArrayList(); + private SessionRecordLog sessionRecordLog; @@ -53,6 +53,18 @@ public class ConnParseBolt extends BaseBasicBolt { private long connOutputSize = 0L; private long connInputSize = 0L; + private long yys_error=0L; + + private long ydy_inside=0L; + private long mddy_inside=0L; + + private long ydy_outside=0L; + private long mddy_outside=0L; + + private long ydy_unknown=0L; + private long mddy_unknown=0L; + + private Schema vpnSchema; private Schema ipSchema; private Schema httpSchema; @@ -62,8 +74,8 @@ public class ConnParseBolt extends BaseBasicBolt { private Schema mailSchema; private Schema jstxSchema; - String bucketName = null; - String filename = null; + String hosBucketName = null; + String fileName = null; public void prepare(Map stormConf, TopologyContext context) { ipSchema = new Schema.Parser().parse(StreamAggregateConfig.IP_RZZB_SCHEMA); @@ -87,185 +99,226 @@ public class ConnParseBolt extends BaseBasicBolt { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { if(TupleUtils.isTick(tuple)){ try { - if (!VPNList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.CONN_VPN_STREAMID, new Values((Object) generate(vpnSchema, VPNList),VPNList.size())); - VPNList.clear(); + if (!vpnList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.CONN_VPN_STREAMID, new Values(generate(vpnSchema, vpnList),vpnList.size())); + vpnList.clear(); } - if (!DNSList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.CONN_DNS_STREAMID, new Values((Object) generate(dnsSchema, DNSList),DNSList.size())); - DNSList.clear(); + if (!dnsList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.CONN_DNS_STREAMID, new Values((generate(dnsSchema, dnsList)),dnsList.size())); + dnsList.clear(); } - if (!MAILList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.CONN_MAIL_STREAMID, new Values((Object) generate(mailSchema, MAILList),MAILList.size())); - MAILList.clear(); + if (!mailList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.CONN_MAIL_STREAMID, new Values((generate(mailSchema, mailList)),mailList.size())); + mailList.clear(); } - if (!FTPList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.CONN_FTP_STREAMID, new Values((Object) generate(ftpSchema, FTPList),FTPList.size())); - FTPList.clear(); + if (!ftpList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.CONN_FTP_STREAMID, new Values(generate(ftpSchema, ftpList),ftpList.size())); + ftpList.clear(); } - if (!JSTXList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.CONN_JSTX_STREAMID, new Values((Object) generate(jstxSchema, JSTXList),JSTXList.size())); - JSTXList.clear(); + if (!jstxList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.CONN_JSTX_STREAMID, new Values(generate(jstxSchema, jstxList),jstxList.size())); + jstxList.clear(); } }catch (Exception e){ e.printStackTrace(); - logger.error("----------批数据发送异常-----------" + e); + logger.error("----------批数据发送异常-----------" + e ); } - basicOutputCollector.emit(StreamAggregateConfig.CONN_YBInputSize_STREAMID, new Values(connInputSize,count,loadFail,connOutputSize)); + + basicOutputCollector.emit(StreamAggregateConfig.CONN_YBInputSize_STREAMID, new Values(connInputSize,count,loadFail,connOutputSize,yys_error,ydy_inside,ydy_outside,ydy_unknown,mddy_inside,mddy_outside,mddy_unknown)); connInputSize = 0; count = 0; loadFail = 0; connOutputSize = 0; + ydy_inside=0L; + mddy_inside=0L; + ydy_outside=0L; + mddy_outside=0L; + ydy_unknown=0L; + mddy_unknown=0L; + yys_error=0L; }else { try { String line = tuple.getString(0); sessionRecordLog = JSONObject.parseObject(line, SessionRecordLog.class); + if(StringUtil.isBlank(sessionRecordLog.getCommon_data_center())){ + yys_error++; + logger.error("yys信息为空,请检查日志字段。"+"异常数据为"+line); + } + sessionRecordLog.setYys(getYys(sessionRecordLog.getCommon_data_center())); + sessionRecordLog.setYdy(getDy(sessionRecordLog.getCommon_client_location())); + sessionRecordLog.setMddy(getDy(sessionRecordLog.getCommon_server_location())); + if(sessionRecordLog.getYys()!=StreamAggregateConfig.YYS_NUM){ + yys_error++; + logger.error("yys信息为空,请检查日志字段。"+"异常数据为"+line); + } + + if (StringUtil.startsWith(sessionRecordLog.getYdy(),"65")&&sessionRecordLog.getYdy().length()==6){ + ydy_inside++; + }else if (sessionRecordLog.getYdy().equals("0")){ + ydy_unknown++; + }else { + ydy_outside++; + } + + if (StringUtil.startsWith(sessionRecordLog.getMddy(),"65")&&sessionRecordLog.getMddy().length()==6){ + mddy_inside++; + }else if(sessionRecordLog.getMddy().equals("0")){ + mddy_unknown++; + }else { + mddy_outside++; + } + connInputSize = connInputSize + sessionRecordLog.getCommon_s2c_byte_num(); connOutputSize = connOutputSize + sessionRecordLog.getCommon_c2s_byte_num(); - count = count + 1; + //按common_schema_type的值对数据进行分类发 - if (sessionRecordLog.getCommon_app_id() >= 100100) { + if (sessionRecordLog.getCommon_app_label()!=null&&StringUtil.startsWith(sessionRecordLog.getCommon_app_label(),"05")) { sessionRecordLog.setCommon_schema_type("JSTX"); try { + count++; GenericRecord jstx_rzzb = new GenericData.Record(jstxSchema); setJSTX_RZZB(jstx_rzzb, sessionRecordLog); - JSTXList.add(jstx_rzzb); - if (JSTXList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.CONN_JSTX_STREAMID, new Values((Object) generate(jstxSchema, JSTXList), JSTXList.size())); - JSTXList.clear(); + jstxList.add(jstx_rzzb); + if (jstxList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.CONN_JSTX_STREAMID, new Values(generate(jstxSchema, jstxList), jstxList.size())); + jstxList.clear(); } } catch (Exception e) { e.printStackTrace(); - logger.error("-----------jstx_rzzb数据封装处理异常-----------" + e); + logger.error("-----------jstx_rzzb数据封装处理异常-----------" + e + "异常数据为:" + line); loadFail++; } - } else { switch (sessionRecordLog.getCommon_schema_type()) { case "BASE": if (sessionRecordLog.getCommon_l7_protocol().equals("PPTP") || sessionRecordLog.getCommon_l7_protocol().equals("L2TP") || sessionRecordLog.getCommon_l7_protocol().equals("OPENVPN") || sessionRecordLog.getCommon_l7_protocol().equals("ISAKMP") || sessionRecordLog.getCommon_protocol_label().contains("PPTP")||sessionRecordLog.getCommon_protocol_label().contains("L2TP")||sessionRecordLog.getCommon_protocol_label().contains("OPENVPN")||sessionRecordLog.getCommon_protocol_label().contains("IPSEC")){ - sessionRecordLog.setCommon_schema_type("VPN"); try { + count++; + sessionRecordLog.setCommon_schema_type("VPN"); GenericRecord vpn_rzzb = new GenericData.Record(vpnSchema); setVPN_RZZB(vpn_rzzb, sessionRecordLog); - VPNList.add(vpn_rzzb); - if (VPNList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.CONN_VPN_STREAMID, new Values((Object) generate(vpnSchema, VPNList), VPNList.size())); - VPNList.clear(); + vpnList.add(vpn_rzzb); + if (vpnList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.CONN_VPN_STREAMID, new Values(generate(vpnSchema, vpnList), vpnList.size())); + vpnList.clear(); } } catch (Exception e) { - logger.error("-----------vpn_rzzb数据封装处理异常-----------" + e); + logger.error("-----------vpn_rzzb数据封装处理异常-----------" + e + "异常数据为:" + line); loadFail++; } - } else if (sessionRecordLog.getCommon_l7_protocol().equals("FTP")) { - try { - sessionRecordLog.setCommon_schema_type("FTP"); - GenericRecord ftp_rzzb = new GenericData.Record(ftpSchema); - setFTP_RZZB(ftp_rzzb, sessionRecordLog); - FTPList.add(ftp_rzzb); - if (FTPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.CONN_FTP_STREAMID, new Values((Object) generate(ftpSchema, FTPList), FTPList.size())); - FTPList.clear(); - } - } catch (Exception e) { - e.printStackTrace(); - logger.error("-----------ftp_rzzb数据封装处理异常-----------" + e); - loadFail++; - } - } else { - sessionRecordLog.setCommon_schema_type("IP"); try { + count++; + sessionRecordLog.setCommon_schema_type("IP"); GenericRecord ip_rzzb = new GenericData.Record(ipSchema); setIP_RZZB(ip_rzzb, sessionRecordLog); - IPList.add(ip_rzzb); - if (IPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.CONN_IP_STREAMID, new Values((Object) generate(ipSchema, IPList), IPList.size())); - IPList.clear(); + ipList.add(ip_rzzb); + if (ipList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.CONN_IP_STREAMID, new Values(generate(ipSchema, ipList), ipList.size())); + ipList.clear(); } } catch (Exception e) { e.printStackTrace(); - logger.error("-----------ip_rzzb数据封装处理异常-----：------" + e); + logger.error("-----------ip_rzzb数据封装处理异常-----：------" + e + "异常数据为:" + line); loadFail++; } } break; + case "FTP": + try { + count++; + GenericRecord ftp_rzzb = new GenericData.Record(ftpSchema); + setFTP_RZZB(ftp_rzzb, sessionRecordLog); + ftpList.add(ftp_rzzb); + if (ftpList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.CONN_FTP_STREAMID, new Values(generate(ftpSchema, ftpList), ftpList.size())); + ftpList.clear(); + } + } catch (Exception e) { + e.printStackTrace(); + logger.error("-----------ftp_rzzb数据封装处理异常-----------" + e + "异常数据为:" + line); + loadFail++; + } case "HTTP": try { if (sessionRecordLog.getHttp_response_body() != null) { String[] split = sessionRecordLog.getHttp_response_body().split("/"); - bucketName = split[split.length - 2]; - filename = split[split.length - 1]; - basicOutputCollector.emit("conn-unstructured", new Values(bucketName, filename, StreamAggregateConfig.T_HTTP_RZZB)); - sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_HTTP_RZZB, filename)); + hosBucketName = split[split.length - 2]; + fileName = split[split.length - 1]; + basicOutputCollector.emit("conn-unstructured", new Values(hosBucketName, fileName, StreamAggregateConfig.T_HTTP_RZZB)); + sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_HTTP_RZZB, fileName)); } + count++; GenericRecord http_rzzb = new GenericData.Record(httpSchema); sessionRecordLog.setCommon_schema_type("HTTP"); setHTTP_RZZB(http_rzzb, sessionRecordLog); - HTTPList.add(http_rzzb); - if (HTTPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.CONN_HTTP_STREAMID, new Values((Object) generate(httpSchema, HTTPList), HTTPList.size())); - HTTPList.clear(); + httpList.add(http_rzzb); + if (httpList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.CONN_HTTP_STREAMID, new Values(generate(httpSchema, httpList), httpList.size())); + httpList.clear(); } } catch (Exception e) { e.printStackTrace(); - logger.error("-----------http_rzzb数据封装处理异常-----------" + e); + logger.error("-----------http_rzzb数据封装处理异常-----------" + e +"异常数据为:" + line); loadFail++; } break; case "SSL": - sessionRecordLog.setCommon_schema_type("SSL"); try { + count++; + sessionRecordLog.setCommon_schema_type("SSL"); GenericRecord ssl_rzzb = new GenericData.Record(sslSchema); setSSL_RZZB(ssl_rzzb, sessionRecordLog); - SSLList.add(ssl_rzzb); - if (SSLList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.CONN_SSL_STREAMID, new Values((Object) generate(sslSchema, SSLList), SSLList.size())); - SSLList.clear(); + sslList.add(ssl_rzzb); + if (sslList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.CONN_SSL_STREAMID, new Values(generate(sslSchema, sslList), sslList.size())); + sslList.clear(); } } catch (Exception e) { e.printStackTrace(); - logger.error("-----------ssl_rzzb数据封装处理异常-----------" + e); + logger.error("-----------ssl_rzzb数据封装处理异常-----------" + e +"异常数据为:" + line); loadFail++; } break; case "DNS": - sessionRecordLog.setCommon_schema_type("DNS"); try { + count++; + sessionRecordLog.setCommon_schema_type("DNS"); GenericRecord dns_rzzb = new GenericData.Record(dnsSchema); setDNS_RZZB(dns_rzzb, sessionRecordLog); if (dns_rzzb.get("DNS_NAME")!=("")){ - DNSList.add(dns_rzzb); + dnsList.add(dns_rzzb); }else { loadFail++; } - if (DNSList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.CONN_DNS_STREAMID, new Values((Object) generate(dnsSchema, DNSList), DNSList.size())); - DNSList.clear(); + if (dnsList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.CONN_DNS_STREAMID, new Values(generate(dnsSchema, dnsList), dnsList.size())); + dnsList.clear(); } } catch (Exception e) { e.printStackTrace(); - logger.error("-----------dns_rzzb数据封装处理异常-----------" + e); + logger.error("-----------dns_rzzb数据封装处理异常-----------" + e + "异常数据为:" + line); loadFail++; } break; case "MAIL": + count++; try { if (sessionRecordLog.getHttp_response_body() != null) { String[] split = sessionRecordLog.getHttp_response_body().split("/"); - bucketName = split[split.length - 2]; - filename = split[split.length - 1]; - basicOutputCollector.emit("conn-unstructured", new Values(bucketName, filename, StreamAggregateConfig.T_MAIL_RZZB)); - sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MAIL_RZZB, filename)); + hosBucketName = split[split.length - 2]; + fileName = split[split.length - 1]; + basicOutputCollector.emit("conn-unstructured", new Values(hosBucketName, fileName, StreamAggregateConfig.T_MAIL_RZZB)); + sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MAIL_RZZB, fileName)); } - GenericRecord mail_rzzb = new GenericData.Record(mailSchema); sessionRecordLog.setCommon_schema_type("MAIL"); + GenericRecord mail_rzzb = new GenericData.Record(mailSchema); setMAIL_RZZB(mail_rzzb, sessionRecordLog); - MAILList.add(mail_rzzb); - if (MAILList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.CONN_MAIL_STREAMID, new Values((Object) generate(mailSchema, MAILList), MAILList.size())); - MAILList.clear(); + mailList.add(mail_rzzb); + if (mailList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.CONN_MAIL_STREAMID, new Values(generate(mailSchema, mailList), mailList.size())); + mailList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -277,9 +330,9 @@ public class ConnParseBolt extends BaseBasicBolt { break; } } - } catch(Exception e){ - logger.error("通联数据格式错误"+e); + logger.error("通联数据格式错误"+ e ); + loadFail++; } } @@ -295,9 +348,8 @@ public class ConnParseBolt extends BaseBasicBolt { outputFieldsDeclarer.declareStream(StreamAggregateConfig.CONN_JSTX_STREAMID, new Fields(StreamAggregateConfig.CONN_JSTX_STREAMID,"jstxcount")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.CONN_FTP_STREAMID, new Fields(StreamAggregateConfig.CONN_FTP_STREAMID,"ftpcount")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.CONN_SSL_STREAMID, new Fields(StreamAggregateConfig.CONN_SSL_STREAMID,"sslcount")); - outputFieldsDeclarer.declareStream(StreamAggregateConfig.CONN_YBInputSize_STREAMID, new Fields("connInputSize","count","loadFail","connOutputSize")); - outputFieldsDeclarer.declareStream("conn-unstructured", new Fields("bucketName","filename","bucketname")); - + outputFieldsDeclarer.declareStream(StreamAggregateConfig.CONN_YBInputSize_STREAMID, new Fields("connInputSize","count","loadFail","connOutputSize","yys_error","ydy_inside","ydy_outside","ydy_unknown","mddy_inside","mddy_outside","mddy_unknown")); + outputFieldsDeclarer.declareStream("conn-unstructured", new Fields("hosBucketName","fileName","s3BucketName")); } diff --git a/src/main/java/cn/ac/iie/storm/bolt/conn/ConnSSLSendBolt.java b/src/main/java/cn/ac/iie/storm/bolt/conn/ConnSSLSendBolt.java index 339e10e..febb5c7 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/conn/ConnSSLSendBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/conn/ConnSSLSendBolt.java @@ -12,7 +12,6 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.TupleUtils; import java.util.HashMap; -import java.util.List; import java.util.Map; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; diff --git a/src/main/java/cn/ac/iie/storm/bolt/conn/ConnSendBolt.java b/src/main/java/cn/ac/iie/storm/bolt/conn/ConnSendBolt.java index 9305c56..758a916 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/conn/ConnSendBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/conn/ConnSendBolt.java @@ -12,7 +12,6 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.TupleUtils; import java.util.HashMap; -import java.util.List; import java.util.Map; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; diff --git a/src/main/java/cn/ac/iie/storm/bolt/conn/ConnUnstructuredSendBolt.java b/src/main/java/cn/ac/iie/storm/bolt/conn/ConnUnstructuredSendBolt.java index 55a14bd..2138ebb 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/conn/ConnUnstructuredSendBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/conn/ConnUnstructuredSendBolt.java @@ -8,31 +8,30 @@ import org.apache.storm.topology.base.BaseBasicBolt; import org.apache.storm.tuple.Tuple; import java.io.IOException; -import java.util.List; + import static cn.ac.iie.storm.utils.hos.HosService.httpPostObject; import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; public class ConnUnstructuredSendBolt extends BaseBasicBolt { - String bucketName = null; - String filename = null; - String bucketname = null; + String hosBucketName = null; + String fileName = null; + String s3BucketName = null; String s3Url; private final static Logger logger = Logger.getLogger(ConnUnstructuredSendBolt.class); @Override public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { try { - bucketName = tuple.getStringByField("bucketName"); - filename = tuple.getStringByField("filename"); - bucketname = tuple.getStringByField("bucketname"); - byte[] bytes = httpPostObject(bucketName, filename); + hosBucketName = tuple.getStringByField("hosBucketName"); + fileName = tuple.getStringByField("fileName"); + s3BucketName = tuple.getStringByField("s3BucketName"); + byte[] bytes = httpPostObject(hosBucketName, fileName); int connAnsyHttpPost =0; - if(bytes!=null) { int i=0; while (i< StreamAggregateConfig.S3_SENT_COUNT) { - s3Url = getS3Url(bucketname, filename); + s3Url = getS3Url(s3BucketName, fileName); connAnsyHttpPost = getConnAnsyHttpPost(s3Url, bytes); if (connAnsyHttpPost==200){ break; @@ -40,12 +39,11 @@ public class ConnUnstructuredSendBolt extends BaseBasicBolt { i++; } if (connAnsyHttpPost!=200){ - logger.error("桶："+bucketName+"下的文件"+filename+"发送失败"); + logger.error("桶："+hosBucketName+"下的文件"+fileName+"发送失败"); } - } } catch (IOException e) { - logger.error("桶："+bucketName+"下的文件"+filename+"发送失败原因为："+e); + logger.error("桶："+hosBucketName+"下的文件"+fileName+"发送失败原因为："+e); } } diff --git a/src/main/java/cn/ac/iie/storm/bolt/gk/GKHTTPSendBolt.java b/src/main/java/cn/ac/iie/storm/bolt/gk/GKHTTPSendBolt.java index eacc445..a3dbc9b 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/gk/GKHTTPSendBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/gk/GKHTTPSendBolt.java @@ -12,14 +12,12 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.TupleUtils; import java.util.HashMap; -import java.util.List; import java.util.Map; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; public class GKHTTPSendBolt extends BaseBasicBolt { private final static Logger logger = Logger.getLogger(GKHTTPSendBolt.class); - private long gk_http_success=0L; private long gk_http_fail=0L; String postUrl; diff --git a/src/main/java/cn/ac/iie/storm/bolt/gk/GKIPSendBolt.java b/src/main/java/cn/ac/iie/storm/bolt/gk/GKIPSendBolt.java index 9aeed64..e171be8 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/gk/GKIPSendBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/gk/GKIPSendBolt.java @@ -12,7 +12,6 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.TupleUtils; import java.util.HashMap; -import java.util.List; import java.util.Map; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; diff --git a/src/main/java/cn/ac/iie/storm/bolt/gk/GKSSLSendBolt.java b/src/main/java/cn/ac/iie/storm/bolt/gk/GKSSLSendBolt.java index 7317af1..cd636b7 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/gk/GKSSLSendBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/gk/GKSSLSendBolt.java @@ -12,7 +12,6 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.TupleUtils; import java.util.HashMap; -import java.util.List; import java.util.Map; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; diff --git a/src/main/java/cn/ac/iie/storm/bolt/gk/GkParseBolt.java b/src/main/java/cn/ac/iie/storm/bolt/gk/GkParseBolt.java index 3c33bcb..ecb31f0 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/gk/GkParseBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/gk/GkParseBolt.java @@ -36,15 +36,15 @@ public class GkParseBolt extends BaseBasicBolt { private SessionRecordLog sessionRecordLog; - private List IPList = new ArrayList(); - private List HTTPList = new ArrayList(); - private List FTPList = new ArrayList(); - private List SSLList = new ArrayList(); - private List VPNList = new ArrayList(); - private List DNSList = new ArrayList(); - private List MAILList = new ArrayList(); - private List JSTXList = new ArrayList(); - private List WYGJCList = new ArrayList(); + private List ipList = new ArrayList(); + private List httpList = new ArrayList(); + private List ftpList = new ArrayList(); + private List sslList = new ArrayList(); + private List vpnList = new ArrayList(); + private List dnsList = new ArrayList(); + private List mailList = new ArrayList(); + private List jstxList = new ArrayList(); + private List wygjcList = new ArrayList(); private long count = 0L; @@ -60,9 +60,10 @@ public class GkParseBolt extends BaseBasicBolt { private Schema mailSchema; private Schema jstxSchema; private Schema wygjcSchema; + private String line; - String bucketName = null; - String filename = null; + String hosBucketName = null; + String fileName = null; @@ -83,42 +84,42 @@ public class GkParseBolt extends BaseBasicBolt { //首先获取一条数据 if (TupleUtils.isTick(tuple)) { try { - if (!IPList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.GK_IP_STREAMID, new Values((Object) generate(ipSchema, IPList),IPList.size())); - IPList.clear(); + if (!ipList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.GK_IP_STREAMID, new Values(generate(ipSchema, ipList),ipList.size())); + ipList.clear(); } - if (!HTTPList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.GK_HTTP_STREAMID, new Values((Object) generate(httpSchema, HTTPList),HTTPList.size())); - HTTPList.clear(); + if (!httpList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.GK_HTTP_STREAMID, new Values(generate(httpSchema, httpList),httpList.size())); + httpList.clear(); } - if (!SSLList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.GK_SSL_STREAMID, new Values((Object) generate(sslSchema, SSLList),SSLList.size())); - SSLList.clear(); + if (!sslList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.GK_SSL_STREAMID, new Values((generate(sslSchema, sslList)),sslList.size())); + sslList.clear(); } - if (!DNSList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.GK_DNS_STREAMID, new Values((Object) generate(dnsSchema, DNSList),DNSList.size())); - DNSList.clear(); + if (!dnsList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.GK_DNS_STREAMID, new Values(generate(dnsSchema, dnsList),dnsList.size())); + dnsList.clear(); } - if (!(FTPList.isEmpty())) { - basicOutputCollector.emit(StreamAggregateConfig.GK_FTP_STREAMID, new Values((Object) generate(ftpSchema, FTPList),FTPList.size())); - FTPList.clear(); + if (!(ftpList.isEmpty())) { + basicOutputCollector.emit(StreamAggregateConfig.GK_FTP_STREAMID, new Values(generate(ftpSchema, ftpList),ftpList.size())); + ftpList.clear(); } - if (!(WYGJCList.isEmpty())) { - basicOutputCollector.emit(StreamAggregateConfig.GK_WYGJC_STREAMID, new Values((Object) generate(wygjcSchema, WYGJCList),WYGJCList.size())); - WYGJCList.clear(); + if (!(wygjcList.isEmpty())) { + basicOutputCollector.emit(StreamAggregateConfig.GK_WYGJC_STREAMID, new Values(generate(wygjcSchema, wygjcList),wygjcList.size())); + wygjcList.clear(); } - if (!(JSTXList.isEmpty())) { - basicOutputCollector.emit(StreamAggregateConfig.GK_JSTX_STREAMID, new Values((Object) generate(jstxSchema, JSTXList),JSTXList.size())); - JSTXList.clear(); + if (!(wygjcList.isEmpty())) { + basicOutputCollector.emit(StreamAggregateConfig.GK_JSTX_STREAMID, new Values(generate(jstxSchema, wygjcList),wygjcList.size())); + wygjcList.clear(); } - if (!VPNList.isEmpty()) { - basicOutputCollector.emit(StreamAggregateConfig.GK_VPN_STREAMID, new Values((Object) generate(vpnSchema, VPNList),VPNList.size())); - VPNList.clear(); + if (!vpnList.isEmpty()) { + basicOutputCollector.emit(StreamAggregateConfig.GK_VPN_STREAMID, new Values(generate(vpnSchema, vpnList),vpnList.size())); + vpnList.clear(); } - if (!(MAILList.isEmpty())) { - basicOutputCollector.emit(StreamAggregateConfig.GK_MAIL_STREAMID, new Values((Object) generate(mailSchema, MAILList),MAILList.size())); - MAILList.clear(); + if (!(mailList.isEmpty())) { + basicOutputCollector.emit(StreamAggregateConfig.GK_MAIL_STREAMID, new Values(generate(mailSchema, mailList),mailList.size())); + mailList.clear(); } basicOutputCollector.emit(StreamAggregateConfig.GK_COUNT_STREAMID, new Values(count,loadfail)); @@ -130,10 +131,11 @@ public class GkParseBolt extends BaseBasicBolt { loadfail = 0; } else { try { - String line = tuple.getString(0); + line = tuple.getString(0); sessionRecordLog = JSONObject.parseObject(line, SessionRecordLog.class); UserRegion userRegion = JSONObject.parseObject(sessionRecordLog.getCommon_user_region(), UserRegion.class); Traffic_Forward traffic_forward = JSONObject.parseObject(userRegion.getTraffic_forward(), Traffic_Forward.class); + sessionRecordLog.setPzid(traffic_forward.getPzid()); sessionRecordLog.setFlid(traffic_forward.getFlid()); sessionRecordLog.setXzid(traffic_forward.getXzid()); @@ -141,11 +143,10 @@ public class GkParseBolt extends BaseBasicBolt { sessionRecordLog.setPzlx(traffic_forward.getPzlx()); sessionRecordLog.setGklb(traffic_forward.getGklb()); sessionRecordLog.setLabels(traffic_forward.getLabels()); - sessionRecordLog.setYyzl(traffic_forward.getYyzl()); - sessionRecordLog.setGjc(traffic_forward.getGjc()); - sessionRecordLog.setZsmc(traffic_forward.getZsmc()); - sessionRecordLog.setYjxy(traffic_forward.getYjxy()); - sessionRecordLog.setProtocol(userRegion.getProtocol()); + +// sessionRecordLog.setZsmc(traffic_forward.getZsmc()); + // sessionRecordLog.setYjxy(traffic_forward.getYjxy()); + int pzlx = traffic_forward.getPzlx(); //按pzlx的值对数据进行分类发送 //按pzlx的值对数据进行分类发送 @@ -160,10 +161,10 @@ public class GkParseBolt extends BaseBasicBolt { GenericRecord ip_gkrz = new GenericData.Record(ipSchema); sessionRecordLog.setCommon_schema_type("IP"); setIP_GKRZ(ip_gkrz, sessionRecordLog); - IPList.add(ip_gkrz); - if (IPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.GK_IP_STREAMID, new Values((Object) generate(ipSchema, IPList),IPList.size())); - IPList.clear(); + ipList.add(ip_gkrz); + if (ipList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.GK_IP_STREAMID, new Values((Object) generate(ipSchema, ipList),ipList.size())); + ipList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -181,13 +182,13 @@ public class GkParseBolt extends BaseBasicBolt { sessionRecordLog.setCommon_schema_type("DNS"); setDNS_GKZB(dns_gkzb, sessionRecordLog); if (dns_gkzb.get("DNS_NAME")!=("")){ - DNSList.add(dns_gkzb); + dnsList.add(dns_gkzb); }else { loadfail++; } - if (DNSList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.GK_DNS_STREAMID, new Values((Object) generate(dnsSchema, DNSList),DNSList.size())); - DNSList.clear(); + if (dnsList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.GK_DNS_STREAMID, new Values(generate(dnsSchema, dnsList),dnsList.size())); + dnsList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -201,20 +202,21 @@ public class GkParseBolt extends BaseBasicBolt { case 703: //反诈URL管控 count++; try { + sessionRecordLog.setYyzl(traffic_forward.getYyzl()); if (sessionRecordLog.getHttp_response_body()!=null){ String[] split = sessionRecordLog.getHttp_response_body().split("/"); - bucketName = split[split.length-2]; - filename = split[split.length-1]; - basicOutputCollector.emit("gk-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_MAIL_GKZB)); - sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_HTTP_GKZB, filename)); + hosBucketName = split[split.length-2]; + fileName = split[split.length-1]; + basicOutputCollector.emit("gk-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_MAIL_GKZB)); + sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_HTTP_GKZB, fileName)); } GenericRecord http_gkzb = new GenericData.Record(httpSchema); sessionRecordLog.setCommon_schema_type("HTTP"); setHTTP_GKZB(http_gkzb, sessionRecordLog); - HTTPList.add(http_gkzb); - if (HTTPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.GK_HTTP_STREAMID, new Values((Object) generate(httpSchema, HTTPList),HTTPList.size())); - HTTPList.clear(); + httpList.add(http_gkzb); + if (httpList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.GK_HTTP_STREAMID, new Values(generate(httpSchema, httpList),httpList.size())); + httpList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -225,13 +227,14 @@ public class GkParseBolt extends BaseBasicBolt { case 405: //特定证书封堵 count++; try { + sessionRecordLog.setZsmc(traffic_forward.getZsmc()); GenericRecord ssl_gkzb = new GenericData.Record(sslSchema); sessionRecordLog.setCommon_schema_type("SSL"); setSSL_GKZB(ssl_gkzb, sessionRecordLog); - SSLList.add(ssl_gkzb); - if (SSLList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.GK_SSL_STREAMID, new Values((Object) generate(sslSchema, SSLList),SSLList.size())); - SSLList.clear(); + sslList.add(ssl_gkzb); + if (sslList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.GK_SSL_STREAMID, new Values(generate(sslSchema, sslList),sslList.size())); + sslList.clear(); } } catch (Exception e) { @@ -245,21 +248,22 @@ public class GkParseBolt extends BaseBasicBolt { case 409: //搜索词封堵 count++; try { + sessionRecordLog.setYyzl(traffic_forward.getYyzl()); + sessionRecordLog.setGjc(traffic_forward.getGjc()); if (sessionRecordLog.getHttp_response_body()!=null){ String[] split = sessionRecordLog.getHttp_response_body().split("/"); - bucketName = split[split.length-2]; - filename = split[split.length-1]; - basicOutputCollector.emit("gk-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_MAIL_GKZB)); - sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_WYGJC_GKRZ, filename)); + hosBucketName = split[split.length-2]; + fileName = split[split.length-1]; + basicOutputCollector.emit("gk-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_MAIL_GKZB)); + sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_WYGJC_GKRZ, fileName)); } GenericRecord wygjc_gkzb = new GenericData.Record(wygjcSchema); sessionRecordLog.setCommon_schema_type("HTTP"); - setWYGJC_GKZB(wygjc_gkzb, sessionRecordLog); - WYGJCList.add(wygjc_gkzb); - if (WYGJCList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.GK_WYGJC_STREAMID, new Values((Object) generate(wygjcSchema, WYGJCList),WYGJCList.size())); - WYGJCList.clear(); + wygjcList.add(wygjc_gkzb); + if (wygjcList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.GK_WYGJC_STREAMID, new Values(generate(wygjcSchema, wygjcList),wygjcList.size())); + wygjcList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -271,20 +275,23 @@ public class GkParseBolt extends BaseBasicBolt { case 410: count++; try { + sessionRecordLog.setYjxy(traffic_forward.getYjxy()); + sessionRecordLog.setYyzl(traffic_forward.getYyzl()); + sessionRecordLog.setGjc(traffic_forward.getGjc()); if (sessionRecordLog.getHttp_response_body()!=null){ String[] split = sessionRecordLog.getHttp_response_body().split("/"); - bucketName = split[split.length-2]; - filename = split[split.length-1]; - basicOutputCollector.emit("gk-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_MAIL_GKZB)); - sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MAIL_GKZB, filename)); + hosBucketName = split[split.length-2]; + fileName = split[split.length-1]; + basicOutputCollector.emit("gk-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_MAIL_GKZB)); + sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MAIL_GKZB, fileName)); } GenericRecord mail_gkzb = new GenericData.Record(mailSchema); sessionRecordLog.setCommon_schema_type("MAIL"); setMAIL_GKZB(mail_gkzb, sessionRecordLog); - MAILList.add(mail_gkzb); - if (MAILList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.GK_MAIL_STREAMID, new Values((Object) generate(mailSchema, MAILList),MAILList.size())); - MAILList.clear(); + mailList.add(mail_gkzb); + if (mailList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.GK_MAIL_STREAMID, new Values(generate(mailSchema, mailList),mailList.size())); + mailList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -295,13 +302,15 @@ public class GkParseBolt extends BaseBasicBolt { case 408: //FTP关键字封堵 count++; try { + sessionRecordLog.setYyzl(traffic_forward.getYyzl()); + sessionRecordLog.setGjc(traffic_forward.getGjc()); GenericRecord ftp_gkzb = new GenericData.Record(ftpSchema); sessionRecordLog.setCommon_schema_type("FTP"); setFTP_GKZB(ftp_gkzb, sessionRecordLog); - FTPList.add(ftp_gkzb); - if (FTPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.GK_FTP_STREAMID, new Values((Object) generate(ftpSchema, FTPList),FTPList.size())); - FTPList.clear(); + ftpList.add(ftp_gkzb); + if (ftpList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.GK_FTP_STREAMID, new Values(generate(ftpSchema, ftpList),ftpList.size())); + ftpList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -313,14 +322,18 @@ public class GkParseBolt extends BaseBasicBolt { count++; try { GenericRecord vpn_gkzb = new GenericData.Record(vpnSchema); + sessionRecordLog.setLx(traffic_forward.getLx()); sessionRecordLog.setCommon_schema_type("VPN"); setVPN_GKZB(vpn_gkzb, sessionRecordLog); if(StringUtil.isNotEmpty(vpn_gkzb.get("DETAIL_ID"))) { - VPNList.add(vpn_gkzb); + vpnList.add(vpn_gkzb); + }else { + logger.error("vpnjc流量DETAIL_ID为空具体日志为:"+line); + loadfail++; } - if (VPNList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.GK_VPN_STREAMID, new Values((Object) generate(vpnSchema, VPNList),VPNList.size())); - VPNList.clear(); + if (vpnList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.GK_VPN_STREAMID, new Values(generate(vpnSchema, vpnList),vpnList.size())); + vpnList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -332,13 +345,15 @@ public class GkParseBolt extends BaseBasicBolt { case 413: //社交应用封堵 count++; try { + sessionRecordLog.setYyzl(traffic_forward.getYyzl()); + sessionRecordLog.setProtocol(userRegion.getProtocol()); GenericRecord jstx_gkzb = new GenericData.Record(jstxSchema); sessionRecordLog.setCommon_schema_type("JSTX"); setJSTX_GKZB(jstx_gkzb, sessionRecordLog); - JSTXList.add(jstx_gkzb); - if (JSTXList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.GK_JSTX_STREAMID, new Values((Object) generate(jstxSchema, JSTXList),JSTXList.size())); - JSTXList.clear(); + jstxList.add(jstx_gkzb); + if (jstxList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.GK_JSTX_STREAMID, new Values(generate(jstxSchema, jstxList),jstxList.size())); + jstxList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -350,7 +365,7 @@ public class GkParseBolt extends BaseBasicBolt { break; } } catch(Exception e){ - logger.error("GK数据处理异常"+e); + logger.error("GK数据处理异常"+e+"异常数据"+line); } } } @@ -367,7 +382,7 @@ public class GkParseBolt extends BaseBasicBolt { outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_VPN_STREAMID,new Fields(StreamAggregateConfig.GK_VPN_STREAMID,"vpncount")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_JSTX_STREAMID,new Fields(StreamAggregateConfig.GK_JSTX_STREAMID,"jstxcount")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_COUNT_STREAMID, new Fields("count","loadfail")); - outputFieldsDeclarer.declareStream("gk-unstructured", new Fields("bucketName","filename","bucketname")); + outputFieldsDeclarer.declareStream("gk-unstructured", new Fields("hosBucketName","fileName","s3BucketName")); } @Override diff --git a/src/main/java/cn/ac/iie/storm/bolt/gk/GkSendBolt.java b/src/main/java/cn/ac/iie/storm/bolt/gk/GkSendBolt.java index 4f5ce1a..360ece9 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/gk/GkSendBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/gk/GkSendBolt.java @@ -12,7 +12,6 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.TupleUtils; import java.util.HashMap; -import java.util.List; import java.util.Map; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; diff --git a/src/main/java/cn/ac/iie/storm/bolt/gk/GkUnstructuredSendBolt.java b/src/main/java/cn/ac/iie/storm/bolt/gk/GkUnstructuredSendBolt.java index 4f59f7d..f622881 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/gk/GkUnstructuredSendBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/gk/GkUnstructuredSendBolt.java @@ -9,32 +9,31 @@ import org.apache.storm.topology.base.BaseBasicBolt; import org.apache.storm.tuple.Tuple; import java.io.IOException; -import java.util.List; import static cn.ac.iie.storm.utils.hos.HosService.httpPostObject; import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; public class GkUnstructuredSendBolt extends BaseBasicBolt { - String bucketName = null; - String filename = null; - String bucketname = null; + String hosBucketName = null; + String fileName = null; + String s3BucketName = null; byte[] bytes; private final static Logger logger = Logger.getLogger(GkUnstructuredSendBolt.class); @Override public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { try { - bucketName = tuple.getStringByField("bucketName"); - filename = tuple.getStringByField("filename"); - bucketname = tuple.getStringByField("bucketname"); - bytes = httpPostObject(bucketName, filename); + hosBucketName = tuple.getStringByField("hosBucketName"); + fileName = tuple.getStringByField("fileName"); + s3BucketName = tuple.getStringByField("s3BucketName"); + bytes = httpPostObject(hosBucketName, fileName); int connAnsyHttpPost =0; String s3Url; if(bytes!=null) { int i = 0; String ip; while (i < StreamAggregateConfig.S3_SENT_COUNT) { - s3Url = getS3Url(bucketname, filename); + s3Url = getS3Url(s3BucketName, fileName); connAnsyHttpPost = getConnAnsyHttpPost(s3Url, bytes); if (connAnsyHttpPost == 200) { break; @@ -42,11 +41,11 @@ public class GkUnstructuredSendBolt extends BaseBasicBolt { i++; } if (connAnsyHttpPost != 200) { - logger.error("桶：" + bucketName + "下的文件" + filename + "发送失败"); + logger.error("桶：" + hosBucketName + "下的文件" + fileName + "发送失败"); } } } catch (IOException e) { - logger.error("桶："+bucketName+"下的文件"+filename+"发送失败原因为："+e); + logger.error("桶："+hosBucketName+"下的文件"+fileName+"发送失败原因为："+e); } } diff --git a/src/main/java/cn/ac/iie/storm/bolt/index/ConnCountBolt.java b/src/main/java/cn/ac/iie/storm/bolt/index/ConnCountBolt.java index fa34c34..736fd33 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/index/ConnCountBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/index/ConnCountBolt.java @@ -46,10 +46,21 @@ public class ConnCountBolt extends BaseBasicBolt { private long conn_ssl_success = 0L; private long conn_ssl_fail = 0L; - InetAddress ip=null; - private String localip; private float inputrate; private float outputrate; + + + private long yys_error=0L; + + private long ydy_inside=0L; + private long mddy_inside=0L; + + private long ydy_outside=0L; + private long mddy_outside=0L; + + private long ydy_unknown=0L; + private long mddy_unknown=0L; + static final Gauge InputRate = Gauge.build() .name("InputRate").labelNames("ServName","Duration").help("Program receiving flow rate, the unit is Mbps").register(); @@ -119,6 +130,30 @@ public class ConnCountBolt extends BaseBasicBolt { .name("connParseFail").labelNames("ServName","Duration").help("Program output flow rate, the unit is Mbps").register(); + static final Gauge ConnYysError = Gauge.build() + .name("ConnYysError").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register(); + + + static final Gauge ConnYdyInside = Gauge.build() + .name("ConnYdyInside").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register(); + + static final Gauge ConnYdyOutside = Gauge.build() + .name("ConnYdyOutside").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register(); + + static final Gauge ConnYdyUnknown = Gauge.build() + .name("ConnYdyUnknown").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register(); + + static final Gauge ConnMddyInside = Gauge.build() + .name("ConnMddyInside").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register(); + + static final Gauge ConnMddyOutside = Gauge.build() + .name("ConnmddyOutside").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register(); + + static final Gauge ConnMddyUnknown = Gauge.build() + .name("ConnMddyUnknown").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register(); + + + @Override public void execute(Tuple input, BasicOutputCollector collector) { @@ -162,6 +197,16 @@ public class ConnCountBolt extends BaseBasicBolt { connFtpSuccess.labels("conn-ftp-success",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(conn_ftp_success); connFtpFail.labels("conn-ftp-fail",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(conn_ftp_fail); + ConnYysError.labels("conn-yys-error",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(yys_error); + + ConnYdyInside.labels("conn-ydy-inside",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(ydy_inside); + ConnMddyInside.labels("conn-mddy-inside",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(mddy_inside); + + ConnYdyOutside.labels("conn-ydy-outside",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(ydy_outside); + ConnMddyOutside.labels("conn-mddy-outside",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(mddy_outside); + + ConnYdyUnknown.labels("conn-ydy-unknow",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(ydy_unknown); + ConnMddyUnknown.labels("conn-mddy-unknow",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(mddy_unknown); try { @@ -169,6 +214,9 @@ public class ConnCountBolt extends BaseBasicBolt { } catch (IOException e) { e.printStackTrace(); } + + inputrate = 0L; + outputrate = 0L; connCount= 0L; ConnSendCount = 0L; InputSize = 0L; @@ -191,17 +239,31 @@ public class ConnCountBolt extends BaseBasicBolt { conn_ftp_fail=0L; conn_ssl_success = 0L; conn_ssl_fail = 0L; + + yys_error=0L; + ydy_inside=0L; + mddy_inside=0L; + ydy_outside=0L; + mddy_outside=0L; + ydy_unknown=0L; + mddy_unknown=0L; }else { String sourceComponent = input.getSourceComponent(); switch (sourceComponent){ case "CONN-PARSE-COMPONENTID": if (input.getSourceStreamId().equals(StreamAggregateConfig.CONN_YBInputSize_STREAMID)){ - InputSize = InputSize + input.getLongByField("connInputSize"); connCount = connCount + input.getLongByField("count"); connParseFail = connParseFail + input.getLongByField("loadFail"); OutputSize = OutputSize + input.getLongByField("connOutputSize"); + yys_error = yys_error + input.getLongByField("yys_error"); + ydy_inside = ydy_inside + input.getLongByField("ydy_inside"); + ydy_outside = ydy_outside + input.getLongByField("ydy_outside"); + ydy_unknown = ydy_unknown + input.getLongByField("ydy_unknown"); + mddy_inside = mddy_inside + input.getLongByField("mddy_inside"); + mddy_outside = mddy_outside + input.getLongByField("mddy_outside"); + mddy_unknown = mddy_unknown + input.getLongByField("mddy_unknown"); } break; case "CONN-SEND": @@ -217,9 +279,6 @@ public class ConnCountBolt extends BaseBasicBolt { conn_jstx_success =conn_jstx_success + input.getLongByField("conn_jstx_success"); conn_jstx_fail =conn_jstx_fail + input.getLongByField("conn_jstx_fail"); - conn_jstx_success =conn_jstx_success + input.getLongByField("conn_jstx_success"); - conn_jstx_fail =conn_jstx_fail + input.getLongByField("conn_jstx_fail"); - conn_ftp_success =conn_ftp_success + input.getLongByField("conn_ftp_success"); conn_ftp_fail =conn_ftp_fail + input.getLongByField("conn_ftp_fail"); break; diff --git a/src/main/java/cn/ac/iie/storm/bolt/index/GkCountBolt.java b/src/main/java/cn/ac/iie/storm/bolt/index/GkCountBolt.java index b15bd61..98fdc62 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/index/GkCountBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/index/GkCountBolt.java @@ -213,8 +213,6 @@ public class GkCountBolt extends BaseBasicBolt { gk_jstx_success = gk_jstx_success + tuple.getLongByField("gk_jstx_success"); gk_jstx_fail = gk_jstx_fail + tuple.getLongByField("gk_jstx_fail"); - gk_jstx_success = gk_jstx_success + tuple.getLongByField("gk_jstx_success"); - gk_jstx_fail = gk_jstx_fail + tuple.getLongByField("gk_jstx_fail"); gk_ftp_success = gk_ftp_success + tuple.getLongByField("gk_ftp_success"); gk_ftp_fail = gk_ftp_fail + tuple.getLongByField("gk_ftp_fail"); diff --git a/src/main/java/cn/ac/iie/storm/bolt/jc/JCHTTPSendBolt.java b/src/main/java/cn/ac/iie/storm/bolt/jc/JCHTTPSendBolt.java index c2072b3..f1a565f 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/jc/JCHTTPSendBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/jc/JCHTTPSendBolt.java @@ -1,6 +1,6 @@ package cn.ac.iie.storm.bolt.jc; -import cn.ac.iie.storm.bolt.gk.GKHTTPSendBolt; + import cn.ac.iie.storm.utils.config.StreamAggregateConfig; import cn.ac.iie.storm.utils.http.HttpClientService; import org.apache.log4j.Logger; @@ -13,7 +13,6 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.TupleUtils; import java.util.HashMap; -import java.util.List; import java.util.Map; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; diff --git a/src/main/java/cn/ac/iie/storm/bolt/jc/JCParseBolt.java b/src/main/java/cn/ac/iie/storm/bolt/jc/JCParseBolt.java index 622b4d7..f1e9f67 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/jc/JCParseBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/jc/JCParseBolt.java @@ -26,7 +26,6 @@ import java.util.Map; import static cn.ac.iie.storm.utils.general.JCAssignment.*; import static cn.ac.iie.storm.utils.general.TransFormUtils.generate; -import static cn.ac.iie.storm.utils.http.HttpGetUrl.getS3Url; import static cn.ac.iie.storm.utils.http.HttpGetUrl.getYL5; public class JCParseBolt extends BaseBasicBolt { @@ -38,17 +37,17 @@ public class JCParseBolt extends BaseBasicBolt { - private List IPList = new ArrayList(); - private List HTTPList = new ArrayList(); - private List FTPList = new ArrayList(); - private List SSLList = new ArrayList(); - private List VPNList = new ArrayList(); - private List DNSList = new ArrayList(); - private List MAILList = new ArrayList(); - private List JSTXList = new ArrayList(); - private List WYGJCList = new ArrayList(); - private List PICList = new ArrayList(); - private List MEDIAList = new ArrayList(); + private List ipList = new ArrayList(); + private List httpList = new ArrayList(); + private List ftpList = new ArrayList(); + private List sslList = new ArrayList(); + private List vpnList = new ArrayList(); + private List dnsList = new ArrayList(); + private List mailList = new ArrayList(); + private List jstxList = new ArrayList(); + private List wygjcList = new ArrayList(); + private List picList = new ArrayList(); + private List mediaList = new ArrayList(); private long count = 0L; private long loadfail = 0L; private Schema vpnSchema; @@ -62,9 +61,12 @@ public class JCParseBolt extends BaseBasicBolt { private Schema wygjcSchema; private Schema picSchema; private Schema mediaSchema; + private String line; - String bucketName = null; - String filename = null; + + + String hosBucketName = null; + String fileName = null; @@ -88,61 +90,61 @@ public class JCParseBolt extends BaseBasicBolt { //首先获取一条数据 if (TupleUtils.isTick(tuple)) { try { - if (!IPList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.JC_IP_STREAMID, new Values((Object) generate(ipSchema, IPList),IPList.size())); - IPList.clear(); + if (!ipList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.JC_IP_STREAMID, new Values(generate(ipSchema, ipList),ipList.size())); + ipList.clear(); } - if (!HTTPList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.JC_HTTP_STREAMID, new Values((Object) generate(httpSchema, HTTPList),HTTPList.size())); - HTTPList.clear(); + if (!httpList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.JC_HTTP_STREAMID, new Values(generate(httpSchema, httpList),httpList.size())); + httpList.clear(); } - if (!SSLList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.JC_SSL_STREAMID, new Values((Object) generate(sslSchema, SSLList),SSLList.size())); - SSLList.clear(); + if (!sslList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.JC_SSL_STREAMID, new Values(generate(sslSchema, sslList),sslList.size())); + sslList.clear(); } - if (!DNSList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.JC_DNS_STREAMID, new Values((Object) generate(dnsSchema, DNSList),DNSList.size())); - DNSList.clear(); + if (!dnsList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.JC_DNS_STREAMID, new Values(generate(dnsSchema, dnsList),dnsList.size())); + dnsList.clear(); } - if (!(WYGJCList.isEmpty())) { - basicOutputCollector.emit(StreamAggregateConfig.JC_WYGJC_STREAMID, new Values((Object) generate(wygjcSchema,WYGJCList),WYGJCList.size())); - WYGJCList.clear(); + if (!(wygjcList.isEmpty())) { + basicOutputCollector.emit(StreamAggregateConfig.JC_WYGJC_STREAMID, new Values(generate(wygjcSchema,wygjcList),wygjcList.size())); + wygjcList.clear(); } - if (!(JSTXList.isEmpty())) { - basicOutputCollector.emit(StreamAggregateConfig.JC_JSTX_STREAMID, new Values((Object) generate(jstxSchema,JSTXList),JSTXList.size())); - JSTXList.clear(); + if (!(jstxList.isEmpty())) { + basicOutputCollector.emit(StreamAggregateConfig.JC_JSTX_STREAMID, new Values(generate(jstxSchema,jstxList),jstxList.size())); + jstxList.clear(); } - if (!(FTPList.isEmpty())) { - basicOutputCollector.emit(StreamAggregateConfig.JC_FTP_STREAMID, new Values((Object) generate(ftpSchema,FTPList),FTPList.size())); - FTPList.clear(); + if (!(ftpList.isEmpty())) { + basicOutputCollector.emit(StreamAggregateConfig.JC_FTP_STREAMID, new Values(generate(ftpSchema,ftpList),ftpList.size())); + ftpList.clear(); } - if (!(PICList.isEmpty())) { - basicOutputCollector.emit(StreamAggregateConfig.JC_PIC_STREAMID, new Values((Object) generate(picSchema,PICList),PICList.size())); - PICList.clear(); + if (!(picList.isEmpty())) { + basicOutputCollector.emit(StreamAggregateConfig.JC_PIC_STREAMID, new Values(generate(picSchema,picList),picList.size())); + picList.clear(); } - if (!(MEDIAList.isEmpty())) { - basicOutputCollector.emit(StreamAggregateConfig.JC_MEDIA_STREAMID, new Values((Object) generate(mediaSchema,MEDIAList),MEDIAList.size())); - MEDIAList.clear(); + if (!(mediaList.isEmpty())) { + basicOutputCollector.emit(StreamAggregateConfig.JC_MEDIA_STREAMID, new Values(generate(mediaSchema,mediaList),mediaList.size())); + mediaList.clear(); } - if(!(MAILList.isEmpty())) { - basicOutputCollector.emit(StreamAggregateConfig.JC_MAIL_STREAMID, new Values((Object) generate(mailSchema,MAILList),MAILList.size())); - MAILList.clear(); + if(!(mailList.isEmpty())) { + basicOutputCollector.emit(StreamAggregateConfig.JC_MAIL_STREAMID, new Values(generate(mailSchema,mailList),mailList.size())); + mailList.clear(); } - if (!VPNList.isEmpty()){ - basicOutputCollector.emit(StreamAggregateConfig.JC_VPN_STREAMID, new Values((Object) generate(vpnSchema, VPNList),VPNList.size())); - VPNList.clear(); + if (!vpnList.isEmpty()){ + basicOutputCollector.emit(StreamAggregateConfig.JC_VPN_STREAMID, new Values(generate(vpnSchema, vpnList),vpnList.size())); + vpnList.clear(); } basicOutputCollector.emit(StreamAggregateConfig.JC_COUNT_STREAMID, new Values(count,loadfail)); }catch (Exception e) { logger.error("----------批数据发送异常-----------" +e); } - count = 0; - loadfail = 0; + count = 0L; + loadfail = 0L; } else { try { - String line = tuple.getString(0); + line = tuple.getString(0); sessionRecordLog = JSONObject.parseObject(line, SessionRecordLog.class); UserRegion userRegion = JSONObject.parseObject(sessionRecordLog.getCommon_user_region(), UserRegion.class); Traffic_Forward traffic_forward = JSONObject.parseObject(userRegion.getTraffic_forward(), Traffic_Forward.class); @@ -153,28 +155,27 @@ public class JCParseBolt extends BaseBasicBolt { sessionRecordLog.setPzlx(traffic_forward.getPzlx()); sessionRecordLog.setGklb(traffic_forward.getGklb()); sessionRecordLog.setLabels(traffic_forward.getLabels()); - sessionRecordLog.setYyzl(traffic_forward.getYyzl()); - sessionRecordLog.setGjc(traffic_forward.getGjc()); - sessionRecordLog.setZsmc(traffic_forward.getZsmc()); - sessionRecordLog.setYjxy(traffic_forward.getYjxy()); - sessionRecordLog.setProtocol(userRegion.getProtocol()); +//弃用 +// sessionRecordLog.setYyzl(traffic_forward.getYyzl()); +// sessionRecordLog.setGjc(traffic_forward.getGjc()); +// sessionRecordLog.setZsmc(traffic_forward.getZsmc()); +// sessionRecordLog.setYjxy(traffic_forward.getYjxy()); +// sessionRecordLog.setProtocol(userRegion.getProtocol()); + int pzlx = traffic_forward.getPzlx(); - if (sessionRecordLog.getHttp_response_body_key()!=null) { - System.out.println("--------------" + sessionRecordLog.getHttp_response_body_key() + "--------------"); - } - //按pzlx的值对数据进行分类发送 + //按pzlx的值对数据进行分类发送 switch (pzlx) { case 501: //IP监测 - count++; try { + count ++; GenericRecord ip_jcrz = new GenericData.Record(ipSchema); sessionRecordLog.setCommon_schema_type("IP"); setIP_JCRZ(ip_jcrz, sessionRecordLog); - IPList.add(ip_jcrz); - if (IPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.JC_IP_STREAMID, new Values((Object) generate(ipSchema, IPList),IPList.size())); - IPList.clear(); + ipList.add(ip_jcrz); + if (ipList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.JC_IP_STREAMID, new Values(generate(ipSchema, ipList),ipList.size())); + ipList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -183,19 +184,19 @@ public class JCParseBolt extends BaseBasicBolt { } break; case 502: //DNS监测 - count++; try { + count++; GenericRecord dns_jczb = new GenericData.Record(dnsSchema); sessionRecordLog.setCommon_schema_type("DNS"); setDNS_JCZB(dns_jczb, sessionRecordLog); if (dns_jczb.get("DNS_NAME")!=("")){ - DNSList.add(dns_jczb); + dnsList.add(dns_jczb); }else { loadfail++; } - if (DNSList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.JC_DNS_STREAMID, new Values((Object) generate(dnsSchema, DNSList),DNSList.size())); - DNSList.clear(); + if (dnsList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.JC_DNS_STREAMID, new Values(generate(dnsSchema, dnsList),dnsList.size())); + dnsList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -205,22 +206,23 @@ public class JCParseBolt extends BaseBasicBolt { break; case 503: //URL监测 case 504: //网站监测 - count++; try { + count++; + sessionRecordLog.setYyzl(traffic_forward.getYyzl()); if (sessionRecordLog.getHttp_response_body()!=null){ String[] split = sessionRecordLog.getHttp_response_body().split("/"); - bucketName = split[split.length-2]; - filename = split[split.length-1]; - basicOutputCollector.emit("jc-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_HTTP_JCZB)); - sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_HTTP_JCZB, filename)); + hosBucketName = split[split.length-2]; + fileName = split[split.length-1]; + basicOutputCollector.emit("jc-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_HTTP_JCZB)); + sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_HTTP_JCZB, fileName)); } GenericRecord http_jczb = new GenericData.Record(httpSchema); sessionRecordLog.setCommon_schema_type("HTTP"); setHTTP_JCZB(http_jczb, sessionRecordLog); - HTTPList.add(http_jczb); - if (HTTPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.JC_HTTP_STREAMID, new Values((Object) generate(httpSchema, HTTPList),HTTPList.size())); - HTTPList.clear(); + httpList.add(http_jczb); + if (httpList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.JC_HTTP_STREAMID, new Values(generate(httpSchema, httpList),httpList.size())); + httpList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -229,15 +231,16 @@ public class JCParseBolt extends BaseBasicBolt { } break; case 505: //特定证书监测 - count++; try { + count++; + sessionRecordLog.setZsmc(traffic_forward.getZsmc()); GenericRecord ssl_jczb = new GenericData.Record(sslSchema); sessionRecordLog.setCommon_schema_type("SSL"); setSSL_JCZB(ssl_jczb, sessionRecordLog); - SSLList.add(ssl_jczb); - if (SSLList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.JC_SSL_STREAMID, new Values((Object) generate(sslSchema, SSLList),SSLList.size())); - SSLList.clear(); + sslList.add(ssl_jczb); + if (sslList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.JC_SSL_STREAMID, new Values(generate(sslSchema, sslList),sslList.size())); + sslList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -247,22 +250,24 @@ public class JCParseBolt extends BaseBasicBolt { break; case 506: //网站关键词监测 case 509: //搜索词监测 - count++; try { + count++; + sessionRecordLog.setYyzl(traffic_forward.getYyzl()); + sessionRecordLog.setGjc(traffic_forward.getGjc()); if (sessionRecordLog.getHttp_response_body()!=null){ String[] split = sessionRecordLog.getHttp_response_body().split("/"); - bucketName = split[split.length-2]; - filename = split[split.length-1]; - basicOutputCollector.emit("jc-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_WYGJC_JCRZ)); - sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_WYGJC_JCRZ, filename)); + hosBucketName = split[split.length-2]; + fileName = split[split.length-1]; + basicOutputCollector.emit("jc-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_WYGJC_JCRZ)); + sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_WYGJC_JCRZ, fileName)); } GenericRecord wygjc_jczb = new GenericData.Record(wygjcSchema); sessionRecordLog.setCommon_schema_type("HTTP"); setWYGJC_JCZB(wygjc_jczb, sessionRecordLog); - WYGJCList.add(wygjc_jczb); - if (WYGJCList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.JC_WYGJC_STREAMID, new Values((Object) generate(wygjcSchema,WYGJCList),WYGJCList.size())); - WYGJCList.clear(); + wygjcList.add(wygjc_jczb); + if (wygjcList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.JC_WYGJC_STREAMID, new Values(generate(wygjcSchema,wygjcList),wygjcList.size())); + wygjcList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -272,22 +277,25 @@ public class JCParseBolt extends BaseBasicBolt { break; case 507: //邮件监测 case 510: //邮件监测 - count++; try { + count++; + sessionRecordLog.setYyzl(traffic_forward.getYyzl()); + sessionRecordLog.setGjc(traffic_forward.getGjc()); + sessionRecordLog.setYjxy(traffic_forward.getYjxy()); if (sessionRecordLog.getHttp_response_body()!=null){ String[] split = sessionRecordLog.getHttp_response_body().split("/"); - bucketName = split[split.length-2]; - filename = split[split.length-1]; - basicOutputCollector.emit("jc-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_MAIL_JCZB)); - sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MAIL_JCZB, filename)); + hosBucketName = split[split.length-2]; + fileName = split[split.length-1]; + basicOutputCollector.emit("jc-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_MAIL_JCZB)); + sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MAIL_JCZB, fileName)); } GenericRecord mail_jczb = new GenericData.Record(mailSchema); sessionRecordLog.setCommon_schema_type("MAIL"); setMAIL_JCZB(mail_jczb, sessionRecordLog); - MAILList.add(mail_jczb); - if (MAILList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.JC_MAIL_STREAMID, new Values((Object) generate(mailSchema,MAILList),MAILList.size())); - MAILList.clear(); + mailList.add(mail_jczb); + if (mailList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.JC_MAIL_STREAMID, new Values(generate(mailSchema,mailList),mailList.size())); + mailList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -296,15 +304,17 @@ public class JCParseBolt extends BaseBasicBolt { } break; case 508: //FTP关键词监测 - count++; try { + count++; + sessionRecordLog.setYyzl(traffic_forward.getYyzl()); + sessionRecordLog.setGjc(traffic_forward.getGjc()); GenericRecord ftp_jczb = new GenericData.Record(ftpSchema); sessionRecordLog.setCommon_schema_type("FTP"); setFTP_JCZB(ftp_jczb,sessionRecordLog); - FTPList.add(ftp_jczb); - if (MAILList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.JC_FTP_STREAMID, new Values((Object) generate(ftpSchema,MAILList),FTPList.size())); - FTPList.clear(); + ftpList.add(ftp_jczb); + if (mailList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.JC_FTP_STREAMID, new Values(generate(ftpSchema,mailList),ftpList.size())); + ftpList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -313,18 +323,21 @@ public class JCParseBolt extends BaseBasicBolt { } break; case 511: //VPN监测 - count++; try { + count++; GenericRecord vpn_jczb = new GenericData.Record(vpnSchema); sessionRecordLog.setCommon_schema_type("VPN"); + sessionRecordLog.setLx(traffic_forward.getLx()); setVPN_JCZB(vpn_jczb, sessionRecordLog); if(StringUtil.isNotEmpty(vpn_jczb.get("DETAIL_ID"))) { - VPNList.add(vpn_jczb); + vpnList.add(vpn_jczb); + }else { + logger.error("vpnjc流量DETAIL_ID为空具体日志为:"+line); + loadfail++; } - VPNList.add(vpn_jczb); - if (VPNList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.JC_VPN_STREAMID, new Values((Object) generate(vpnSchema,VPNList),VPNList.size())); - VPNList.clear(); + if (vpnList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.JC_VPN_STREAMID, new Values(generate(vpnSchema,vpnList),vpnList.size())); + vpnList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -334,15 +347,18 @@ public class JCParseBolt extends BaseBasicBolt { break; case 512: //特定应用监测 case 513: //社交应用监测 - count++; try { + count++; + sessionRecordLog.setYyzl(traffic_forward.getYyzl()); + sessionRecordLog.setProtocol(userRegion.getProtocol()); GenericRecord jstx_jczb = new GenericData.Record(jstxSchema); + sessionRecordLog.setProtocol(userRegion.getProtocol()); sessionRecordLog.setCommon_schema_type("JSTX"); setJSTX_JCZB(jstx_jczb, sessionRecordLog); - JSTXList.add(jstx_jczb); - if (JSTXList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.JC_JSTX_STREAMID, new Values((Object) generate(jstxSchema,JSTXList),JSTXList.size())); - JSTXList.clear(); + jstxList.add(jstx_jczb); + if (jstxList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.JC_JSTX_STREAMID, new Values(generate(jstxSchema,jstxList),jstxList.size())); + jstxList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -351,23 +367,23 @@ public class JCParseBolt extends BaseBasicBolt { } break; case 514: //图片监测 - count++; try { + count++; + sessionRecordLog.setYyzl(traffic_forward.getYyzl()); GenericRecord pic_jczb = new GenericData.Record(picSchema); - if (sessionRecordLog.getHttp_response_body()!=null){ String[] split = sessionRecordLog.getHttp_response_body().split("/"); - bucketName = split[split.length-2]; - filename = split[split.length-1]; - basicOutputCollector.emit("jc-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_PIC_JCZB)); - sessionRecordLog.setHttp_response_body(getYL5( StreamAggregateConfig.T_PIC_JCZB,filename)); + hosBucketName = split[split.length-2]; + fileName = split[split.length-1]; + basicOutputCollector.emit("jc-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_PIC_JCZB)); + sessionRecordLog.setHttp_response_body(getYL5( StreamAggregateConfig.T_PIC_JCZB,fileName)); } sessionRecordLog.setCommon_schema_type("HTTP"); setPIC_JCZB(pic_jczb, sessionRecordLog); - PICList.add(pic_jczb); - if (PICList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.JC_PIC_STREAMID, new Values((Object) generate(picSchema,PICList),PICList.size())); - PICList.clear(); + picList.add(pic_jczb); + if (picList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.JC_PIC_STREAMID, new Values(generate(picSchema,picList),picList.size())); + picList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -376,22 +392,23 @@ public class JCParseBolt extends BaseBasicBolt { } break; case 515: //多媒体监测 - count++; try { + count++; + sessionRecordLog.setYyzl(traffic_forward.getYyzl()); if (sessionRecordLog.getHttp_response_body()!=null){ String[] split = sessionRecordLog.getHttp_response_body().split("/"); - bucketName = split[split.length-2]; - filename = split[split.length-1]; - basicOutputCollector.emit("jc-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_MEDIA_JCZB)); - sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MEDIA_JCZB, filename)); + hosBucketName = split[split.length-2]; + fileName = split[split.length-1]; + basicOutputCollector.emit("jc-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_MEDIA_JCZB)); + sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MEDIA_JCZB, fileName)); } GenericRecord media_jczb = new GenericData.Record(mediaSchema); sessionRecordLog.setCommon_schema_type("HTTP"); setMEDIA_JCZB(media_jczb, sessionRecordLog); - MEDIAList.add(media_jczb); - if (MEDIAList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { - basicOutputCollector.emit(StreamAggregateConfig.JC_MEDIA_STREAMID, new Values((Object) generate(mediaSchema,MEDIAList),MEDIAList.size())); - MEDIAList.clear(); + mediaList.add(media_jczb); + if (mediaList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { + basicOutputCollector.emit(StreamAggregateConfig.JC_MEDIA_STREAMID, new Values(generate(mediaSchema,mediaList),mediaList.size())); + mediaList.clear(); } } catch (Exception e) { e.printStackTrace(); @@ -403,7 +420,8 @@ public class JCParseBolt extends BaseBasicBolt { break; } } catch(Exception e){ - logger.error("JC数据处理异常"+e); + logger.error("JC数据处理异常"+e+"异常数据为:"+line); + loadfail++; } } } @@ -422,7 +440,7 @@ public class JCParseBolt extends BaseBasicBolt { outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_PIC_STREAMID,new Fields(StreamAggregateConfig.JC_PIC_STREAMID,"piccount")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_MEDIA_STREAMID,new Fields(StreamAggregateConfig.JC_MEDIA_STREAMID,"mediacount")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_COUNT_STREAMID, new Fields("count","loadfail")); - outputFieldsDeclarer.declareStream("jc-unstructured", new Fields("bucketName","filename","bucketname")); + outputFieldsDeclarer.declareStream("jc-unstructured", new Fields("hosBucketName","fileName","s3BucketName")); } @Override diff --git a/src/main/java/cn/ac/iie/storm/bolt/jc/JCSendBolt.java b/src/main/java/cn/ac/iie/storm/bolt/jc/JCSendBolt.java index 7d908b7..2848f4d 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/jc/JCSendBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/jc/JCSendBolt.java @@ -13,7 +13,6 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.TupleUtils; import java.util.HashMap; -import java.util.List; import java.util.Map; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; diff --git a/src/main/java/cn/ac/iie/storm/bolt/jc/JcUnstructuredSendBolt.java b/src/main/java/cn/ac/iie/storm/bolt/jc/JcUnstructuredSendBolt.java index 51dc34e..a4a4812 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/jc/JcUnstructuredSendBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/jc/JcUnstructuredSendBolt.java @@ -1,6 +1,5 @@ package cn.ac.iie.storm.bolt.jc; -import cn.ac.iie.storm.bolt.conn.ConnUnstructuredSendBolt; import cn.ac.iie.storm.utils.config.StreamAggregateConfig; import org.apache.log4j.Logger; import org.apache.storm.topology.BasicOutputCollector; @@ -9,32 +8,30 @@ import org.apache.storm.topology.base.BaseBasicBolt; import org.apache.storm.tuple.Tuple; import java.io.IOException; -import java.util.List; import static cn.ac.iie.storm.utils.hos.HosService.httpPostObject; import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; public class JcUnstructuredSendBolt extends BaseBasicBolt { - String bucketName = null; - String filename = null; - String bucketname = null; - private final static Logger logger = Logger.getLogger(JcUnstructuredSendBolt.class); + + String hosBucketName = null; + String fileName = null; + String s3BucketName = null; @Override public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { try { - bucketName = tuple.getStringByField("bucketName"); - filename = tuple.getStringByField("filename"); - bucketname = tuple.getStringByField("bucketname"); - byte[] bytes = httpPostObject(bucketName, filename); + hosBucketName = tuple.getStringByField("hosBucketName"); + fileName = tuple.getStringByField("fileName"); + s3BucketName = tuple.getStringByField("s3BucketName"); + byte[] bytes = httpPostObject(hosBucketName, fileName); int connAnsyHttpPost =0; String s3Url; if(bytes!=null) { int i = 0; - String ip; while (i < StreamAggregateConfig.S3_SENT_COUNT) { - s3Url = getS3Url(bucketname, filename ); + s3Url = getS3Url(s3BucketName, fileName ); connAnsyHttpPost = getConnAnsyHttpPost(s3Url, bytes); if (connAnsyHttpPost == 200) { break; @@ -42,11 +39,11 @@ public class JcUnstructuredSendBolt extends BaseBasicBolt { i++; } if (connAnsyHttpPost != 200) { - logger.error("桶：" + bucketName + "下的文件" + filename + "发送失败"); + logger.error("桶：" + hosBucketName + "下的文件" + fileName + "发送失败"); } } } catch (IOException e) { - logger.error("桶："+bucketName+"下的文件"+filename+"发送失败原因为："+e); + logger.error("桶："+hosBucketName+"下的文件"+fileName+"发送失败原因为："+e); } } @Override diff --git a/src/main/java/cn/ac/iie/storm/spout/FloodingKafkaSpout.java b/src/main/java/cn/ac/iie/storm/spout/FloodingKafkaSpout.java deleted file mode 100644 index 25a77b5..0000000 --- a/src/main/java/cn/ac/iie/storm/spout/FloodingKafkaSpout.java +++ /dev/null @@ -1,74 +0,0 @@ -package cn.ac.iie.storm.spout; - -import cn.ac.iie.storm.utils.config.StreamAggregateConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.log4j.Logger; -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichSpout; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Values; - -import java.util.Collections; -import java.util.Map; -import java.util.Properties; - -/** - * kafkaSpout - */ -public class FloodingKafkaSpout extends BaseRichSpout { - private static final long serialVersionUID = -3363788553406229592L; - private KafkaConsumer consumer; - private SpoutOutputCollector collector = null; - private TopologyContext context = null; - private final static Logger logger = Logger.getLogger(FloodingKafkaSpout.class); - - private static Properties createConsumerConfig() { - Properties props = new Properties(); - props.put("bootstrap.servers", StreamAggregateConfig.BOOTSTRAP_SERVERS); - props.put("group.id", StreamAggregateConfig.GROUP_ID); - props.put("session.timeout.ms", "60000"); - props.put("max.poll.records", 3000); - props.put("max.partition.fetch.bytes", 31457280); - props.put("auto.offset.reset", StreamAggregateConfig.AUTO_OFFSET_RESET); - props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); - props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); - logger.info("链接kafka"+StreamAggregateConfig.BOOTSTRAP_SERVERS+"成功"); - return props; - } - - @Override - public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) { - this.collector=spoutOutputCollector; - this.context=topologyContext; - Properties prop = createConsumerConfig(); - this.consumer = new KafkaConsumer<>(prop); - this.consumer.subscribe(Collections.singletonList(StreamAggregateConfig.KAFKA_TOPIC)); - } - @Override - public void close() { - consumer.close(); - } - @Override - public void nextTuple() { - try { - ConsumerRecords records = consumer.poll(StreamAggregateConfig.KAFKA_COMSUMER_POLL); - Thread.sleep(StreamAggregateConfig.TOPOLOGY_SPOUT_SLEEP_TIME); - for (ConsumerRecord record : records) { - this.collector.emit(new Values(record.value())); - } - } catch (Exception e) { - logger.error("KafkaSpout发送消息出现异常!", e); - e.printStackTrace(); - } - } - - - @Override - public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { - outputFieldsDeclarer.declare(new Fields("flooding-source")); - } -} diff --git a/src/main/java/cn/ac/iie/storm/spout/SafeKafkaSpout.java b/src/main/java/cn/ac/iie/storm/spout/SafeKafkaSpout.java index a516b9c..1b8f8ca 100644 --- a/src/main/java/cn/ac/iie/storm/spout/SafeKafkaSpout.java +++ b/src/main/java/cn/ac/iie/storm/spout/SafeKafkaSpout.java @@ -49,6 +49,8 @@ public class SafeKafkaSpout extends BaseRichSpout { try { ConsumerRecords records = consumer.poll(StreamAggregateConfig.KAFKA_COMSUMER_POLL); Thread.sleep(StreamAggregateConfig.TOPOLOGY_SPOUT_SLEEP_TIME); + + for (ConsumerRecord record : records) { this.collector.emit(new Values(record.value())); } diff --git a/src/main/java/cn/ac/iie/storm/topology/StreamAggregateTopology.java b/src/main/java/cn/ac/iie/storm/topology/StreamAggregateTopology.java index a12ba72..c452cea 100644 --- a/src/main/java/cn/ac/iie/storm/topology/StreamAggregateTopology.java +++ b/src/main/java/cn/ac/iie/storm/topology/StreamAggregateTopology.java @@ -8,8 +8,9 @@ import cn.ac.iie.storm.bolt.index.ConnCountBolt; import cn.ac.iie.storm.bolt.index.GkCountBolt; import cn.ac.iie.storm.bolt.index.JCCountBolt; import cn.ac.iie.storm.bolt.jc.*; -import cn.ac.iie.storm.spout.FloodingKafkaSpout; +import cn.ac.iie.storm.spout.ConnectionKafkaSpout; import cn.ac.iie.storm.spout.SafeKafkaSpout; +import cn.ac.iie.storm.spout.TransactionKafkaSpout; import cn.ac.iie.storm.utils.config.StreamAggregateConfig; import org.apache.log4j.Logger; import org.apache.storm.Config; @@ -61,7 +62,7 @@ public class StreamAggregateTopology { //设置拓扑关系(Bolt) --切分行 ---分发bolt switch (StreamAggregateConfig.TOPOLOGY_LOGO) { case "conn": - topologyBuilder.setSpout(StreamAggregateConfig.KAFKA_SPOUT, new FloodingKafkaSpout(), StreamAggregateConfig.SPOUT_PARALLELISM); + topologyBuilder.setSpout(StreamAggregateConfig.KAFKA_SPOUT, new ConnectionKafkaSpout(), StreamAggregateConfig.SPOUT_PARALLELISM); topologyBuilder.setBolt(StreamAggregateConfig.CONN_PARSE_COMPONENTID, new ConnParseBolt(), StreamAggregateConfig.PARSE_PARALLELISM) .shuffleGrouping(StreamAggregateConfig.KAFKA_SPOUT); @@ -148,6 +149,44 @@ public class StreamAggregateTopology { .shuffleGrouping("JC-HTTP-SEND") .shuffleGrouping("JC-SSL-SEND"); break; + case "conn1": + topologyBuilder.setSpout(StreamAggregateConfig.KAFKA_SPOUT, new ConnectionKafkaSpout(), StreamAggregateConfig.SPOUT_PARALLELISM); + topologyBuilder.setSpout(StreamAggregateConfig.TRANSACTION_SPOUT, new TransactionKafkaSpout(), StreamAggregateConfig.SPOUT_PARALLELISM); +// topologyBuilder.setSpout(StreamAggregateConfig.SESSION_SPOUT, new TransactionKafkaSpout(), StreamAggregateConfig.SPOUT_PARALLELISM); + + + + topologyBuilder.setBolt(StreamAggregateConfig.CONN_PARSE_COMPONENTID, new ConnParseBolt(), StreamAggregateConfig.PARSE_PARALLELISM) + .shuffleGrouping(StreamAggregateConfig.TRANSACTION_SPOUT) + .shuffleGrouping(StreamAggregateConfig.KAFKA_SPOUT); +// .shuffleGrouping(StreamAggregateConfig.SESSION_SPOUT); + + topologyBuilder.setBolt("CONN-SEND", new ConnSendBolt(),StreamAggregateConfig.SEND_PARALLELISM) + .shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_DNS_STREAMID) + .shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_VPN_STREAMID) + .shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_MAIL_STREAMID) + .shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_JSTX_STREAMID) + .shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_FTP_STREAMID); + + topologyBuilder.setBolt("CONN-IP-SEND",new ConnIPSendBolt(),StreamAggregateConfig.IP_SEND_PARALLELISM) + .shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_IP_STREAMID); + + topologyBuilder.setBolt("CONN-HTTP-SEND",new ConnHTTPSendBolt(),StreamAggregateConfig.HTTP_SEND_PARALLELISM) + .shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_HTTP_STREAMID); + + topologyBuilder.setBolt("CONN-SSL-SEND",new ConnSSLSendBolt(),StreamAggregateConfig.SSL_SEND_PARALLELISM) + .shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_SSL_STREAMID); + + topologyBuilder.setBolt("CONN-UNSTRUCTURED-SEND",new ConnUnstructuredSendBolt(),StreamAggregateConfig.UNSTRUCTURED_SEND_PARALLELISM) + .shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,"conn-unstructured"); + + topologyBuilder.setBolt("CONN-COUNT", new ConnCountBolt(),StreamAggregateConfig.COUNT_SEND_PARALLELISM) + .shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_YBInputSize_STREAMID) + .shuffleGrouping("CONN-SEND") + .shuffleGrouping("CONN-IP-SEND") + .shuffleGrouping("CONN-HTTP-SEND") + .shuffleGrouping("CONN-SSL-SEND"); + break; default: break; } diff --git a/src/main/java/cn/ac/iie/storm/utils/combine/httpurl/Http_URL_process.java b/src/main/java/cn/ac/iie/storm/utils/combine/httpurl/Http_URL_process.java index 326350a..c188338 100644 --- a/src/main/java/cn/ac/iie/storm/utils/combine/httpurl/Http_URL_process.java +++ b/src/main/java/cn/ac/iie/storm/utils/combine/httpurl/Http_URL_process.java @@ -2,9 +2,6 @@ package cn.ac.iie.storm.utils.combine.httpurl; import org.apache.log4j.Logger; -import java.net.URL; -import java.util.regex.Matcher; -import java.util.regex.Pattern; public class Http_URL_process { private final static Logger logger = Logger.getLogger(Http_URL_process.class); diff --git a/src/main/java/cn/ac/iie/storm/utils/config/GetMap.java b/src/main/java/cn/ac/iie/storm/utils/config/GetMap.java deleted file mode 100644 index 9f58f38..0000000 --- a/src/main/java/cn/ac/iie/storm/utils/config/GetMap.java +++ /dev/null @@ -1,192 +0,0 @@ -package cn.ac.iie.storm.utils.config; - -import org.apache.logging.log4j.util.PropertiesUtil; - -import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -public class GetMap { - private static Properties propService = new Properties(); - private static Properties propService1 = new Properties(); - - - private static Properties propService_city = new Properties(); - private static Properties propService_province = new Properties(); - private static Properties propService_country = new Properties(); - private static Properties propService_jstx = new Properties(); - - - private static Map labelMap; - private static Map appMap; - private static Map cityMap; - private static Map provinceMap; - private static Map countryMap; - private static Map jstxIdMap; - - - - public static String getJstxDetail_id(int common_app_id){ - String appId = String.valueOf(common_app_id); - if (appMap.get(appId) != null) { - return appMap.get(appId); - } else { - return appId; - } - } - - public static String getProto_id(String common_schema_type){ - return labelMap.get(common_schema_type); - } - - public static String getDy(String location){ - String[] split; - if(location==null){ - return "0"; - } else if (location.contains(",,")){ - split = location.split(",,"); - if (split.length == 2) { - String s = split[0]; - if (s.length()<1) { - s = split[1]; - if (s == null) { - s = "0"; - } - } - return s; - }else { - return "0"; - } - } else { - String loc = location.replace(" ",""); - String str = cityMap.get(loc); - split = loc.split(","); - if ((str == null&&split.length==3)) { - str = provinceMap.get(split[1]); - if (str == null) { - String ss = split[2]; - str = countryMap.get(ss); - if (str == null) { - str = "0"; - } - } - return str; - }else if(str !=null){ - return str; - }else { - return "0"; - } - } - } - - - public static String getFjSzd(String location,String mail_from,String mail_protocol_type) { - if (mail_from != null&& mail_protocol_type.equals("SMTP")) { - String[] split; - if (location == null) { - return String.valueOf(3); - } else if (location.contains(",,")) { - return String.valueOf(1); - } else { - split = location.split(","); - if (split.length == 3) { - String country = split[2].replace(" ", ""); - String s = countryMap.get(country); - if (country.equals("China")) { - return String.valueOf(1); - } else if (s != null) { - return String.valueOf(2); - } else { - return String.valueOf(3); - } - } else { - return String.valueOf(3); - } - } - }else { - return String.valueOf(3); - } - } - - - public static String getSjSzd(String location,String mail_to,String mail_protocol_type) { - if (mail_to != null&&(mail_protocol_type.equals("POP3")||mail_protocol_type.equals("IMAP"))) { - String[] split; - if (location == null) { - return String.valueOf(3); - } else if (location.contains(",,")) { - return String.valueOf(1); - } else { - split = location.split(","); - if (split.length == 3) { - String country = split[2].replace(" ", ""); - String s = countryMap.get(country); - if (country.equals("China")) { - return String.valueOf(1); - } else if (s != null) { - return String.valueOf(2); - } else { - return String.valueOf(3); - } - } else { - return String.valueOf(3); - } - } - }else { - return String.valueOf(3); - } - } - - - - public static String getDns_name(String dns_qname){ - if (dns_qname==null){ - return ""; - }else if (dns_qname.contains(" ")){ - return ""; - }else { - return dns_qname; - } - } - - - static { - try { - InputStream applabel = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_app_label_coding.properties"); - InputStream app = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_app_id_coding.properties"); - InputStream jstxid = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_jstx_id_coding.properties"); - - InputStream cityNumbering = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_city_coding.properties"); - InputStream provinceNumbering = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_province_coding.properties"); - InputStream countryNumbering = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_country_coding.properties"); - - - propService1.load(app); - propService.load(applabel); - propService_jstx.load(jstxid); - - propService_city.load(cityNumbering); - propService_province.load(provinceNumbering); - propService_country.load(countryNumbering); - - - appMap = new HashMap((Map)propService1); - labelMap = new HashMap((Map) propService); - jstxIdMap = new HashMap((Map) propService_jstx); - cityMap = new HashMap((Map) propService_city); - provinceMap = new HashMap((Map) propService_province); - countryMap = new HashMap((Map) propService_country); - } catch (Exception e) { -// propCommon = null; - propService = null; - propService1 = null; - propService_jstx = null; - propService_city = null; - propService_province = null; - propService_country = null; - } - } - - -} diff --git a/src/main/java/cn/ac/iie/storm/utils/config/StreamAggregateConfig.java b/src/main/java/cn/ac/iie/storm/utils/config/StreamAggregateConfig.java index 241f4b2..c75fea5 100644 --- a/src/main/java/cn/ac/iie/storm/utils/config/StreamAggregateConfig.java +++ b/src/main/java/cn/ac/iie/storm/utils/config/StreamAggregateConfig.java @@ -10,6 +10,10 @@ public class StreamAggregateConfig { public static final String TOPOLOGY_LOGO = StreamAggregateConfigurations.getStringProperty(0, "topology.logo"); + /** + * yys字段 + */ + public static final Integer YYS_NUM = StreamAggregateConfigurations.getIntProperty(0,"yys_num"); /** * System @@ -30,6 +34,9 @@ public class StreamAggregateConfig { public static final String BOOTSTRAP_SERVERS = StreamAggregateConfigurations.getStringProperty(0, "bootstrap.servers"); public static final String GROUP_ID = StreamAggregateConfigurations.getStringProperty(0, "group.id"); + + public static final String KAFKA_TOPIC_SESSION = StreamAggregateConfigurations.getStringProperty(0, "kafka.topic.session"); + public static final String KAFKA_TOPIC_TRANSACTION = StreamAggregateConfigurations.getStringProperty(0, "kafka.topic.transaction"); public static final String KAFKA_TOPIC = StreamAggregateConfigurations.getStringProperty(0, "kafka.topic"); public static final String KAFKA_SAFE_TOPIC = StreamAggregateConfigurations.getStringProperty(0, "kafka.safe.topic"); public static final String AUTO_OFFSET_RESET = StreamAggregateConfigurations.getStringProperty(0, "auto.offset.reset"); @@ -43,6 +50,12 @@ public class StreamAggregateConfig { //conn-ComponentId-spout public static final String KAFKA_SPOUT = StreamAggregateConfigurations.getStringProperty(0, "kafka.spout"); + public static final String TRANSACTION_SPOUT = StreamAggregateConfigurations.getStringProperty(0, "transaction.spout"); + public static final String SESSION_SPOUT = StreamAggregateConfigurations.getStringProperty(0, "session.spout"); + + + + //#conn-bolt-parallelism_hint public static final int PARSE_PARALLELISM = StreamAggregateConfigurations.getIntProperty(0,"parse.parallelism"); public static final int SEND_PARALLELISM = StreamAggregateConfigurations.getIntProperty(0,"send.parallelism"); diff --git a/src/main/java/cn/ac/iie/storm/utils/general/CONNAssignment.java b/src/main/java/cn/ac/iie/storm/utils/general/CONNAssignment.java index 7df3c60..4f888af 100644 --- a/src/main/java/cn/ac/iie/storm/utils/general/CONNAssignment.java +++ b/src/main/java/cn/ac/iie/storm/utils/general/CONNAssignment.java @@ -1,7 +1,6 @@ package cn.ac.iie.storm.utils.general; import cn.ac.iie.storm.bean.SessionRecordLog; -import cn.ac.iie.storm.utils.config.GetMap; import org.apache.avro.generic.GenericRecord; import org.apache.log4j.Logger; @@ -10,7 +9,7 @@ import java.io.IOException; import static cn.ac.iie.storm.utils.combine.httpurl.Http_URL_process.getFile_Name; -import static cn.ac.iie.storm.utils.config.GetMap.*; +import static cn.ac.iie.storm.utils.general.GetMap.*; import static cn.ac.iie.storm.utils.general.MailTrans.*; import static cn.ac.iie.storm.utils.general.TransFormUtils.*; @@ -19,8 +18,8 @@ public class CONNAssignment { private final static Logger logger = Logger.getLogger(CONNAssignment.class); public static void setDNS_RZZB(GenericRecord dns_rzzb, SessionRecordLog sessionRecordLog) throws IOException { dns_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); - dns_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); - dns_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); + dns_rzzb.put("YDY",sessionRecordLog.getYdy()); + dns_rzzb.put("MDDY",sessionRecordLog.getMddy()); dns_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); dns_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); dns_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); @@ -29,7 +28,7 @@ public class CONNAssignment { dns_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); dns_rzzb.put("DETAIL_ID","02999"); dns_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); - dns_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); + dns_rzzb.put("YYS",sessionRecordLog.getYys()); dns_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); //dns特有 dns_rzzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num()); @@ -42,8 +41,8 @@ public class CONNAssignment { public static void setIP_RZZB(GenericRecord IP_rzzb, SessionRecordLog sessionRecordLog) throws IOException { IP_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); - IP_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); - IP_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); + IP_rzzb.put("YDY",sessionRecordLog.getYdy()); + IP_rzzb.put("MDDY",sessionRecordLog.getMddy()); IP_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); IP_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); IP_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); @@ -52,7 +51,7 @@ public class CONNAssignment { IP_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); IP_rzzb.put("DETAIL_ID","08003"); IP_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); - IP_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); + IP_rzzb.put("YYS",sessionRecordLog.getYys()); IP_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); //dns特有 IP_rzzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num()); @@ -64,8 +63,8 @@ public class CONNAssignment { public static void setHTTP_RZZB(GenericRecord http_rzzb, SessionRecordLog sessionRecordLog) throws IOException { http_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); - http_rzzb.put("YDY", getDy(sessionRecordLog.getCommon_client_location())); - http_rzzb.put("MDDY", getDy(sessionRecordLog.getCommon_server_location())); + http_rzzb.put("YDY",sessionRecordLog.getYdy()); + http_rzzb.put("MDDY", sessionRecordLog.getMddy()); http_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); http_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(), sessionRecordLog.getCommon_client_ip())); http_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(), sessionRecordLog.getCommon_server_ip())); @@ -74,7 +73,7 @@ public class CONNAssignment { http_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); http_rzzb.put("DETAIL_ID","01999"); http_rzzb.put("PROTOCOL", getProtocol(sessionRecordLog.getCommon_l4_protocol())); - http_rzzb.put("YYS", getYys(sessionRecordLog.getCommon_data_center())); + http_rzzb.put("YYS", sessionRecordLog.getYys()); http_rzzb.put("YL1", sessionRecordLog.getCommon_address_type()); http_rzzb.put("YL5",sessionRecordLog.getHttp_response_body()); //http特有 @@ -92,8 +91,8 @@ public class CONNAssignment { } public static void setVPN_RZZB(GenericRecord vpn_rzzb, SessionRecordLog sessionRecordLog){ vpn_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); - vpn_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); - vpn_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); + vpn_rzzb.put("YDY",sessionRecordLog.getYdy()); + vpn_rzzb.put("MDDY",sessionRecordLog.getMddy()); vpn_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); vpn_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); vpn_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); @@ -102,7 +101,7 @@ public class CONNAssignment { vpn_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); vpn_rzzb.put("DETAIL_ID",getVpnDetail_id(sessionRecordLog.getCommon_l7_protocol(),sessionRecordLog.getCommon_protocol_label())); vpn_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); - vpn_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); + vpn_rzzb.put("YYS",sessionRecordLog.getYys()); vpn_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); //vpn特有 vpn_rzzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num()); @@ -113,8 +112,8 @@ public class CONNAssignment { public static void setSSL_RZZB(GenericRecord ssl_rzzb, SessionRecordLog sessionRecordLog) throws IOException { ssl_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); - ssl_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); - ssl_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); + ssl_rzzb.put("YDY",sessionRecordLog.getYdy()); + ssl_rzzb.put("MDDY",sessionRecordLog.getMddy()); ssl_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); ssl_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); ssl_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); @@ -123,7 +122,7 @@ public class CONNAssignment { ssl_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); ssl_rzzb.put("DETAIL_ID","07999"); //同上 ssl_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); - ssl_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); + ssl_rzzb.put("YYS",sessionRecordLog.getYys()); ssl_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); //ssl特有 @@ -131,14 +130,13 @@ public class CONNAssignment { ssl_rzzb.put("CS_BYTES",sessionRecordLog.getCommon_c2s_byte_num()); ssl_rzzb.put("SC_PKTS",sessionRecordLog.getCommon_s2c_pkt_num()); ssl_rzzb.put("SC_BYTES",sessionRecordLog.getCommon_s2c_byte_num()); - ssl_rzzb.put("FWQMC",sessionRecordLog.getSsl_sni()); } public static void setFTP_RZZB(GenericRecord ftp_rzzb, SessionRecordLog sessionRecordLog) throws IOException { ftp_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); - ftp_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); - ftp_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); + ftp_rzzb.put("YDY",sessionRecordLog.getYdy()); + ftp_rzzb.put("MDDY",sessionRecordLog.getMddy()); ftp_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); ftp_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); ftp_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); @@ -147,7 +145,7 @@ public class CONNAssignment { ftp_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); ftp_rzzb.put("DETAIL_ID","06999"); //同上 ftp_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); - ftp_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); + ftp_rzzb.put("YYS",sessionRecordLog.getYys()); ftp_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); //ftp特有 @@ -161,8 +159,8 @@ public class CONNAssignment { public static void setMAIL_RZZB(GenericRecord mail_rzzb, SessionRecordLog sessionRecordLog) throws IOException { mail_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); - mail_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); - mail_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); + mail_rzzb.put("YDY",sessionRecordLog.getYdy()); + mail_rzzb.put("MDDY",sessionRecordLog.getMddy()); mail_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); mail_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); mail_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); @@ -171,7 +169,7 @@ public class CONNAssignment { mail_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); mail_rzzb.put("DETAIL_ID",getMailDetail_id(sessionRecordLog.getMail_protocol_type())); mail_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); - mail_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); + mail_rzzb.put("YYS",sessionRecordLog.getYys()); mail_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); mail_rzzb.put("YL5",sessionRecordLog.getHttp_response_body()); //根据mail_bcc填充 @@ -203,17 +201,17 @@ public class CONNAssignment { public static void setJSTX_RZZB(GenericRecord jstx_rzzb, SessionRecordLog sessionRecordLog) throws IOException { jstx_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); - jstx_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); - jstx_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); + jstx_rzzb.put("YDY",sessionRecordLog.getYdy()); + jstx_rzzb.put("MDDY",sessionRecordLog.getMddy()); jstx_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); jstx_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); jstx_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); jstx_rzzb.put("SPORT",sessionRecordLog.getCommon_client_port()); jstx_rzzb.put("DPORT",sessionRecordLog.getCommon_server_port()); jstx_rzzb.put("PROTO_ID", GetMap.getProto_id(sessionRecordLog.getCommon_schema_type())); - jstx_rzzb.put("DETAIL_ID", GetMap.getJstxDetail_id(sessionRecordLog.getCommon_app_id())); + jstx_rzzb.put("DETAIL_ID",sessionRecordLog.getCommon_app_label()); jstx_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); - jstx_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); + jstx_rzzb.put("YYS",sessionRecordLog.getYys()); jstx_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); //ftp特有 jstx_rzzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num()); diff --git a/src/main/java/cn/ac/iie/storm/utils/general/GKAssignment.java b/src/main/java/cn/ac/iie/storm/utils/general/GKAssignment.java index a113f8b..c73a073 100644 --- a/src/main/java/cn/ac/iie/storm/utils/general/GKAssignment.java +++ b/src/main/java/cn/ac/iie/storm/utils/general/GKAssignment.java @@ -1,13 +1,12 @@ package cn.ac.iie.storm.utils.general; import cn.ac.iie.storm.bean.SessionRecordLog; -import cn.ac.iie.storm.utils.config.GetMap; import org.apache.avro.generic.GenericRecord; import java.io.IOException; import static cn.ac.iie.storm.utils.combine.httpurl.Http_URL_process.getFile_Name; -import static cn.ac.iie.storm.utils.config.GetMap.*; +import static cn.ac.iie.storm.utils.general.GetMap.*; import static cn.ac.iie.storm.utils.general.MailTrans.*; import static cn.ac.iie.storm.utils.general.TransFormUtils.*; import static cn.ac.iie.storm.utils.general.TransFormUtils.getYys; @@ -259,7 +258,8 @@ public class GKAssignment { vpn_gkzb.put("DPORT",sessionRecordLog.getCommon_server_port()); vpn_gkzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); ///--转化表 还没弄 - vpn_gkzb.put("DETAIL_ID",getVpnDetail_id(sessionRecordLog.getCommon_l7_protocol(),sessionRecordLog.getCommon_protocol_label())); //同上 + // vpn_gkzb.put("DETAIL_ID",getVpnDetail_id(sessionRecordLog.getCommon_l7_protocol(),sessionRecordLog.getCommon_protocol_label())); //同上 + vpn_gkzb.put("DETAIL_ID",sessionRecordLog.getLx()); vpn_gkzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); //-common_l4_protocol其实是这个字段 vpn_gkzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); //-**common_isp这个字段 vpn_gkzb.put("YL1",sessionRecordLog.getCommon_address_type()); diff --git a/src/main/java/cn/ac/iie/storm/utils/general/JCAssignment.java b/src/main/java/cn/ac/iie/storm/utils/general/JCAssignment.java index c719cfc..0dbc72f 100644 --- a/src/main/java/cn/ac/iie/storm/utils/general/JCAssignment.java +++ b/src/main/java/cn/ac/iie/storm/utils/general/JCAssignment.java @@ -1,14 +1,13 @@ package cn.ac.iie.storm.utils.general; import cn.ac.iie.storm.bean.SessionRecordLog; -import cn.ac.iie.storm.utils.config.GetMap; import org.apache.avro.generic.GenericRecord; import java.io.IOException; import static cn.ac.iie.storm.utils.combine.httpurl.Http_URL_process.getFile_Name; -import static cn.ac.iie.storm.utils.config.GetMap.*; +import static cn.ac.iie.storm.utils.general.GetMap.*; import static cn.ac.iie.storm.utils.general.MailTrans.*; import static cn.ac.iie.storm.utils.general.TransFormUtils.*; @@ -284,7 +283,8 @@ public class JCAssignment { vpn_jczb.put("SPORT",sessionRecordLog.getCommon_client_port()); vpn_jczb.put("DPORT",sessionRecordLog.getCommon_server_port()); vpn_jczb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); - vpn_jczb.put("DETAIL_ID",getVpnDetail_id(sessionRecordLog.getCommon_l7_protocol(),sessionRecordLog.getCommon_protocol_label())); +// vpn_jczb.put("DETAIL_ID",getVpnDetail_id(sessionRecordLog.getCommon_l7_protocol(),sessionRecordLog.getCommon_protocol_label())); + vpn_jczb.put("DETAIL_ID",sessionRecordLog.getLx()); vpn_jczb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); vpn_jczb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); //-**common_isp这个字段 vpn_jczb.put("YL1",sessionRecordLog.getCommon_address_type()); diff --git a/src/main/java/cn/ac/iie/storm/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/storm/utils/general/TransFormUtils.java index 3cda7cb..5cc4de9 100644 --- a/src/main/java/cn/ac/iie/storm/utils/general/TransFormUtils.java +++ b/src/main/java/cn/ac/iie/storm/utils/general/TransFormUtils.java @@ -203,10 +203,6 @@ public class TransFormUtils { - - - - /** * avro对象的序列化方式 * @param schema diff --git a/src/resources/common_app_id_coding.properties b/src/resources/common_app_id_coding.properties deleted file mode 100644 index 5161ca0..0000000 --- a/src/resources/common_app_id_coding.properties +++ /dev/null @@ -1 +0,0 @@ -#common_app_id_conding HTTP=01999 DNS=02999 WEBMAIL=03001 SMTP=03002 POP3=03003 IMAP=03004 PPTP=04001 L2TP=04002 OpenVPN=04003 IPSec=04004 QQ=05001 BlackLight=05002 钉钉=05003 翼聊=05004 Hangouts=05005 个信=05006 微信电话本=05007 飞聊=05008 环聊=05009 有信电话=05010 友约=05011 遇见=05012 Kik=05013 微爱=05014 Tango=05015 敢聊=05016 微信=05017 易信=05018 旺信=05019 Tumblr=05020 Blued=05021 网聊=05022 快传=05023 唱吧=05024 比邻=05025 千牛=05026 派派=05027 找对象=05028 IM+=05029 微聚=05030 BBM=05031 LOFTER=05032 Weico=05033 米聊=05034 YY语音=05035 Skype=05036 Path=05037 网易泡泡=05038 有你=05039 微乐=05040 GO短信=05041 和通讯录=05042 约爱=05043 无秘=05044 Wicker=05045 约会吧=05046 hike messager=05047 摩擦=05048 探探=05049 陌陌=05050 来往点点虫=05051 百度Hi=05052 ZANK=05053 小恩爱=05054 飞信=05055 ChatON=05056 QT语音=05057 爱聊=05058 nice好赞=05059 朋友印象=05060 Googletalk=05061 友加=05062 Keechat=05063 啪啪啪啪音乐圈=05064 微会=05065 Radicall=05066 槽厂=05067 想恋爱=05068 颜值颜值点评=05069 LinkedIn=05070 Telegram=05071 DiDi=05072 ICQ=05073 Zello=05074 NextDoor=05075 Pinterest=05076 叮咚=05077 床上=05078 Nimbuzz=05079 微人脉=05080 陪陪=05081 微密=05082 Vine=05083 Line=05084 between=05085 facebook=05086 Viber=05087 MeetUP=05088 MeetMe=05089 OkCupidDating=05090 碰碰=05091 Diaspora=05092 咚呱=05093 陪聊=05094 Twitter=05095 Kakao=05096 MSN=05097 Voxer=05098 Airetalk=05099 ooVoo=05100 Mico=05101 Banjo=05102 TextNow=05103 抱抱=05104 多多的鱼=05105 个性=05106 闺蜜圈=05107 Instagram=05108 Whatsapp=05109 Talkbox=05110 ChatOn=05111 Coco=05112 Badoo=05113 Match.com=05114 TextFree=05115 Tapatalk=05116 KC网络电话=05117 爱唱=05118 酷我=05119 荔枝FM=05120 铁血=05121 酷友社区=05122 花田交友=05123 红娘网=05124 知乎=05125 百度贴吧=05126 百合婚恋=05127 珍爱网=05128 猫扑=05129 新浪微博=05130 开心网=05131 天际好好约=05132 天涯社区=05133 人人网=05134 世纪佳缘=05135 We heart it=05136 Avocado=05137 Tinder=05138 Swarm=05139 Snapchat=05140 Dating for everyone=05141 Skout=05142 QQ空间=05143 Couple=05144 Happn=05145 FTP=06999 SSL=07999 IP黑=08001 IP监测=08002 IP日志总表=08003 100100=05001 100200=05002 100300=05003 100400=05004 100500=05005 100600=05006 100700=05007 100800=05008 100900=05009 101000=05010 101100=05011 101200=05012 101300=05013 101400=05014 101500=05015 101600=05016 101700=05017 101800=05018 101900=05019 102000=05020 102100=05021 102200=05022 102300=05023 102400=05024 102500=05025 102600=05026 102700=05027 102800=05028 102900=05029 103000=05030 103100=05031 103200=05032 103300=05033 103400=05034 103500=05035 103600=05036 103700=05037 103800=05038 103900=05039 104000=05040 104100=05041 104200=05042 104300=05043 104400=05044 104500=05045 104600=05046 104700=05047 104800=05048 104900=05049 105000=05050 105100=05051 105200=05052 105300=05053 105400=05054 105500=05055 105600=05056 105700=05057 105800=05058 105900=05059 106000=05060 106100=05061 106200=05062 106300=05063 106400=05064 106500=05065 106600=05066 106700=05067 106800=05068 106900=05069 107000=05070 107100=05071 107200=05072 107300=05073 107400=05074 107500=05075 107600=05076 107700=05077 107800=05078 107900=05079 108000=05080 108100=05081 108200=05082 108300=05083 108400=05084 108500=05085 108600=05086 108700=05087 108800=05088 108900=05089 109000=05090 109100=05091 109200=05092 109300=05093 109400=05094 109500=05095 109600=05096 109700=05097 109800=05098 109900=05099 110000=05100 110100=05101 110200=05102 110300=05103 110400=05104 110500=05105 110600=05106 110700=05107 110800=05108 110900=05109 111000=05110 111100=05111 111200=05112 111300=05113 111400=05114 111500=05115 111600=05116 111700=05117 111800=05118 111900=05119 112000=05120 112100=05121 112200=05122 112300=05123 112400=05124 112500=05125 112600=05126 112700=05127 112800=05128 112900=05129 113000=05130 113100=05131 113200=05132 113300=05133 113400=05134 113500=05135 113600=05136 113700=05137 113800=05138 113900=05139 114000=05140 114100=05141 114200=05142 114300=05143 114400=05144 114500=05145 \ No newline at end of file diff --git a/src/resources/common_app_label_coding.properties b/src/resources/common_app_label_coding.properties deleted file mode 100644 index f89b09f..0000000 --- a/src/resources/common_app_label_coding.properties +++ /dev/null @@ -1 +0,0 @@ -全部=00 HTTP=01 DNS=02 MAIL=03 VPN=04 JSTX=05 FTP=06 SSL=07 IP=08 \ No newline at end of file diff --git a/src/resources/common_city_coding.properties b/src/resources/common_city_coding.properties deleted file mode 100644 index 2f5ceb5..0000000 --- a/src/resources/common_city_coding.properties +++ /dev/null @@ -1,359 +0,0 @@ -#河北 -Shijiazhuang,HeBei,China=130100 -Tangshan,Hebei,China=130200 -Qinhuangdao,Hebei,China=130300 -Handan,Hebei,China=130400 -Xingtai,Hebei,China=130500 -Baoding,Hebei,China=130600 -Zhangjiakou,Hebei,China=130700 -Chengde,Hebei,China=130800 -Cangzhou,Hebei,China=130900 -Langfang,Hebei,China=131000 -Hengshui,Hebei,China=131100 - -#山西 -Taiyuan,Shanxi,China=140100 -Datong,Shanxi,China=140200 -Yangquan,Shanxi,China=140300 -Changzhi,Shanxi,China=140400 -Jincheng,Shanxi,China=140500 -Shuozhou,Shanxi,China=140600 -Jinzhong,Shanxi,China=140700 -Yuncheng,Shanxi,China=140800 -Xinzhou,Shanxi,China=140900 -Linfen,Shanxi,China=141000 -Lvliang,Shanxi,China=141100 - -#内蒙古 -Other,InnerMongoliaAutonomousRegion,China=150000 -Huhehaote,InnerMongoliaAutonomousRegion,China=150100 -Baotou,InnerMongoliaAutonomousRegion,China=150200 -Wuhai,InnerMongoliaAutonomousRegion,China=150300 -Chifeng,InnerMongoliaAutonomousRegion,China=150400 -Tongliao,InnerMongoliaAutonomousRegion,China=150500 -Eerduosi,InnerMongoliaAutonomousRegion,China=150600 -Hulunbeier,InnerMongoliaAutonomousRegion,China=150700 -Bayannaoer,InnerMongoliaAutonomousRegion,China=150800 -Wulanchabu,InnerMongoliaAutonomousRegion,China=150900 -Xingan,InnerMongoliaAutonomousRegion,China=152200 -Xilinguolei,InnerMongoliaAutonomousRegion,China=152500 -Alashan,InnerMongoliaAutonomousRegion,China=152900 - -#辽宁 -Shenyang,Liaoning,China=210100 -Dalian,Liaoning,China=210200 -Anshan,Liaoning,China=210300 -Funshun,Liaoning,China=210400 -Benxi,Liaoning,China=210500 -Dandong,Liaoning,China=210600 -Jinzhou,Liaoning,China=210700 -Yingkou,Liaoning,China=210800 -Fuxin,Liaoning,China=210900 -LiaoYang,Liaoning,China=211000 -Panjin,Liaoning,China=211100 -Tieling,Liaoning,China=211200 -Chaoyang,Liaoning,China=211300 -Huludao,Liaoning,China=211400 - -#吉林 -Changchun,Jilin,China=220100 -Jilin,Jilin,China=220200 -Siping,Jilin,China=220300 -Liaoyuan,Jilin,China=220400 -Tonghua,Jilin,China=220500 -Baishan,Jilin,China=220600 -Songyuan,Jilin,China=220700 -Baicheng,Jilin,China=220800 - - -#黑龙江 -Ha'erbin,Heilongjiang,China=230100 -Qiqiha'er,Heilongjiang,China=230200 -Jixi,Heilongjiang,China=230300 -Hegang,Heilongjiang,China=230400 -Shuangyashan,Heilongjiang,China=230500 -Daqing,Heilongjiang,China=230600 -Yichun,Heilongjiang,China=230700 -Jiamusi,Heilongjiang,China=230800 -Qitaihe,Heilongjiang,China=230900 -Mudanjiang,Heilongjiang,China=231000 -Heihe,Heilongjiang,China=231100 -Suihua,Heilongjiang,China=231200 - -#江苏省 -Nanjing,Jiangsu,China=320100 -Wuxi,Jiangsu,China=320200 -Xuzhou,Jiangsu,China=320300 -Changzhou,Jiangsu,China=320400 -Suzhou,Jiangsu,China=320500 -Nantong,Jiangsu,China=320600 -Lianyungang,Jiangsu,China=320700 -Huai'an,Jiangsu,China=320800 -Yancheng,Jiangsu,China=320900 -Yangzhou,Jiangsu,China=321000 -Zhenjiang,Jiangsu,China=321100 -Suqian,Jiangsu,China=321300 - - -#浙江省 -Hangzhou,Zhejiang,China=330100 -Ningbo,Zhejiang,China=330200 -Wenzhou,Zhejiang,China=330300 -Jiaxing,Zhejiang,China=330400 -Huzhou,Zhejiang,China=330500 -Shaoxing,Zhejiang,China=330600 -Jinhua,Zhejiang,China=330700 -Quzhou,Zhejiang,China=330800 -Zhoushan,Zhejiang,China=330900 -Taizhou,Zhejiang,China=331000 -Lishui,Zhejiang,China=331100 - -#安徽省 -Hefei,Anhui,China=340100 -Wuhu,Anhui,China=340200 -Bangbu,Anhui,China=340300 -Huainan,Anhui,China=340400 -Ma'anshan,Anhui,China=340500 -Huaibei,Anhui,China=340600 -Tongling,Anhui,China=340700 -Anqing,Anhui,China=340800 -Huangshan,Anhui,China=341000 -Chuzhou,Anhui,China=341100 -Fuyang,Anhui,China=341200 -Suzhou,Anhui,China=341300 -Liu'an,Anhui,China=341500 -Haozhou,Anhui,China=341600 -Chizhou,Anhui,China=341700 -Xuancheng,Anhui,China=341800 - -#福建省 -Fuzhou,Fujian,China=350100 -Xiamen,Fujian,China=350200 -Putian,Fujian,China=350300 -Sanming,Fujian,China=350400 -Quanzhou,Fujian,China=350500 -Zhangzhou,Fujian,China=350600 -Nanping,Fujian,China=350700 -Longyan,Fujian,China=350800 -Ningde,Fujian,China=350900 - -#江西省 -Nanchang,Jiangxi,China=360100 -Jingdezhen,Jiangxi,China=360200 -Pingxiang,Jiangxi,China=360300 -Jiujiang,Jiangxi,China=360400 -Xinyu,Jiangxi,China=360500 -Yingtan,Jiangxi,China=360600 -Ganzhou,Jiangxi,China=360700 -Ji'an,Jiangxi,China=360800 -Yichun,Jiangxi,China=360900 -Fuzhou,Jiangxi,China=361000 -Shang'rao,Jiangxi,China=361100 - -#山东省 -Jinan,Shandong,China=370100 -Qingdao,Shandong,China=370200 -Zibo,Shandong,China=370300 -Zaozhuang,Shandong,China=370400 -Dongying,Shandong,China=370500 -Yantai,Shandong,China=370600 -Weifang,Shandong,China=370700 -Jining,Shandong,China=370800 -Tai'an,Shandong,China=370900 -Weihai,Shandong,China=371000 -RiZhao,Shandong,China=371100 -Laiwu,Shandong,China=371200 -Linyi,Shandong,China=371300 -Dezhou,Shandong,China=371400 -Liaocheng,Shandong,China=371500 -Binzhou,Shandong,China=371600 -Heze,Shandong,China=371700 - -#河南省 -Zhengzhou,Henan,China=410100 -Kaifeng,Henan,China=410200 -Luoyang,Henan,China=410300 -Pingdingshan,Henan,China=410400 -Anyang,Henan,China=410500 -Hebi,Henan,China=410600 -Xinxiang,Henan,China=410700 -Jiaozuo,Henan,China=410800 -Puyang,Henan,China=410900 -Xuchang,Henan,China=411000 -Luohe,Henan,China=411100 -Sanmenxia,Henan,China=411200 -Nanyang,Henan,China=411300 -Shangqiu,Henan,China=411400 -Xinyang,Henan,China=411500 -Zhoukou,Henan,China=411600 -Zhumadian,Henan,China=411700 - -#湖北省 -Wuhan,Hubei,China=420100 -Huangshi,Hubei,China=420200 -Shiyan,Hubei,China=420300 -Yichang,Hubei,China=420500 -Xiangyang,Hubei,China=420600 -Ezhou,Hubei,China=420700 -Xingmen,Hubei,China=420800 -Xiaogan,Hubei,China=420900 -Jingzhou,Hubei,China=420100 -Huanggang,Hubei,China=421100 -Xianning,Hubei,China=421200 -Suizhou,Hubei,China=421300 -Enshi,Hubei,China=422800 - - -#湖南省 -Changsha,Hunan,China=430100 -Zhuzhou,Hunan,China=430200 -Xiangtan,Hunan,China=430300 -Hengyang,Hunan,China=430400 -Shaoyang,Hunan,China=430500 -Yueyang,Hunan,China=430600 -Changde,Hunan,China=430700 -Zhangjiajie,Hunan,China=430800 -Yiyang,Hunan,China=430900 -Chenzhou,Hunan,China=431000 -Yongzhou,Hunan,China=431100 -Huaihua,Hunan,China=431200 -Loudi,Hunan,China=431300 - - -#广东省 -Guangzhou,Guangdong,China=440100 -Shaoguan,Guangdong,China=440200 -Shenzhen,Guangdong,China=440300 -Zhuhai,Guangdong,China=440400 -Shantou,Guangdong,China=440500 -Foshan,Guangdong,China=440600 -Jiangmen,Guangdong,China=440700 -Zhanjiang,Guangdong,China=440800 -Maoming,Guangdong,China=440900 -Zhaoqing,Guangdong,China=441200 -Huizhou,Guangdong,China=441300 -Meizhou,Guangdong,China=441400 -Shanwei,Guangdong,China=441500 -Heyuan,Guangdong,China=441600 -Yangjiang,Guangdong,China=441700 -Qingyuan,Guangdong,China=441800 -Dongguan,Guangdong,China=441900 -Zhongshan,Guangdong,China=442000 -Chaozhou,Guangdong,China=445100 -Jieyang,Guangdong,China=445200 -Yunfu,Guangdong,China=445300 - - -#广西壮族自治区 -Nanning,GuangxiZhuangAutonomousRegion,China=450100 -Liuzhou,GuangxiZhuangAutonomousRegion,China=450200 -Guilin,GuangxiZhuangAutonomousRegion,China=450300 -Wuzhou,GuangxiZhuangAutonomousRegion,China=450400 -Beihai,GuangxiZhuangAutonomousRegion,China=450500 -Fangchenggang,GuangxiZhuangAutonomousRegion,China=450600 -Qinzhou,GuangxiZhuangAutonomousRegion,China=450700 -Guigang,GuangxiZhuangAutonomousRegion,China=450800 -Yulin,GuangxiZhuangAutonomousRegion,China=450900 -Baise,GuangxiZhuangAutonomousRegion,China=451000 -Hezhou,GuangxiZhuangAutonomousRegion,China=451100 -Hechi,GuangxiZhuangAutonomousRegion,China=451200 -Laibin,GuangxiZhuangAutonomousRegion,China=451300 -Chongzuo,GuangxiZhuangAutonomousRegion,China=451400 - - - -#海南省 -Haikou,Hainan,China=460100 -Sanya,Hainan,China=460200 -Sansha,Hainan,China=460300 -Danzhou,Hainan,China=460400 - -#四川 -Chengdou,Sichuan,China=510100 -Zigong,Sichuan,China=510300 -Panzhihua,Sichuan,China=510400 -Luzhou,Sichuan,China=510500 -Deyang,Sichuan,China=510600 -Mianyang,Sichuan,China=510700 -Guangyuan,Sichuan,China=510800 -Suining,Sichuan,China=510900 -Neijiang,Sichuan,China=511000 -Leshan,Sichuan,China=511100 -Nanchong,Sichuan,China=511300 -Meishan,Sichuan,China=511400 -Yibin,Sichuan,China=511500 -Guang'an,Sichuan,China=511600 -Dazhou,Sichuan,China=511700 -Ya'an,Sichuan,China=511800 -Bazhong,Sichuan,China=511900 -Ziyang,Sichuan,China=512000 - - -#贵州省 -Guiyang,Guizhou,China=520100 -Liupanshui,Sichuan,China=520200 -Zunyi,Sichuan,China=520300 -Anshun,Sichuan,China=520400 -Bijie,Sichuan,China=520500 -Tongren,Sichuan,China=520600 - -#云南省 -Kunming,Yunnan,China=530100 -Qujing,Yunnan,China=530300 -Yuxi,Yunnan,China=530400 -Baoshan,Yunnan,China=530500 -Zhaotong,Yunnan,China=530600 -Lijiang,Yunnan,China=530700 -Pu'er,Yunnan,China=530800 -Lincang,Yunnan,China=530900 -XishuangbannaDaiAutonomousPrefecture,Yunnan,China=532800 -Dali,Yunnan,China=532900 - - -#陕西省 -xi'an,Shaanxi,China=610100 -Tongchuan,Shaanxi,China=610200 -Baoji,Shaanxi,China=610300 -Xianyang,Shaanxi,China=610400 -Weinan,Shaanxi,China=610500 -Yan'an,Shaanxi,China=610600 -Hanzhong,Shaanxi,China=610700 -Yulin,Shaanxi,China=610800 -Ankang,Shaanxi,China=610900 -Shangluo,Shaanxi,China=611000 - - -#甘肃省 -Lanzhou,Gansu,China=620100 -Jiayuguan,Gansu,China=620200 -Jinchang,Gansu,China=620300 -Baiyin,Gansu,China=620400 -Tianshui,Gansu,China=620500 -Wuwei,Gansu,China=620600 -Zhangye,Gansu,China=620700 -Pingliang,Gansu,China=620800 -Jiuquan,Gansu,China=620900 -Qingyang,Gansu,China=621000 -Dingxi,Gansu,China=621100 -Longnan,Gansu,China=621200 - -#青海省 -Xining,Qinghai,China=630100 -Haidong,Qinghai,China=630200 - - -#宁夏回族自治区 -Yinchuan,NingxiaHuiAutonomousRegion,China=640100 -Zuishan,NingxiaHuiAutonomousRegion,China=640200 -Wuzhong,NingxiaHuiAutonomousRegion,China=640300 -Guyuan,NingxiaHuiAutonomousRegion,China=640400 -Zhongwei,NingxiaHuiAutonomousRegion,China=640500 - - -#台湾 - - - -#西藏 -Lhasa,Tibet,China=540100 -PrivateIP=000000 \ No newline at end of file diff --git a/src/resources/common_country_coding.properties b/src/resources/common_country_coding.properties deleted file mode 100644 index f961472..0000000 --- a/src/resources/common_country_coding.properties +++ /dev/null @@ -1,253 +0,0 @@ -UnitedStates=1 -Canada=1 -Russia=7 -Egypt=20 -SouthAfrica=27 -Greece=30 -Netherlands=31 -Belgium=32 -France=33 -Spain=34 -Hungary=36 -EastGermany=37 -Yugoslavia=38 -Italy=39 -Romania=40 -Switzerland=41 -Czechoslovakia=42 -Austria=43 -UnitedKingdom=44 -Denmark=44 -Sweden=46 -Norway=47 -Poland=48 -Germany=49 -Peru=51 -Mexico=52 -Cuba=53 -Argentina=54 -Brazil=55 -Chile=56 -Colombia=57 -Venezuela=58 -Malaysia=60 -Australia=61 -Indonesia=62 -Philippines=63 -newZealand=64 -Singapore=65 -Thailand=66 -Japa=81 -Korea=82 -Vietnam=84 -China=86 -Turkey=90 -India=91 -Pakistan=92 -Afghanistan=93 -SriLanka=94 -Myanmar=95 -Iran=98 -SouthSudan=211 -Morocco=212 -Algeria=213 -Tunisia=216 -Libya=218 -Gambia=220 -Senegal=221 -Mauritania=222 -Mali=223 -Guinea=224 -Coted'Ivoire=225 -BurkinaFaso=226 -Niger=227 -Togo=228 -Benin=229 -Mauritius=230 -Liberia=231 -SierraLeone=232 -Ghana=233 -Nigeria=234 -Chad=235 -CentralAfricanRepublic=236 -Cameroon=237 -CapeVerde=238 -SaoTomeandPrincipe=239 -EquatorialGuinea=240 -Gabon=241 -therepublicofCongo=242 -DemocraticRepublicofCongo=243 -Angola=244 -Guinea-Bissau=245 -DiegoGarcia=246 -AscensionIsland=247 -Seychelles=248 -Sudan=249 -Rwanda=250 -Ethiopia=251 -Somalia=252 -Djibouti=253 -Kenya=254 -Tanzania=255 -Uganda=256 -Burundi=257 -Mozambique=258 -Zanzibar=259 -Zambia=260 -Madagascar=261 -Reunion=262 -Zimbabwe=263 -Namibia=264 -Malawi=265 -Lesotho=266 -Botswana=267 -Swaziland=268 -ComorosandMayotte=269 -St.Helena=290 -Eritrea=291 -Aruba=297 -FaroeIslands=298 -Greenland=299 -Gibraltar=350 -Portugal=351 -Luxembourg=352 -Ireland=353 -Iceland=354 -Albania=355 -Malta=356 -Cyprus=357 -Finland=358 -Bulgaria=359 -Lithuania=370 -Latvia=371 -Estonia=372 -Moldova=373 -Armenia=374 -Belarus=375 -Andorra=376 -Monaco=377 -SanMarino=378 -Vatican=379 -Ukraine=380 -Serbia=381 -Montenegro=382 -Croatia=385 -Slovenia=386 -BosniaandHerzegovina=387 -Europeantelephonenumberspace-ringEuropeservice=388 -Macedonia=389 -CzechRepublic=420 -Slovakia=421 -Liechtenstein=423 -FalklandIslands=500 -Belize=501 -Guatemala=502 -Salvador=503 -Honduras=504 -Nicaragua=505 -CostaRica=506 -Panama=507 -SaintPierreandMiquelon=508 -Haiti=509 -Guadeloupe(includingFrenchSaintMartinandSaintBarthelemy)=590 -Bolivia=591 -Guyana=592 -Ecuador=593 -FrenchGuiana=594 -Paraguay=595 -Martinique=596 -Suriname=597 -Uruguay=598 -NetherlandsAntilles=599 -EastTimor=670 -Australianoverseasterritories:Antarctica,ChristmasIsland,CocosIslands,NorfolkIsland=672 -Brunei=673 -Nauru=674 -PapuaNewGuinea=675 -Tonga=676 -SolomonIslands=677 -Vanuatu=678 -Fiji=679 -Palau=680 -WallisandFutunaIslands=681 -CookIslands=682 -Niue=683 -Samoa=685 -Kiribati,GilbertIslands=686 -NewCaledonia=687 -Tuvalu,EllisIslands=688 -FrenchPolynesia=689 -Tokelau=690 -FederatedStatesofMicronesia=691 -MarshallIslands=692 -NorthKorea=850 -Macao,China=853 -Cambodia=855 -Laos=856 -InternationalMaritimeSatelliteOrganization=870 -GlobalPersonalCommunicationService=878 -Bangladesh=880 -Mobilesatellitesystem=881 -Internet=882 - -Maldives=960 -Lebanon=961 -Jordan=962 -Syria=963 -Iraq=964 -Kuwait=965 -SaudiArabia=966 -Yemen=967 -Oman=968 -YemenDemocraticRepublic=969 -Palestine=970 -UnitedArabEmirates=971 -Israel=972 -Bahrain=973 -Qatar=974 -Bhutan=975 -Mongolia=976 -Nepal=977 -Internationalrateservice=979 -InternationalTelecommunicationsPublicCommunicationServiceTest=991 -Tajikistan=992 -Turkmenistan=993 -Azerbaijan=994 -Georgia=995 -Kyrgyzstan=996 -Uzbekistan=998 -Bahamas=1242 -Barbados=1246 -Anguilla=1264 -AntiguaandBarbuda=1268 -BritishVirginIslands=1284 -U.S.VirginIslands=1340 -CaymanIslands=1345 -Bermuda=1441 -Grenada=1473 -TurksandCaicosIslands=1649 -Montserrat=1664 -NorthernMarianaIslands=1670 -Guam=1671 -AmericanSamoa=1684 -SaintLucia=1758 -Dominica=1767 -SaintVincentandheGrenadines=1784 -PuertoRico=1787 -DominicanRepublic=1809 -TrinidadandTobago=1868 -SaintKittsandNevis=1869 -Jamaica=1876 -Curacao=5999 -Macao=820000 -HongKong=810000 -Taiwan=710000 - - - - - - - - - diff --git a/src/resources/common_jstx_id_coding.properties b/src/resources/common_jstx_id_coding.properties deleted file mode 100644 index ac60791..0000000 --- a/src/resources/common_jstx_id_coding.properties +++ /dev/null @@ -1,194 +0,0 @@ -100=Unknown_TCP -101=Unknown_UDP -102=Unknown_Other -103=DNS -104=FTP -105=FTPS -106=HTTP -107=HTTPS -108=ICMP -109=IKE -110=MAIL -111=IMAPS -112=IPSEC -113=XMPP -114=L2TP -115=NTP -117=POP3S -118=PPTP -119=QUIC -120=SIP -121=SMB -123=SMTPS -124=SPDY -125=SSH -126=SSL -127=SOCKS -128=TELNET -129=DHCP -130=RADIUS -131=OPENVPN -132=STUN -133=TEREDO -134=DTLS -135=DoH -136=ISAKMP -137=MDNS -138=NETBIOS -139=NETFLOW -140=RDP -141=RTCP -142=RTP -143=SLP -144=SNMP -145=SSDP -146=TFTP -147=BJNP -148=LDAP -149=RTMP -150=RTSP -200=05001 -201=05002 -202=05003 -203=05004 -204=05005 -205=05006 -206=05007 -207=05008 -208=05009 -209=05010 -210=05011 -211=05012 -212=05013 -213=05014 -214=05015 -215=05016 -216=05017 -217=05018 -218=05019 -219=05020 -220=05021 -221=05022 -222=05023 -223=05024 -224=05025 -225=05026 -226=05027 -227=05028 -228=05029 -229=05030 -230=05031 -231=05032 -232=05033 -233=05034 -234=05035 -235=05036 -236=05037 -237=05038 -238=05039 -239=05040 -240=05041 -241=05042 -242=05043 -243=05044 -244=05045 -245=05046 -246=05047 -247=05048 -248=05049 -249=05050 -250=05051 -251=05052 -252=05053 -253=05054 -254=05055 -255=05056 -256=05057 -257=05058 -258=05059 -259=05060 -260=05061 -261=05062 -262=05063 -263=05064 -264=05065 -265=05066 -266=05067 -267=05068 -268=05069 -269=05070 -270=05071 -271=05072 -272=05073 -273=05074 -274=05075 -275=05076 -276=05077 -277=05078 -278=05079 -279=05080 -280=05081 -281=05082 -282=05083 -283=05084 -284=05085 -285=05086 -286=05087 -287=05088 -288=05089 -289=05090 -290=05091 -291=05092 -292=05093 -293=05094 -294=05095 -295=05096 -296=05097 -297=05098 -298=05099 -299=05100 -300=05101 -301=05102 -302=05103 -303=05104 -304=05105 -305=05106 -306=05107 -307=05108 -308=05109 -309=05110 -310=05111 -311=05112 -312=05113 -313=05114 -314=05115 -315=05116 -316=05117 -317=05118 -318=05119 -319=05120 -320=05121 -321=05122 -322=05123 -323=05124 -324=05125 -325=05126 -326=05127 -327=05128 -328=05129 -329=05130 -330=05131 -331=05132 -332=05133 -333=05134 -334=05135 -335=05136 -336=05137 -337=05138 -338=05139 -339=05140 -340=05141 -341=05142 -342=05143 -343=05144 -344=05145 \ No newline at end of file diff --git a/src/resources/common_province_coding.properties b/src/resources/common_province_coding.properties deleted file mode 100644 index 7244591..0000000 --- a/src/resources/common_province_coding.properties +++ /dev/null @@ -1,55 +0,0 @@ -Beijing=110000 -Tianjin=120000 -Hebei=130000 -Shanxi=140000 -InnerMongoliaAutonomousRegion=150000 -Liaoning=210000 -Jilin=220000 -Heilongjiang=230000 -Shanghai=310000 -Jiangsu=320000 -Zhejiang=330000 -Anhui=340000 -Fujian=350000 -Jiangxi=360000 -Shandong=370000 -Henan=410000 -Hubei=420000 -Hunan=430000 -Guangdong=440000 -GuangxiZhuangAutonomousRegion=450000 -Hainan=460000 -Chongqing=500000 -Sichuan=510000 -Guizhou=520000 -Yunnan=530000 -TibetAutonomousRegion=540000 -Shaanxi=610000 -Gansu=620000 -Qinghai=630000 -NingxiaHuiAutonomousRegion=640000 -Taiwan=710000 -HongKong=810000 -Macao=820000 -Tibet=540000 - -TaipeiCity=710100 -Kaohsiung=710200 -Tainan=710300 -TaichungCity=710400 -Nantou=710600 -Keelung=710700 -Hsinchu=710800 -Chiayi=710900 -NewTaipei=711100 -Yilan=711200 -HsinchuCounty=711300 -Taoyuan=711400 -Miaoli=711500 -Changhua=711700 -ChiayiCounty=711900 -Yunlin=712100 -Pingtung=712400 -Taitung=712500 -Hualien=712600 -PenghuCounty=712700 \ No newline at end of file diff --git a/src/resources/service_bolt_config.properties b/src/resources/service_bolt_config.properties deleted file mode 100644 index fba3993..0000000 --- a/src/resources/service_bolt_config.properties +++ /dev/null @@ -1,68 +0,0 @@ -#Flooding-spout-field - - -#conn-bolt-StreamID -conn.dns.streamid=CONN-DNS-SOURCE -conn.ftp.streamid=CONN-FTP-SOURCE -conn.http.streamid=CONN-HTTP-SOURCE -conn.ip.streamid=CONN-IP-SOURCE -conn.jstx.streamid=CONN-JSTX-SOURCE -conn.mail.streamid=CONN-MAIL-SOURCE -conn.ssl.streamid=CONN-SSL-SOURCE -conn.vpn.streamid=CONN-VPN-SOURCE - - -#conn-ComponentId-spout -kafka.spout=KAFKA-SPOUT - -#conn-ComponentId-bolt -conn.parse.componentid=CONN-PARSE-COMPONENTID - - -conn.ybinputsize.streamid=CONN-YBInputSize-STREAMID - - - -#safe-spout-field -safe.spout.field=SAFE-SOURCE - - -#GK-SOURCE-STREAMID -gk.ip.streamid=GK-IP-STREAMID -gk.dns.streamid=GK-DNS-STREAMID -gk.http.streamid=GK-HTTP-STREAMID -gk.ssl.streamid=GK-SSL-STREAMID -gk.wygjc.streamid=GK-WYGJC-STREAMID -gk.mail.streamid=GK-MAIL-STREAMID -gk.ftp.streamid=GK-FTP-STREAMID -gk.vpn.streamid=GK-VPN-STREAMID -gk.jstx.streamid=GK-JSTX-STREAMID - - -gk.count.streamid=GK-COUNT-STREAMID - - - -#JC-SOURCE-STREAMID -jc.ip.streamid=JC-IP-STREAMID -jc.dns.streamid=JC-DNS-STREAMID -jc.http.streamid=JC-HTTP-STREAMID -jc.ssl.streamid=JC-SSL-STREAMID -jc.wygjc.streamid=JC-WYGJC-STREAMID -jc.mail.streamid=JC-MAIL-STREAMID -jc.ftp.streamid=JC-FTP-STREAMID -jc.vpn.streamid=JC-VPN-STREAMID -jc.jstx.streamid=JC-JSTX-STREAMID -jc.pic.streamid=JC-PIC-STREAMID -jc.media.streamid=JC-MEDIA-STREAMID -jc.count.streamid=JC-COUNT-STREAMID - - - - - - - - - - diff --git a/src/resources/service_flow_config.properties b/src/resources/service_flow_config.properties deleted file mode 100644 index d84bfdd..0000000 --- a/src/resources/service_flow_config.properties +++ /dev/null @@ -1,200 +0,0 @@ -#管理kafka地址 -bootstrap.servers=10.111.72.5:9092,10.111.72.6:9092,10.111.72.7:9092,10.111.72.8:9092,10.111.72.9:9092,10.111.72.16:9092,10.111.72.17:9092,10.111.72.18:9092 -#latest/earliest -auto.offset.reset=earliest -#压缩模式 none or snappy -kafka.compression.type=none - -topology.logo=jc -#通联日志kafka主题 -kafka.topic=CONNECTION-RECORD-COMPLETED-LOG - -#策略日志kafka主题 -kafka.safe.topic=SECURITY-EVENT-COMPLETED-LOG - - -#消费者组id-读取topic,存储该spout id的消费offset信息，每次启动前需要更改 -group.id=xj-jc-group20210709 - -#kafka消费者每次拉取数据的时间 -kafka.consumer.poll=1000 - -#storm topology workers -topology.workers=8 - -#spout并行度，建议与kafka分区一致 -max.spout.parallelism=8 -spout.parallelism=8 - - - -#ack设置 1启动ack 0不启动ack -topology.num.acks=0 - -#spou睡眠间隔 -topology.spout.sleep.time=1 - - -#通联程序批处理的时间 -topology.tick.tuple.freq.secs=60 -#通联程序没批处理的条数 -batch.inter.num=100000 - - - - -parse.parallelism=100 -send.parallelism=20 -ip.send.parallelism=50 -ssl.send.parallelism=50 -http.send.parallelism=50 -count.send.parallelism=1 -unstructured.send.parallelism=50 - - - -#count指标数据时间批次（单位为秒） -count.topology.tick.tuple.freq.secs=300 - -#prometheus-httpserver -prometheus.pushgateway.address=10.111.143.240:9274 - - - -#http-post请求超时时间 -http.post.socketTimeOut=60 - - -#http-post发送设置 -http.connectionRequestTimeout=1000 -http.connectTimeout=1000 -http.socketTimeout=1000 -http.sent.count=10 -http.retry.count=2 -http.heard.content.type=utf-8 -http.hearder.user=yd_xgs_source -http.hearder.password=yd_xgs_source@2021 -http.hearder.format=avro -http.hearder.conection=keep-alilve -#http-post后端接受地址 -http.post.ip=10.111.151.249,10.111.151.250,10.111.151.251,10.111.151.252,10.111.151.253,10.111.153.248,10.111.153.249,10.111.153.250,10.111.153.251,10.111.153.252,10.111.153.253 -http.post.port=10080 - - - -hos.url=http://10.111.143.200:9098/hos/ -hos.pool.maxconnections=400 -hos.pool.maxerrorretry=3 -hos.pool.requesttimeout=0 -hos.pool.connectiontimeout=60 -hos.pool.connectionttl=-1 -hos.pool.sockettimeout=50 - - - - -s3.connectionRequestTimeout=1000 -s3.connectTimeout=1000 -s3.socketTimeout=1000 -s3.sent.count=10 -s3.hearder.username=yd_xgs_s3 -s3.hearder.password=yd_xgs_s3@2021 -s3.hearder.filetype=pcm -s3.hearder.filetitle=ydtest-file -s3.hearder.isfulltex=false -s3.hearder.contenttype=application/octet-stream - -s3.post.ip=10.111.144.4,10.111.144.5,10.111.144.6 -s3.post.port=8888 - -t.http.rzzb=t-http-rzzb -t.http.jczb=t-http-jczb -t.http.gkzb=t-http-gkzb - -t.mail.rzzb=t-mail-rzzb -t.mail.jczb=t-mail-jczb -t.mail.gkzb=t-mail-gkzb - -t.wygjc.jcrz=t-wygjc-jcrz -t.wygjc.gkrz=t-wygjc-gkrz - -t.pic.jczb=t-pic-jczb -t.media.jczb=t-media-jczb - - - - - - -#通联数据的schema -ftp.rzzb.schema={"type":"record","name":"FTP_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -dns.rzzb.schema={"type":"record","name":"DNS_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DNS_NAME","type":["string","null"]},{"name":"DNS_VALUE","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -http.rzzb.schema={"type":"record","name":"HTTP_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"COOKIE","type":["string","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -ip.rzzb.schema={"type":"record","name":"IP_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -jstx.rzzb.schema={"type":"record","name":"JSTX_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DZ_ID","type":["string","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"URL_AGENT","type":["string","null"]},{"name":"USER_ID","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"RESERVED","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -ssl.rzzb.schema={"type":"record","name":"SSL_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"ZSMC","type":["string","null"]},{"name":"BFZ","type":["string","null"]},{"name":"SYZ","type":["string","null"]},{"name":"FWQMC","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -vpn.rzzb.schema={"type":"record","name":"VPN_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"URL","type":["int","null"]},{"name":"USERNAME","type":["int","null"]},{"name":"PWD","type":["int","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -mail.rzzb.schema={"type":"record","name":"MAIL_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"MAIL_SENDER","type":["string","null"]},{"name":"MAIL_RECEVER","type":["string","null"]},{"name":"RECEVER_NUM","type":["int","null"]},{"name":"MAIL_CC","type":["string","null"]},{"name":"CC_NUM","type":["int","null"]},{"name":"MAIL_SUBJECT","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"ACCESSORY","type":["int","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"FJFWS","type":["string","null"]},{"name":"FJFWS_SZD","type":["string","null"]},{"name":"SJFWS","type":["string","null"]},{"name":"SJFWS_SZD","type":["string","null"]},{"name":"XYLX","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} - - -conn.post.dns.topic=t_dns_rzzb -conn.post.ftp.topic=t_ftp_rzzb -conn.post.http.topic=t_http_rzzb -conn.post.ip.topic=t_ip_rzzb -conn.post.jstx.topic=t_jstx_rzzb -conn.post.mail.topic=t_mail_rzzb -conn.post.ssl.topic=t_ssl_rzzb -conn.post.vpn.topic=t_vpn_rzzb - - -gk.post.dns.topic=t_dns_gkzb -gk.post.ftp.topic=t_ftp_gkzb -gk.post.http.topic=t_http_gkzb -gk.post.ip.topic=t_ip_gkzb -gk.post.jstx.topic=t_jstx_gkzb -gk.post.mail.topic=t_mail_gkzb -gk.post.ssl.topic=t_ssl_gkzb -gk.post.vpn.topic=t_vpn_gkzb -gk.post.wygjc.topic=t_wygjc_gkrz - - -jc.post.dns.topic=t_dns_jczb -jc.post.ftp.topic=t_ftp_jczb -jc.post.http.topic=t_http_jczb -jc.post.ip.topic=t_ip_jczb -jc.post.jstx.topic=t_jstx_jczb -jc.post.mail.topic=t_mail_jczb -jc.post.ssl.topic=t_ssl_jczb -jc.post.vpn.topic=t_vpn_jczb -jc.post.wygjc.topic=t_wygjc_jcrz -jc.post.pic.topic=t_pic_jczb -jc.post.media.topic=t_media_jczb - - - - - - -#gk-schema -ip.gkrz.schema={"type":"record","name":"IP_GKRZ","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -dns.gkzb.schema={"type":"record","name":"DNS_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"DNS_NAME","type":["string","null"]},{"name":"DNS_VALUE","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":["int","null"]},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -http.gkzb.schema={"type":"record","name":"HTTP_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"COOKIE","type":["string","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -ssl.gkzb.schema={"type":"record","name":"SSL_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"ZSMC","type":["string","null"]},{"name":"BFZ","type":["string","null"]},{"name":"SYZ","type":["string","null"]},{"name":"FWQMC","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -wygjc.gkrz.schema={"type":"record","name":"WYGJC_GKRZ","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"JC_URL","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"SSYQ","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -ftp.gkzb.schema={"type":"record","name":"FTPGJZ_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -vpn.gkzb.schema={"type":"record","name":"VPN_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"URL","type":["string","null"]},{"name":"USERNAME","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -jstx.gkzb.schema={"type":"record","name":"JSTX_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"DZ_ID","type":["string","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"USER_ID","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"RESERVED","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -mail.gkzb.schema={"type":"record","name":"MAIL_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"MAIL_SENDER","type":["string","null"]},{"name":"MAIL_RECEVER","type":["string","null"]},{"name":"RECEVER_NUM","type":["int","null"]},{"name":"MAIL_CC","type":["string","null"]},{"name":"CC_NUM","type":["int","null"]},{"name":"MAIL_SUBJECT","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"ACCESSORY","type":["int","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"FJFWS","type":["string","null"]},{"name":"FJFWS_SZD","type":["string","null"]},{"name":"SJFWS","type":["string","null"]},{"name":"SJFWS_SZD","type":["string","null"]},{"name":"XYLX","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} - -#jc-schema -dns.jczb.schema={"type":"record","name":"DNS_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DNS_NAME","type":["string","null"]},{"name":"DNS_VALUE","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -ftp.jczb.schema={"type":"record","name":"FTPGJZ_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -http.jczb.schema={"type":"record","name":"HTTP_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"COOKIE","type":["string","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -ip.jczb.schema={"type":"record","name":"IP_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -jstx.jczb.schema={"type":"record","name":"JSTX_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DZ_ID","type":["string","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"USER_ID","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"RESERVED","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -ssl.jczb.schema={"type":"record","name":"TDZS_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"ZSMC","type":["string","null"]},{"name":"BFZ","type":["string","null"]},{"name":"SYZ","type":["string","null"]},{"name":"FWQMC","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -vpn.jczb.schema={"type":"record","name":"VPN_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"URL","type":["string","null"]},{"name":"USERNAME","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -wygjz.jczb.schema={"type":"record","name":"WYGJC_JCRZ","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"JC_URL","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"SSYQ","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -pic.jczb.schema={"type":"record","name":"PIC_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"YYZL","type":["int","null"]},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"PIC_NAME","type":["string","null"]},{"name":"PIC_URL","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"PIC_TYPE","type":["string","null"]},{"name":"PIC_CONTENT","type":["bytes","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -media.jczb.schema={"type":"record","name":"MEDIA_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YYZL","type":["int","null"]},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"MEDIA_NAME","type":["string","null"]},{"name":"MEDIA_URL","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"MEDIA_TYPE","type":["string","null"]},{"name":"MEDIA_CONTENT","type":["bytes","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} -mail.jczb.schema={"type":"record","name":"MAILGJZ_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"MAIL_SENDER","type":["string","null"]},{"name":"MAIL_RECEVER","type":["string","null"]},{"name":"RECEVER_NUM","type":["int","null"]},{"name":"MAIL_CC","type":["string","null"]},{"name":"CC_NUM","type":["int","null"]},{"name":"MAIL_SUBJECT","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"ACCESSORY","type":["int","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"FJFWS","type":["string","null"]},{"name":"FJFWS_SZD","type":["string","null"]},{"name":"SJFWS","type":["string","null"]},{"name":"SJFWS_SZD","type":["string","null"]},{"name":"XYLX","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} diff --git a/src/test/java/People.java b/src/test/java/People.java deleted file mode 100644 index 46c6902..0000000 --- a/src/test/java/People.java +++ /dev/null @@ -1,29 +0,0 @@ -public class People { - private String name; - private int age; - private String weight; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public int getAge() { - return age; - } - - public void setAge(int age) { - this.age = age; - } - - public String getWeight() { - return weight; - } - - public void setWeight(String weight) { - this.weight = weight; - } -} diff --git a/src/test/java/Test.java b/src/test/java/Test.java index 0ba1f8b..1b0ce4d 100644 --- a/src/test/java/Test.java +++ b/src/test/java/Test.java @@ -15,12 +15,4 @@ public class Test { } return sb.toString(); } - - public static void main(String[] args) { - People people = new People(); - if (2>1){ - people.setAge(2); - } - System.out.println(people.getAge()); - } } diff --git a/src/test/java/TestDy.java b/src/test/java/TestDy.java index 0cbdd33..eace052 100644 --- a/src/test/java/TestDy.java +++ b/src/test/java/TestDy.java @@ -1,16 +1,30 @@ +import com.amazonaws.services.s3.model.CSVOutput; +import org.apache.logging.log4j.util.PropertiesUtil; + import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; -import static cn.ac.iie.storm.utils.config.GetMap.*; - -import static cn.ac.iie.storm.utils.general.MailTrans.*; +import static cn.ac.iie.storm.utils.general.GetMap.getDy; import static cn.ac.iie.storm.utils.general.TransFormUtils.*; public class TestDy { - public static void main(String[] args) { -// String mail = getSecurityMailDetail_id(410, "0300231", "WEBMAIL"); -// System.out.println(mail); - - String imap = getYjxy(510, "03000", "IMAP"); - System.out.println(imap); - } + private static Properties propService_city = new Properties(); + InputStream cityNumbering = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_city_coding.properties"); + // propService_city.load(cityNumbering); +// Map cityMap = new HashMap((Map) propService_city); +// for (String key : cityMap.keySet()) { +// String str = key.replace(",", "."); +// String[] split = str.split("\\."); +// if (split.length != 3) { +// System.out.println(key); +//// System.out.println(split[2] + "." + split[1] + "." + split[0] + "=" + cityMap.get(key)); +// } +// } + public static void main(String[] args) throws IOException { + String dy = getDy("China.Fujian.Other"); + System.out.println(dy); + } } \ No newline at end of file diff --git a/src/test/java/TestHos.java b/src/test/java/TestHos.java index e64a00d..39ed272 100644 --- a/src/test/java/TestHos.java +++ b/src/test/java/TestHos.java @@ -1,7 +1,5 @@ import java.io.*; -import static cn.ac.iie.storm.utils.config.GetMap.getDy; - import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost; public class TestHos { diff --git a/src/test/java/testAppId.java b/src/test/java/testAppId.java index 5b1367d..ecb05b3 100644 --- a/src/test/java/testAppId.java +++ b/src/test/java/testAppId.java @@ -1,6 +1,6 @@ import java.io.IOException; -import static cn.ac.iie.storm.utils.config.GetMap.getJstxDetail_id; +import static cn.ac.iie.storm.utils.general.GetMap.getJstxDetail_id; public class testAppId { public static void main(String[] args) throws IOException {