gfwleak/galaxy-tsg-olap-xj-log-etl
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1.添加yys、ydy、mddy等监控指标。

Browse Source 
2.修改getDy()函数业务逻辑。
This commit is contained in:
wangchengcheng
2022-08-11 14:25:27 +08:00
parent 24d22fb601
commit 37975911b7
44 changed files with 688 additions and 1900 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
pom.xml
View File

@@ -181,12 +181,12 @@
<version>${storm.version}</version> <version>${storm.version}</version>
</dependency> </dependency>
<dependency> <!-- <dependency>-->
<groupId>junit</groupId> <!-- <groupId>junit</groupId>-->
<artifactId>junit</artifactId> <!-- <artifactId>junit</artifactId>-->
<version>4.12</version> <!-- <version>4.12</version>-->
<scope>test</scope> <!-- <scope>test</scope>-->
</dependency> <!-- </dependency>-->
<dependency> <dependency>
<groupId>com.alibaba</groupId> <groupId>com.alibaba</groupId>

46
src/main/java/cn/ac/iie/storm/bean/SessionRecordLog.java
View File

@@ -4,7 +4,9 @@ package cn.ac.iie.storm.bean;
* @author wangchengcheng * @author wangchengcheng
*/ */
public class SessionRecordLog { public class SessionRecordLog {
private String ydy;
private String mddy;
private int yys;
private String http_request_header; private String http_request_header;
private String gjc; private String gjc;
private String common_user_region; private String common_user_region;
@@ -24,7 +26,6 @@ public class SessionRecordLog {
private String common_isp; private String common_isp;
private int common_address_type; private int common_address_type;
private String common_app_label; private String common_app_label;
private int common_app_id;
private int common_c2s_pkt_num; private int common_c2s_pkt_num;
private long common_c2s_byte_num; private long common_c2s_byte_num;
private int common_s2c_pkt_num; private int common_s2c_pkt_num;
@@ -73,6 +74,8 @@ public class SessionRecordLog {
private String yjxy; private String yjxy;
private String common_protocol_label; private String common_protocol_label;
private String lx;
public String getHttp_content_type() { public String getHttp_content_type() {
return http_content_type; return http_content_type;
} }
@@ -289,13 +292,7 @@ public class SessionRecordLog {
this.common_app_label = common_app_label; this.common_app_label = common_app_label;
} }
public int getCommon_app_id() {
return common_app_id;
}
public void setCommon_app_id(int common_app_id) {
this.common_app_id = common_app_id;
}
public int getCommon_c2s_pkt_num() { public int getCommon_c2s_pkt_num() {
return common_c2s_pkt_num; return common_c2s_pkt_num;
@@ -597,7 +594,40 @@ public class SessionRecordLog {
return protocol; return protocol;
} }
public void setProtocol(String protocol) { public void setProtocol(String protocol) {
this.protocol = protocol; this.protocol = protocol;
} }
public int getYys() {
return yys;
}
public void setYys(int yys) {
this.yys = yys;
}
public String getYdy() {
return ydy;
}
public void setYdy(String ydy) {
this.ydy = ydy;
}
public String getMddy() {
return mddy;
}
public void setMddy(String mddy) {
this.mddy = mddy;
}
public String getLx() {
return lx;
}
public void setLx(String lx) {
this.lx = lx;
}
} }

9
src/main/java/cn/ac/iie/storm/bean/Traffic_Forward.java
View File

@@ -16,6 +16,7 @@ public class Traffic_Forward {
private String labels; private String labels;
private String zsmc; private String zsmc;
private String yjxy; private String yjxy;
private String lx;
public String getYjxy() { public String getYjxy() {
return yjxy; return yjxy;
@@ -128,4 +129,12 @@ public class Traffic_Forward {
public void setGjc(String gjc) { public void setGjc(String gjc) {
this.gjc = gjc; this.gjc = gjc;
} }
public String getLx() {
return lx;
}
public void setLx(String lx) {
this.lx = lx;
}
} }

2
src/main/java/cn/ac/iie/storm/bolt/conn/ConnHTTPSendBolt.java
View File

@@ -44,7 +44,7 @@ public class ConnHTTPSendBolt extends BaseBasicBolt {
i++; i++;
} }
if (connHttpPost!=null&&!connHttpPost.containsValue("200")){ if (connHttpPost!=null&&!connHttpPost.containsValue("200")){
logger.error("-------条数为:"+httpcount+"connhttp数据发送到"+postUrl+"成功-----"+connHttpPost); logger.error("-------条数为:"+httpcount+"connhttp数据发送到"+postUrl+"失败-----"+connHttpPost);
conn_http_fail = conn_http_fail + httpcount; conn_http_fail = conn_http_fail + httpcount;
} }
} }

3
src/main/java/cn/ac/iie/storm/bolt/conn/ConnIPSendBolt.java
View File

@@ -12,7 +12,6 @@ import org.apache.storm.tuple.Values;
import org.apache.storm.utils.TupleUtils; import org.apache.storm.utils.TupleUtils;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*;
@@ -46,7 +45,7 @@ public class ConnIPSendBolt extends BaseBasicBolt {
i++; i++;
} }
if (connIpPost!=null&&!connIpPost.containsValue("200")){ if (connIpPost!=null&&!connIpPost.containsValue("200")){
logger.error("-------条数为:"+ipcount+"connip数据发送到"+postUrl+"成功-----"+connIpPost); logger.error("-------条数为:"+ipcount+"connip数据发送到"+postUrl+"失败-----"+connIpPost);
conn_ip_fail = conn_ip_fail + ipcount; conn_ip_fail = conn_ip_fail + ipcount;
} }
} }

260
src/main/java/cn/ac/iie/storm/bolt/conn/ConnParseBolt.java
View File

@@ -4,6 +4,7 @@ import cn.ac.iie.storm.bean.SessionRecordLog;
import cn.ac.iie.storm.utils.config.StreamAggregateConfig; import cn.ac.iie.storm.utils.config.StreamAggregateConfig;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.zdjizhi.utils.StringUtil;
import org.apache.avro.Schema; import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecord;
@@ -23,28 +24,27 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import static cn.ac.iie.storm.utils.general.GetMap.getDy;
import static cn.ac.iie.storm.utils.general.CONNAssignment.*; import static cn.ac.iie.storm.utils.general.CONNAssignment.*;
import static cn.ac.iie.storm.utils.general.TransFormUtils.*; import static cn.ac.iie.storm.utils.general.TransFormUtils.*;
import static cn.ac.iie.storm.utils.hos.HosService.httpPostObject;
import static cn.ac.iie.storm.utils.hos.Send2S3.SendObject2S3;
import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.getS3Url;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.getYL5; import static cn.ac.iie.storm.utils.http.HttpGetUrl.getYL5;
public class ConnParseBolt extends BaseBasicBolt { public class ConnParseBolt extends BaseBasicBolt {
private final static Logger logger = Logger.getLogger(ConnParseBolt.class); private final static Logger logger = Logger.getLogger(ConnParseBolt.class);
private static final long serialVersionUID = 7483439472002862539L; private static final long serialVersionUID = 7483439472002862539L;
private List<GenericRecord> IPList = new ArrayList<GenericRecord>();
private List<GenericRecord> HTTPList = new ArrayList<GenericRecord>();
private List<GenericRecord> FTPList = new ArrayList<GenericRecord>(); private List<GenericRecord> ipList = new ArrayList<GenericRecord>();
private List<GenericRecord> SSLList = new ArrayList<GenericRecord>(); private List<GenericRecord> httpList = new ArrayList<GenericRecord>();
private List<GenericRecord> VPNList = new ArrayList<GenericRecord>(); private List<GenericRecord> ftpList = new ArrayList<GenericRecord>();
private List<GenericRecord> DNSList = new ArrayList<GenericRecord>(); private List<GenericRecord> sslList = new ArrayList<GenericRecord>();
private List<GenericRecord> MAILList = new ArrayList<GenericRecord>(); private List<GenericRecord> vpnList = new ArrayList<GenericRecord>();
private List<GenericRecord> JSTXList = new ArrayList<GenericRecord>(); private List<GenericRecord> dnsList = new ArrayList<GenericRecord>();
private List<GenericRecord> mailList = new ArrayList<GenericRecord>();
private List<GenericRecord> jstxList = new ArrayList<GenericRecord>();
private SessionRecordLog sessionRecordLog; private SessionRecordLog sessionRecordLog;
@@ -53,6 +53,18 @@ public class ConnParseBolt extends BaseBasicBolt {
private long connOutputSize = 0L; private long connOutputSize = 0L;
private long connInputSize = 0L; private long connInputSize = 0L;
private long yys_error=0L;
private long ydy_inside=0L;
private long mddy_inside=0L;
private long ydy_outside=0L;
private long mddy_outside=0L;
private long ydy_unknown=0L;
private long mddy_unknown=0L;
private Schema vpnSchema; private Schema vpnSchema;
private Schema ipSchema; private Schema ipSchema;
private Schema httpSchema; private Schema httpSchema;
@@ -62,8 +74,8 @@ public class ConnParseBolt extends BaseBasicBolt {
private Schema mailSchema; private Schema mailSchema;
private Schema jstxSchema; private Schema jstxSchema;
String bucketName = null; String hosBucketName = null;
String filename = null; String fileName = null;
public void prepare(Map stormConf, TopologyContext context) { public void prepare(Map stormConf, TopologyContext context) {
ipSchema = new Schema.Parser().parse(StreamAggregateConfig.IP_RZZB_SCHEMA); ipSchema = new Schema.Parser().parse(StreamAggregateConfig.IP_RZZB_SCHEMA);
@@ -87,185 +99,226 @@ public class ConnParseBolt extends BaseBasicBolt {
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
if(TupleUtils.isTick(tuple)){ if(TupleUtils.isTick(tuple)){
try { try {
if (!VPNList.isEmpty()){ if (!vpnList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.CONN_VPN_STREAMID, new Values((Object) generate(vpnSchema, VPNList),VPNList.size())); basicOutputCollector.emit(StreamAggregateConfig.CONN_VPN_STREAMID, new Values(generate(vpnSchema, vpnList),vpnList.size()));
VPNList.clear(); vpnList.clear();
} }
if (!DNSList.isEmpty()){ if (!dnsList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.CONN_DNS_STREAMID, new Values((Object) generate(dnsSchema, DNSList),DNSList.size())); basicOutputCollector.emit(StreamAggregateConfig.CONN_DNS_STREAMID, new Values((generate(dnsSchema, dnsList)),dnsList.size()));
DNSList.clear(); dnsList.clear();
} }
if (!MAILList.isEmpty()){ if (!mailList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.CONN_MAIL_STREAMID, new Values((Object) generate(mailSchema, MAILList),MAILList.size())); basicOutputCollector.emit(StreamAggregateConfig.CONN_MAIL_STREAMID, new Values((generate(mailSchema, mailList)),mailList.size()));
MAILList.clear(); mailList.clear();
} }
if (!FTPList.isEmpty()){ if (!ftpList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.CONN_FTP_STREAMID, new Values((Object) generate(ftpSchema, FTPList),FTPList.size())); basicOutputCollector.emit(StreamAggregateConfig.CONN_FTP_STREAMID, new Values(generate(ftpSchema, ftpList),ftpList.size()));
FTPList.clear(); ftpList.clear();
} }
if (!JSTXList.isEmpty()){ if (!jstxList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.CONN_JSTX_STREAMID, new Values((Object) generate(jstxSchema, JSTXList),JSTXList.size())); basicOutputCollector.emit(StreamAggregateConfig.CONN_JSTX_STREAMID, new Values(generate(jstxSchema, jstxList),jstxList.size()));
JSTXList.clear(); jstxList.clear();
} }
}catch (Exception e){ }catch (Exception e){
e.printStackTrace(); e.printStackTrace();
logger.error("----------批数据发送异常-----------" + e); logger.error("----------批数据发送异常-----------" + e );
} }
basicOutputCollector.emit(StreamAggregateConfig.CONN_YBInputSize_STREAMID, new Values(connInputSize,count,loadFail,connOutputSize));
basicOutputCollector.emit(StreamAggregateConfig.CONN_YBInputSize_STREAMID, new Values(connInputSize,count,loadFail,connOutputSize,yys_error,ydy_inside,ydy_outside,ydy_unknown,mddy_inside,mddy_outside,mddy_unknown));
connInputSize = 0; connInputSize = 0;
count = 0; count = 0;
loadFail = 0; loadFail = 0;
connOutputSize = 0; connOutputSize = 0;
ydy_inside=0L;
mddy_inside=0L;
ydy_outside=0L;
mddy_outside=0L;
ydy_unknown=0L;
mddy_unknown=0L;
yys_error=0L;
}else { }else {
try { try {
String line = tuple.getString(0); String line = tuple.getString(0);
sessionRecordLog = JSONObject.parseObject(line, SessionRecordLog.class); sessionRecordLog = JSONObject.parseObject(line, SessionRecordLog.class);
if(StringUtil.isBlank(sessionRecordLog.getCommon_data_center())){
yys_error++;
logger.error("yys信息为空,请检查日志字段。"+"异常数据为"+line);
}
sessionRecordLog.setYys(getYys(sessionRecordLog.getCommon_data_center()));
sessionRecordLog.setYdy(getDy(sessionRecordLog.getCommon_client_location()));
sessionRecordLog.setMddy(getDy(sessionRecordLog.getCommon_server_location()));
if(sessionRecordLog.getYys()!=StreamAggregateConfig.YYS_NUM){
yys_error++;
logger.error("yys信息为空,请检查日志字段。"+"异常数据为"+line);
}
if (StringUtil.startsWith(sessionRecordLog.getYdy(),"65")&&sessionRecordLog.getYdy().length()==6){
ydy_inside++;
}else if (sessionRecordLog.getYdy().equals("0")){
ydy_unknown++;
}else {
ydy_outside++;
}
if (StringUtil.startsWith(sessionRecordLog.getMddy(),"65")&&sessionRecordLog.getMddy().length()==6){
mddy_inside++;
}else if(sessionRecordLog.getMddy().equals("0")){
mddy_unknown++;
}else {
mddy_outside++;
}
connInputSize = connInputSize + sessionRecordLog.getCommon_s2c_byte_num(); connInputSize = connInputSize + sessionRecordLog.getCommon_s2c_byte_num();
connOutputSize = connOutputSize + sessionRecordLog.getCommon_c2s_byte_num(); connOutputSize = connOutputSize + sessionRecordLog.getCommon_c2s_byte_num();
count = count + 1;
//按common_schema_type的值对数据进行分类发 //按common_schema_type的值对数据进行分类发
if (sessionRecordLog.getCommon_app_id() >= 100100) { if (sessionRecordLog.getCommon_app_label()!=null&&StringUtil.startsWith(sessionRecordLog.getCommon_app_label(),"05")) {
sessionRecordLog.setCommon_schema_type("JSTX"); sessionRecordLog.setCommon_schema_type("JSTX");
try { try {
count++;
GenericRecord jstx_rzzb = new GenericData.Record(jstxSchema); GenericRecord jstx_rzzb = new GenericData.Record(jstxSchema);
setJSTX_RZZB(jstx_rzzb, sessionRecordLog); setJSTX_RZZB(jstx_rzzb, sessionRecordLog);
JSTXList.add(jstx_rzzb); jstxList.add(jstx_rzzb);
if (JSTXList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (jstxList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.CONN_JSTX_STREAMID, new Values((Object) generate(jstxSchema, JSTXList), JSTXList.size())); basicOutputCollector.emit(StreamAggregateConfig.CONN_JSTX_STREAMID, new Values(generate(jstxSchema, jstxList), jstxList.size()));
JSTXList.clear(); jstxList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
logger.error("-----------jstx_rzzb数据封装处理异常-----------" + e); logger.error("-----------jstx_rzzb数据封装处理异常-----------" + e + "异常数据为:" + line);
loadFail++; loadFail++;
} }
} else { } else {
switch (sessionRecordLog.getCommon_schema_type()) { switch (sessionRecordLog.getCommon_schema_type()) {
case "BASE": case "BASE":
if (sessionRecordLog.getCommon_l7_protocol().equals("PPTP") || sessionRecordLog.getCommon_l7_protocol().equals("L2TP") || sessionRecordLog.getCommon_l7_protocol().equals("OPENVPN") || sessionRecordLog.getCommon_l7_protocol().equals("ISAKMP") if (sessionRecordLog.getCommon_l7_protocol().equals("PPTP") || sessionRecordLog.getCommon_l7_protocol().equals("L2TP") || sessionRecordLog.getCommon_l7_protocol().equals("OPENVPN") || sessionRecordLog.getCommon_l7_protocol().equals("ISAKMP")
|| sessionRecordLog.getCommon_protocol_label().contains("PPTP")||sessionRecordLog.getCommon_protocol_label().contains("L2TP")||sessionRecordLog.getCommon_protocol_label().contains("OPENVPN")||sessionRecordLog.getCommon_protocol_label().contains("IPSEC")){ || sessionRecordLog.getCommon_protocol_label().contains("PPTP")||sessionRecordLog.getCommon_protocol_label().contains("L2TP")||sessionRecordLog.getCommon_protocol_label().contains("OPENVPN")||sessionRecordLog.getCommon_protocol_label().contains("IPSEC")){
sessionRecordLog.setCommon_schema_type("VPN");
try { try {
count++;
sessionRecordLog.setCommon_schema_type("VPN");
GenericRecord vpn_rzzb = new GenericData.Record(vpnSchema); GenericRecord vpn_rzzb = new GenericData.Record(vpnSchema);
setVPN_RZZB(vpn_rzzb, sessionRecordLog); setVPN_RZZB(vpn_rzzb, sessionRecordLog);
VPNList.add(vpn_rzzb); vpnList.add(vpn_rzzb);
if (VPNList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (vpnList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.CONN_VPN_STREAMID, new Values((Object) generate(vpnSchema, VPNList), VPNList.size())); basicOutputCollector.emit(StreamAggregateConfig.CONN_VPN_STREAMID, new Values(generate(vpnSchema, vpnList), vpnList.size()));
VPNList.clear(); vpnList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
logger.error("-----------vpn_rzzb数据封装处理异常-----------" + e); logger.error("-----------vpn_rzzb数据封装处理异常-----------" + e + "异常数据为:" + line);
loadFail++; loadFail++;
} }
} else if (sessionRecordLog.getCommon_l7_protocol().equals("FTP")) {
try {
sessionRecordLog.setCommon_schema_type("FTP");
GenericRecord ftp_rzzb = new GenericData.Record(ftpSchema);
setFTP_RZZB(ftp_rzzb, sessionRecordLog);
FTPList.add(ftp_rzzb);
if (FTPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.CONN_FTP_STREAMID, new Values((Object) generate(ftpSchema, FTPList), FTPList.size()));
FTPList.clear();
}
} catch (Exception e) {
e.printStackTrace();
logger.error("-----------ftp_rzzb数据封装处理异常-----------" + e);
loadFail++;
}
} else { } else {
sessionRecordLog.setCommon_schema_type("IP");
try { try {
count++;
sessionRecordLog.setCommon_schema_type("IP");
GenericRecord ip_rzzb = new GenericData.Record(ipSchema); GenericRecord ip_rzzb = new GenericData.Record(ipSchema);
setIP_RZZB(ip_rzzb, sessionRecordLog); setIP_RZZB(ip_rzzb, sessionRecordLog);
IPList.add(ip_rzzb); ipList.add(ip_rzzb);
if (IPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (ipList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.CONN_IP_STREAMID, new Values((Object) generate(ipSchema, IPList), IPList.size())); basicOutputCollector.emit(StreamAggregateConfig.CONN_IP_STREAMID, new Values(generate(ipSchema, ipList), ipList.size()));
IPList.clear(); ipList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
logger.error("-----------ip_rzzb数据封装处理异常-----：------" + e); logger.error("-----------ip_rzzb数据封装处理异常-----：------" + e + "异常数据为:" + line);
loadFail++; loadFail++;
} }
} }
break; break;
case "FTP":
try {
count++;
GenericRecord ftp_rzzb = new GenericData.Record(ftpSchema);
setFTP_RZZB(ftp_rzzb, sessionRecordLog);
ftpList.add(ftp_rzzb);
if (ftpList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.CONN_FTP_STREAMID, new Values(generate(ftpSchema, ftpList), ftpList.size()));
ftpList.clear();
}
} catch (Exception e) {
e.printStackTrace();
logger.error("-----------ftp_rzzb数据封装处理异常-----------" + e + "异常数据为:" + line);
loadFail++;
}
case "HTTP": case "HTTP":
try { try {
if (sessionRecordLog.getHttp_response_body() != null) { if (sessionRecordLog.getHttp_response_body() != null) {
String[] split = sessionRecordLog.getHttp_response_body().split("/"); String[] split = sessionRecordLog.getHttp_response_body().split("/");
bucketName = split[split.length - 2]; hosBucketName = split[split.length - 2];
filename = split[split.length - 1]; fileName = split[split.length - 1];
basicOutputCollector.emit("conn-unstructured", new Values(bucketName, filename, StreamAggregateConfig.T_HTTP_RZZB)); basicOutputCollector.emit("conn-unstructured", new Values(hosBucketName, fileName, StreamAggregateConfig.T_HTTP_RZZB));
sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_HTTP_RZZB, filename)); sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_HTTP_RZZB, fileName));
} }
count++;
GenericRecord http_rzzb = new GenericData.Record(httpSchema); GenericRecord http_rzzb = new GenericData.Record(httpSchema);
sessionRecordLog.setCommon_schema_type("HTTP"); sessionRecordLog.setCommon_schema_type("HTTP");
setHTTP_RZZB(http_rzzb, sessionRecordLog); setHTTP_RZZB(http_rzzb, sessionRecordLog);
HTTPList.add(http_rzzb); httpList.add(http_rzzb);
if (HTTPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (httpList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.CONN_HTTP_STREAMID, new Values((Object) generate(httpSchema, HTTPList), HTTPList.size())); basicOutputCollector.emit(StreamAggregateConfig.CONN_HTTP_STREAMID, new Values(generate(httpSchema, httpList), httpList.size()));
HTTPList.clear(); httpList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
logger.error("-----------http_rzzb数据封装处理异常-----------" + e); logger.error("-----------http_rzzb数据封装处理异常-----------" + e +"异常数据为:" + line);
loadFail++; loadFail++;
} }
break; break;
case "SSL": case "SSL":
sessionRecordLog.setCommon_schema_type("SSL");
try { try {
count++;
sessionRecordLog.setCommon_schema_type("SSL");
GenericRecord ssl_rzzb = new GenericData.Record(sslSchema); GenericRecord ssl_rzzb = new GenericData.Record(sslSchema);
setSSL_RZZB(ssl_rzzb, sessionRecordLog); setSSL_RZZB(ssl_rzzb, sessionRecordLog);
SSLList.add(ssl_rzzb); sslList.add(ssl_rzzb);
if (SSLList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (sslList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.CONN_SSL_STREAMID, new Values((Object) generate(sslSchema, SSLList), SSLList.size())); basicOutputCollector.emit(StreamAggregateConfig.CONN_SSL_STREAMID, new Values(generate(sslSchema, sslList), sslList.size()));
SSLList.clear(); sslList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
logger.error("-----------ssl_rzzb数据封装处理异常-----------" + e); logger.error("-----------ssl_rzzb数据封装处理异常-----------" + e +"异常数据为:" + line);
loadFail++; loadFail++;
} }
break; break;
case "DNS": case "DNS":
sessionRecordLog.setCommon_schema_type("DNS");
try { try {
count++;
sessionRecordLog.setCommon_schema_type("DNS");
GenericRecord dns_rzzb = new GenericData.Record(dnsSchema); GenericRecord dns_rzzb = new GenericData.Record(dnsSchema);
setDNS_RZZB(dns_rzzb, sessionRecordLog); setDNS_RZZB(dns_rzzb, sessionRecordLog);
if (dns_rzzb.get("DNS_NAME")!=("")){ if (dns_rzzb.get("DNS_NAME")!=("")){
DNSList.add(dns_rzzb); dnsList.add(dns_rzzb);
}else { }else {
loadFail++; loadFail++;
} }
if (DNSList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (dnsList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.CONN_DNS_STREAMID, new Values((Object) generate(dnsSchema, DNSList), DNSList.size())); basicOutputCollector.emit(StreamAggregateConfig.CONN_DNS_STREAMID, new Values(generate(dnsSchema, dnsList), dnsList.size()));
DNSList.clear(); dnsList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
logger.error("-----------dns_rzzb数据封装处理异常-----------" + e); logger.error("-----------dns_rzzb数据封装处理异常-----------" + e + "异常数据为:" + line);
loadFail++; loadFail++;
} }
break; break;
case "MAIL": case "MAIL":
count++;
try { try {
if (sessionRecordLog.getHttp_response_body() != null) { if (sessionRecordLog.getHttp_response_body() != null) {
String[] split = sessionRecordLog.getHttp_response_body().split("/"); String[] split = sessionRecordLog.getHttp_response_body().split("/");
bucketName = split[split.length - 2]; hosBucketName = split[split.length - 2];
filename = split[split.length - 1]; fileName = split[split.length - 1];
basicOutputCollector.emit("conn-unstructured", new Values(bucketName, filename, StreamAggregateConfig.T_MAIL_RZZB)); basicOutputCollector.emit("conn-unstructured", new Values(hosBucketName, fileName, StreamAggregateConfig.T_MAIL_RZZB));
sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MAIL_RZZB, filename)); sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MAIL_RZZB, fileName));
} }
GenericRecord mail_rzzb = new GenericData.Record(mailSchema);
sessionRecordLog.setCommon_schema_type("MAIL"); sessionRecordLog.setCommon_schema_type("MAIL");
GenericRecord mail_rzzb = new GenericData.Record(mailSchema);
setMAIL_RZZB(mail_rzzb, sessionRecordLog); setMAIL_RZZB(mail_rzzb, sessionRecordLog);
MAILList.add(mail_rzzb); mailList.add(mail_rzzb);
if (MAILList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (mailList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.CONN_MAIL_STREAMID, new Values((Object) generate(mailSchema, MAILList), MAILList.size())); basicOutputCollector.emit(StreamAggregateConfig.CONN_MAIL_STREAMID, new Values(generate(mailSchema, mailList), mailList.size()));
MAILList.clear(); mailList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -277,9 +330,9 @@ public class ConnParseBolt extends BaseBasicBolt {
break; break;
} }
} }
} catch(Exception e){ } catch(Exception e){
logger.error("通联数据格式错误"+e); logger.error("通联数据格式错误"+ e );
loadFail++;
} }
} }
@@ -295,9 +348,8 @@ public class ConnParseBolt extends BaseBasicBolt {
outputFieldsDeclarer.declareStream(StreamAggregateConfig.CONN_JSTX_STREAMID, new Fields(StreamAggregateConfig.CONN_JSTX_STREAMID,"jstxcount")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.CONN_JSTX_STREAMID, new Fields(StreamAggregateConfig.CONN_JSTX_STREAMID,"jstxcount"));
outputFieldsDeclarer.declareStream(StreamAggregateConfig.CONN_FTP_STREAMID, new Fields(StreamAggregateConfig.CONN_FTP_STREAMID,"ftpcount")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.CONN_FTP_STREAMID, new Fields(StreamAggregateConfig.CONN_FTP_STREAMID,"ftpcount"));
outputFieldsDeclarer.declareStream(StreamAggregateConfig.CONN_SSL_STREAMID, new Fields(StreamAggregateConfig.CONN_SSL_STREAMID,"sslcount")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.CONN_SSL_STREAMID, new Fields(StreamAggregateConfig.CONN_SSL_STREAMID,"sslcount"));
outputFieldsDeclarer.declareStream(StreamAggregateConfig.CONN_YBInputSize_STREAMID, new Fields("connInputSize","count","loadFail","connOutputSize")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.CONN_YBInputSize_STREAMID, new Fields("connInputSize","count","loadFail","connOutputSize","yys_error","ydy_inside","ydy_outside","ydy_unknown","mddy_inside","mddy_outside","mddy_unknown"));
outputFieldsDeclarer.declareStream("conn-unstructured", new Fields("bucketName","filename","bucketname")); outputFieldsDeclarer.declareStream("conn-unstructured", new Fields("hosBucketName","fileName","s3BucketName"));
} }

1
src/main/java/cn/ac/iie/storm/bolt/conn/ConnSSLSendBolt.java
View File

@@ -12,7 +12,6 @@ import org.apache.storm.tuple.Values;
import org.apache.storm.utils.TupleUtils; import org.apache.storm.utils.TupleUtils;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*;

1
src/main/java/cn/ac/iie/storm/bolt/conn/ConnSendBolt.java
View File

@@ -12,7 +12,6 @@ import org.apache.storm.tuple.Values;
import org.apache.storm.utils.TupleUtils; import org.apache.storm.utils.TupleUtils;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*;

24
src/main/java/cn/ac/iie/storm/bolt/conn/ConnUnstructuredSendBolt.java
View File

@@ -8,31 +8,30 @@ import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Tuple;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import static cn.ac.iie.storm.utils.hos.HosService.httpPostObject; import static cn.ac.iie.storm.utils.hos.HosService.httpPostObject;
import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost; import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*;
public class ConnUnstructuredSendBolt extends BaseBasicBolt { public class ConnUnstructuredSendBolt extends BaseBasicBolt {
String bucketName = null; String hosBucketName = null;
String filename = null; String fileName = null;
String bucketname = null; String s3BucketName = null;
String s3Url; String s3Url;
private final static Logger logger = Logger.getLogger(ConnUnstructuredSendBolt.class); private final static Logger logger = Logger.getLogger(ConnUnstructuredSendBolt.class);
@Override @Override
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
try { try {
bucketName = tuple.getStringByField("bucketName"); hosBucketName = tuple.getStringByField("hosBucketName");
filename = tuple.getStringByField("filename"); fileName = tuple.getStringByField("fileName");
bucketname = tuple.getStringByField("bucketname"); s3BucketName = tuple.getStringByField("s3BucketName");
byte[] bytes = httpPostObject(bucketName, filename); byte[] bytes = httpPostObject(hosBucketName, fileName);
int connAnsyHttpPost =0; int connAnsyHttpPost =0;
if(bytes!=null) { if(bytes!=null) {
int i=0; int i=0;
while (i< StreamAggregateConfig.S3_SENT_COUNT) { while (i< StreamAggregateConfig.S3_SENT_COUNT) {
s3Url = getS3Url(bucketname, filename); s3Url = getS3Url(s3BucketName, fileName);
connAnsyHttpPost = getConnAnsyHttpPost(s3Url, bytes); connAnsyHttpPost = getConnAnsyHttpPost(s3Url, bytes);
if (connAnsyHttpPost==200){ if (connAnsyHttpPost==200){
break; break;
@@ -40,12 +39,11 @@ public class ConnUnstructuredSendBolt extends BaseBasicBolt {
i++; i++;
} }
if (connAnsyHttpPost!=200){ if (connAnsyHttpPost!=200){
logger.error("桶："+bucketName+"下的文件"+filename+"发送失败"); logger.error("桶："+hosBucketName+"下的文件"+fileName+"发送失败");
} }
} }
} catch (IOException e) { } catch (IOException e) {
logger.error("桶："+bucketName+"下的文件"+filename+"发送失败原因为："+e); logger.error("桶："+hosBucketName+"下的文件"+fileName+"发送失败原因为："+e);
} }
} }

2
src/main/java/cn/ac/iie/storm/bolt/gk/GKHTTPSendBolt.java
View File

@@ -12,14 +12,12 @@ import org.apache.storm.tuple.Values;
import org.apache.storm.utils.TupleUtils; import org.apache.storm.utils.TupleUtils;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*;
public class GKHTTPSendBolt extends BaseBasicBolt { public class GKHTTPSendBolt extends BaseBasicBolt {
private final static Logger logger = Logger.getLogger(GKHTTPSendBolt.class); private final static Logger logger = Logger.getLogger(GKHTTPSendBolt.class);
private long gk_http_success=0L; private long gk_http_success=0L;
private long gk_http_fail=0L; private long gk_http_fail=0L;
String postUrl; String postUrl;

1
src/main/java/cn/ac/iie/storm/bolt/gk/GKIPSendBolt.java
View File

@@ -12,7 +12,6 @@ import org.apache.storm.tuple.Values;
import org.apache.storm.utils.TupleUtils; import org.apache.storm.utils.TupleUtils;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*;

1
src/main/java/cn/ac/iie/storm/bolt/gk/GKSSLSendBolt.java
View File

@@ -12,7 +12,6 @@ import org.apache.storm.tuple.Values;
import org.apache.storm.utils.TupleUtils; import org.apache.storm.utils.TupleUtils;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*;

205
src/main/java/cn/ac/iie/storm/bolt/gk/GkParseBolt.java
View File

@@ -36,15 +36,15 @@ public class GkParseBolt extends BaseBasicBolt {
private SessionRecordLog sessionRecordLog; private SessionRecordLog sessionRecordLog;
private List<GenericRecord> IPList = new ArrayList<GenericRecord>(); private List<GenericRecord> ipList = new ArrayList<GenericRecord>();
private List<GenericRecord> HTTPList = new ArrayList<GenericRecord>(); private List<GenericRecord> httpList = new ArrayList<GenericRecord>();
private List<GenericRecord> FTPList = new ArrayList<GenericRecord>(); private List<GenericRecord> ftpList = new ArrayList<GenericRecord>();
private List<GenericRecord> SSLList = new ArrayList<GenericRecord>(); private List<GenericRecord> sslList = new ArrayList<GenericRecord>();
private List<GenericRecord> VPNList = new ArrayList<GenericRecord>(); private List<GenericRecord> vpnList = new ArrayList<GenericRecord>();
private List<GenericRecord> DNSList = new ArrayList<GenericRecord>(); private List<GenericRecord> dnsList = new ArrayList<GenericRecord>();
private List<GenericRecord> MAILList = new ArrayList<GenericRecord>(); private List<GenericRecord> mailList = new ArrayList<GenericRecord>();
private List<GenericRecord> JSTXList = new ArrayList<GenericRecord>(); private List<GenericRecord> jstxList = new ArrayList<GenericRecord>();
private List<GenericRecord> WYGJCList = new ArrayList<GenericRecord>(); private List<GenericRecord> wygjcList = new ArrayList<GenericRecord>();
private long count = 0L; private long count = 0L;
@@ -60,9 +60,10 @@ public class GkParseBolt extends BaseBasicBolt {
private Schema mailSchema; private Schema mailSchema;
private Schema jstxSchema; private Schema jstxSchema;
private Schema wygjcSchema; private Schema wygjcSchema;
private String line;
String bucketName = null; String hosBucketName = null;
String filename = null; String fileName = null;
@@ -83,42 +84,42 @@ public class GkParseBolt extends BaseBasicBolt {
//首先获取一条数据 //首先获取一条数据
if (TupleUtils.isTick(tuple)) { if (TupleUtils.isTick(tuple)) {
try { try {
if (!IPList.isEmpty()){ if (!ipList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.GK_IP_STREAMID, new Values((Object) generate(ipSchema, IPList),IPList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_IP_STREAMID, new Values(generate(ipSchema, ipList),ipList.size()));
IPList.clear(); ipList.clear();
} }
if (!HTTPList.isEmpty()){ if (!httpList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.GK_HTTP_STREAMID, new Values((Object) generate(httpSchema, HTTPList),HTTPList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_HTTP_STREAMID, new Values(generate(httpSchema, httpList),httpList.size()));
HTTPList.clear(); httpList.clear();
} }
if (!SSLList.isEmpty()){ if (!sslList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.GK_SSL_STREAMID, new Values((Object) generate(sslSchema, SSLList),SSLList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_SSL_STREAMID, new Values((generate(sslSchema, sslList)),sslList.size()));
SSLList.clear(); sslList.clear();
} }
if (!DNSList.isEmpty()){ if (!dnsList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.GK_DNS_STREAMID, new Values((Object) generate(dnsSchema, DNSList),DNSList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_DNS_STREAMID, new Values(generate(dnsSchema, dnsList),dnsList.size()));
DNSList.clear(); dnsList.clear();
} }
if (!(FTPList.isEmpty())) { if (!(ftpList.isEmpty())) {
basicOutputCollector.emit(StreamAggregateConfig.GK_FTP_STREAMID, new Values((Object) generate(ftpSchema, FTPList),FTPList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_FTP_STREAMID, new Values(generate(ftpSchema, ftpList),ftpList.size()));
FTPList.clear(); ftpList.clear();
} }
if (!(WYGJCList.isEmpty())) { if (!(wygjcList.isEmpty())) {
basicOutputCollector.emit(StreamAggregateConfig.GK_WYGJC_STREAMID, new Values((Object) generate(wygjcSchema, WYGJCList),WYGJCList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_WYGJC_STREAMID, new Values(generate(wygjcSchema, wygjcList),wygjcList.size()));
WYGJCList.clear(); wygjcList.clear();
} }
if (!(JSTXList.isEmpty())) { if (!(wygjcList.isEmpty())) {
basicOutputCollector.emit(StreamAggregateConfig.GK_JSTX_STREAMID, new Values((Object) generate(jstxSchema, JSTXList),JSTXList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_JSTX_STREAMID, new Values(generate(jstxSchema, wygjcList),wygjcList.size()));
JSTXList.clear(); wygjcList.clear();
} }
if (!VPNList.isEmpty()) { if (!vpnList.isEmpty()) {
basicOutputCollector.emit(StreamAggregateConfig.GK_VPN_STREAMID, new Values((Object) generate(vpnSchema, VPNList),VPNList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_VPN_STREAMID, new Values(generate(vpnSchema, vpnList),vpnList.size()));
VPNList.clear(); vpnList.clear();
} }
if (!(MAILList.isEmpty())) { if (!(mailList.isEmpty())) {
basicOutputCollector.emit(StreamAggregateConfig.GK_MAIL_STREAMID, new Values((Object) generate(mailSchema, MAILList),MAILList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_MAIL_STREAMID, new Values(generate(mailSchema, mailList),mailList.size()));
MAILList.clear(); mailList.clear();
} }
basicOutputCollector.emit(StreamAggregateConfig.GK_COUNT_STREAMID, new Values(count,loadfail)); basicOutputCollector.emit(StreamAggregateConfig.GK_COUNT_STREAMID, new Values(count,loadfail));
@@ -130,10 +131,11 @@ public class GkParseBolt extends BaseBasicBolt {
loadfail = 0; loadfail = 0;
} else { } else {
try { try {
String line = tuple.getString(0); line = tuple.getString(0);
sessionRecordLog = JSONObject.parseObject(line, SessionRecordLog.class); sessionRecordLog = JSONObject.parseObject(line, SessionRecordLog.class);
UserRegion userRegion = JSONObject.parseObject(sessionRecordLog.getCommon_user_region(), UserRegion.class); UserRegion userRegion = JSONObject.parseObject(sessionRecordLog.getCommon_user_region(), UserRegion.class);
Traffic_Forward traffic_forward = JSONObject.parseObject(userRegion.getTraffic_forward(), Traffic_Forward.class); Traffic_Forward traffic_forward = JSONObject.parseObject(userRegion.getTraffic_forward(), Traffic_Forward.class);
sessionRecordLog.setPzid(traffic_forward.getPzid()); sessionRecordLog.setPzid(traffic_forward.getPzid());
sessionRecordLog.setFlid(traffic_forward.getFlid()); sessionRecordLog.setFlid(traffic_forward.getFlid());
sessionRecordLog.setXzid(traffic_forward.getXzid()); sessionRecordLog.setXzid(traffic_forward.getXzid());
@@ -141,11 +143,10 @@ public class GkParseBolt extends BaseBasicBolt {
sessionRecordLog.setPzlx(traffic_forward.getPzlx()); sessionRecordLog.setPzlx(traffic_forward.getPzlx());
sessionRecordLog.setGklb(traffic_forward.getGklb()); sessionRecordLog.setGklb(traffic_forward.getGklb());
sessionRecordLog.setLabels(traffic_forward.getLabels()); sessionRecordLog.setLabels(traffic_forward.getLabels());
sessionRecordLog.setYyzl(traffic_forward.getYyzl());
sessionRecordLog.setGjc(traffic_forward.getGjc()); // sessionRecordLog.setZsmc(traffic_forward.getZsmc());
sessionRecordLog.setZsmc(traffic_forward.getZsmc()); // sessionRecordLog.setYjxy(traffic_forward.getYjxy());
sessionRecordLog.setYjxy(traffic_forward.getYjxy());
sessionRecordLog.setProtocol(userRegion.getProtocol());
int pzlx = traffic_forward.getPzlx(); int pzlx = traffic_forward.getPzlx();
//按pzlx的值对数据进行分类发送 //按pzlx的值对数据进行分类发送
//按pzlx的值对数据进行分类发送 //按pzlx的值对数据进行分类发送
@@ -160,10 +161,10 @@ public class GkParseBolt extends BaseBasicBolt {
GenericRecord ip_gkrz = new GenericData.Record(ipSchema); GenericRecord ip_gkrz = new GenericData.Record(ipSchema);
sessionRecordLog.setCommon_schema_type("IP"); sessionRecordLog.setCommon_schema_type("IP");
setIP_GKRZ(ip_gkrz, sessionRecordLog); setIP_GKRZ(ip_gkrz, sessionRecordLog);
IPList.add(ip_gkrz); ipList.add(ip_gkrz);
if (IPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (ipList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.GK_IP_STREAMID, new Values((Object) generate(ipSchema, IPList),IPList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_IP_STREAMID, new Values((Object) generate(ipSchema, ipList),ipList.size()));
IPList.clear(); ipList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -181,13 +182,13 @@ public class GkParseBolt extends BaseBasicBolt {
sessionRecordLog.setCommon_schema_type("DNS"); sessionRecordLog.setCommon_schema_type("DNS");
setDNS_GKZB(dns_gkzb, sessionRecordLog); setDNS_GKZB(dns_gkzb, sessionRecordLog);
if (dns_gkzb.get("DNS_NAME")!=("")){ if (dns_gkzb.get("DNS_NAME")!=("")){
DNSList.add(dns_gkzb); dnsList.add(dns_gkzb);
}else { }else {
loadfail++; loadfail++;
} }
if (DNSList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (dnsList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.GK_DNS_STREAMID, new Values((Object) generate(dnsSchema, DNSList),DNSList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_DNS_STREAMID, new Values(generate(dnsSchema, dnsList),dnsList.size()));
DNSList.clear(); dnsList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -201,20 +202,21 @@ public class GkParseBolt extends BaseBasicBolt {
case 703: //反诈URL管控 case 703: //反诈URL管控
count++; count++;
try { try {
sessionRecordLog.setYyzl(traffic_forward.getYyzl());
if (sessionRecordLog.getHttp_response_body()!=null){ if (sessionRecordLog.getHttp_response_body()!=null){
String[] split = sessionRecordLog.getHttp_response_body().split("/"); String[] split = sessionRecordLog.getHttp_response_body().split("/");
bucketName = split[split.length-2]; hosBucketName = split[split.length-2];
filename = split[split.length-1]; fileName = split[split.length-1];
basicOutputCollector.emit("gk-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_MAIL_GKZB)); basicOutputCollector.emit("gk-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_MAIL_GKZB));
sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_HTTP_GKZB, filename)); sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_HTTP_GKZB, fileName));
} }
GenericRecord http_gkzb = new GenericData.Record(httpSchema); GenericRecord http_gkzb = new GenericData.Record(httpSchema);
sessionRecordLog.setCommon_schema_type("HTTP"); sessionRecordLog.setCommon_schema_type("HTTP");
setHTTP_GKZB(http_gkzb, sessionRecordLog); setHTTP_GKZB(http_gkzb, sessionRecordLog);
HTTPList.add(http_gkzb); httpList.add(http_gkzb);
if (HTTPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (httpList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.GK_HTTP_STREAMID, new Values((Object) generate(httpSchema, HTTPList),HTTPList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_HTTP_STREAMID, new Values(generate(httpSchema, httpList),httpList.size()));
HTTPList.clear(); httpList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -225,13 +227,14 @@ public class GkParseBolt extends BaseBasicBolt {
case 405: //特定证书封堵 case 405: //特定证书封堵
count++; count++;
try { try {
sessionRecordLog.setZsmc(traffic_forward.getZsmc());
GenericRecord ssl_gkzb = new GenericData.Record(sslSchema); GenericRecord ssl_gkzb = new GenericData.Record(sslSchema);
sessionRecordLog.setCommon_schema_type("SSL"); sessionRecordLog.setCommon_schema_type("SSL");
setSSL_GKZB(ssl_gkzb, sessionRecordLog); setSSL_GKZB(ssl_gkzb, sessionRecordLog);
SSLList.add(ssl_gkzb); sslList.add(ssl_gkzb);
if (SSLList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (sslList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.GK_SSL_STREAMID, new Values((Object) generate(sslSchema, SSLList),SSLList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_SSL_STREAMID, new Values(generate(sslSchema, sslList),sslList.size()));
SSLList.clear(); sslList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
@@ -245,21 +248,22 @@ public class GkParseBolt extends BaseBasicBolt {
case 409: //搜索词封堵 case 409: //搜索词封堵
count++; count++;
try { try {
sessionRecordLog.setYyzl(traffic_forward.getYyzl());
sessionRecordLog.setGjc(traffic_forward.getGjc());
if (sessionRecordLog.getHttp_response_body()!=null){ if (sessionRecordLog.getHttp_response_body()!=null){
String[] split = sessionRecordLog.getHttp_response_body().split("/"); String[] split = sessionRecordLog.getHttp_response_body().split("/");
bucketName = split[split.length-2]; hosBucketName = split[split.length-2];
filename = split[split.length-1]; fileName = split[split.length-1];
basicOutputCollector.emit("gk-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_MAIL_GKZB)); basicOutputCollector.emit("gk-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_MAIL_GKZB));
sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_WYGJC_GKRZ, filename)); sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_WYGJC_GKRZ, fileName));
} }
GenericRecord wygjc_gkzb = new GenericData.Record(wygjcSchema); GenericRecord wygjc_gkzb = new GenericData.Record(wygjcSchema);
sessionRecordLog.setCommon_schema_type("HTTP"); sessionRecordLog.setCommon_schema_type("HTTP");
setWYGJC_GKZB(wygjc_gkzb, sessionRecordLog); setWYGJC_GKZB(wygjc_gkzb, sessionRecordLog);
WYGJCList.add(wygjc_gkzb); wygjcList.add(wygjc_gkzb);
if (WYGJCList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (wygjcList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.GK_WYGJC_STREAMID, new Values((Object) generate(wygjcSchema, WYGJCList),WYGJCList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_WYGJC_STREAMID, new Values(generate(wygjcSchema, wygjcList),wygjcList.size()));
WYGJCList.clear(); wygjcList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -271,20 +275,23 @@ public class GkParseBolt extends BaseBasicBolt {
case 410: case 410:
count++; count++;
try { try {
sessionRecordLog.setYjxy(traffic_forward.getYjxy());
sessionRecordLog.setYyzl(traffic_forward.getYyzl());
sessionRecordLog.setGjc(traffic_forward.getGjc());
if (sessionRecordLog.getHttp_response_body()!=null){ if (sessionRecordLog.getHttp_response_body()!=null){
String[] split = sessionRecordLog.getHttp_response_body().split("/"); String[] split = sessionRecordLog.getHttp_response_body().split("/");
bucketName = split[split.length-2]; hosBucketName = split[split.length-2];
filename = split[split.length-1]; fileName = split[split.length-1];
basicOutputCollector.emit("gk-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_MAIL_GKZB)); basicOutputCollector.emit("gk-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_MAIL_GKZB));
sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MAIL_GKZB, filename)); sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MAIL_GKZB, fileName));
} }
GenericRecord mail_gkzb = new GenericData.Record(mailSchema); GenericRecord mail_gkzb = new GenericData.Record(mailSchema);
sessionRecordLog.setCommon_schema_type("MAIL"); sessionRecordLog.setCommon_schema_type("MAIL");
setMAIL_GKZB(mail_gkzb, sessionRecordLog); setMAIL_GKZB(mail_gkzb, sessionRecordLog);
MAILList.add(mail_gkzb); mailList.add(mail_gkzb);
if (MAILList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (mailList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.GK_MAIL_STREAMID, new Values((Object) generate(mailSchema, MAILList),MAILList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_MAIL_STREAMID, new Values(generate(mailSchema, mailList),mailList.size()));
MAILList.clear(); mailList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -295,13 +302,15 @@ public class GkParseBolt extends BaseBasicBolt {
case 408: //FTP关键字封堵 case 408: //FTP关键字封堵
count++; count++;
try { try {
sessionRecordLog.setYyzl(traffic_forward.getYyzl());
sessionRecordLog.setGjc(traffic_forward.getGjc());
GenericRecord ftp_gkzb = new GenericData.Record(ftpSchema); GenericRecord ftp_gkzb = new GenericData.Record(ftpSchema);
sessionRecordLog.setCommon_schema_type("FTP"); sessionRecordLog.setCommon_schema_type("FTP");
setFTP_GKZB(ftp_gkzb, sessionRecordLog); setFTP_GKZB(ftp_gkzb, sessionRecordLog);
FTPList.add(ftp_gkzb); ftpList.add(ftp_gkzb);
if (FTPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (ftpList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.GK_FTP_STREAMID, new Values((Object) generate(ftpSchema, FTPList),FTPList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_FTP_STREAMID, new Values(generate(ftpSchema, ftpList),ftpList.size()));
FTPList.clear(); ftpList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -313,14 +322,18 @@ public class GkParseBolt extends BaseBasicBolt {
count++; count++;
try { try {
GenericRecord vpn_gkzb = new GenericData.Record(vpnSchema); GenericRecord vpn_gkzb = new GenericData.Record(vpnSchema);
sessionRecordLog.setLx(traffic_forward.getLx());
sessionRecordLog.setCommon_schema_type("VPN"); sessionRecordLog.setCommon_schema_type("VPN");
setVPN_GKZB(vpn_gkzb, sessionRecordLog); setVPN_GKZB(vpn_gkzb, sessionRecordLog);
if(StringUtil.isNotEmpty(vpn_gkzb.get("DETAIL_ID"))) { if(StringUtil.isNotEmpty(vpn_gkzb.get("DETAIL_ID"))) {
VPNList.add(vpn_gkzb); vpnList.add(vpn_gkzb);
}else {
logger.error("vpnjc流量DETAIL_ID为空具体日志为:"+line);
loadfail++;
} }
if (VPNList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (vpnList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.GK_VPN_STREAMID, new Values((Object) generate(vpnSchema, VPNList),VPNList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_VPN_STREAMID, new Values(generate(vpnSchema, vpnList),vpnList.size()));
VPNList.clear(); vpnList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -332,13 +345,15 @@ public class GkParseBolt extends BaseBasicBolt {
case 413: //社交应用封堵 case 413: //社交应用封堵
count++; count++;
try { try {
sessionRecordLog.setYyzl(traffic_forward.getYyzl());
sessionRecordLog.setProtocol(userRegion.getProtocol());
GenericRecord jstx_gkzb = new GenericData.Record(jstxSchema); GenericRecord jstx_gkzb = new GenericData.Record(jstxSchema);
sessionRecordLog.setCommon_schema_type("JSTX"); sessionRecordLog.setCommon_schema_type("JSTX");
setJSTX_GKZB(jstx_gkzb, sessionRecordLog); setJSTX_GKZB(jstx_gkzb, sessionRecordLog);
JSTXList.add(jstx_gkzb); jstxList.add(jstx_gkzb);
if (JSTXList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (jstxList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.GK_JSTX_STREAMID, new Values((Object) generate(jstxSchema, JSTXList),JSTXList.size())); basicOutputCollector.emit(StreamAggregateConfig.GK_JSTX_STREAMID, new Values(generate(jstxSchema, jstxList),jstxList.size()));
JSTXList.clear(); jstxList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -350,7 +365,7 @@ public class GkParseBolt extends BaseBasicBolt {
break; break;
} }
} catch(Exception e){ } catch(Exception e){
logger.error("GK数据处理异常"+e); logger.error("GK数据处理异常"+e+"异常数据"+line);
} }
} }
} }
@@ -367,7 +382,7 @@ public class GkParseBolt extends BaseBasicBolt {
outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_VPN_STREAMID,new Fields(StreamAggregateConfig.GK_VPN_STREAMID,"vpncount")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_VPN_STREAMID,new Fields(StreamAggregateConfig.GK_VPN_STREAMID,"vpncount"));
outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_JSTX_STREAMID,new Fields(StreamAggregateConfig.GK_JSTX_STREAMID,"jstxcount")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_JSTX_STREAMID,new Fields(StreamAggregateConfig.GK_JSTX_STREAMID,"jstxcount"));
outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_COUNT_STREAMID, new Fields("count","loadfail")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_COUNT_STREAMID, new Fields("count","loadfail"));
outputFieldsDeclarer.declareStream("gk-unstructured", new Fields("bucketName","filename","bucketname")); outputFieldsDeclarer.declareStream("gk-unstructured", new Fields("hosBucketName","fileName","s3BucketName"));
} }
@Override @Override

1
src/main/java/cn/ac/iie/storm/bolt/gk/GkSendBolt.java
View File

@@ -12,7 +12,6 @@ import org.apache.storm.tuple.Values;
import org.apache.storm.utils.TupleUtils; import org.apache.storm.utils.TupleUtils;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*;

21
src/main/java/cn/ac/iie/storm/bolt/gk/GkUnstructuredSendBolt.java
View File

@@ -9,32 +9,31 @@ import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Tuple;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import static cn.ac.iie.storm.utils.hos.HosService.httpPostObject; import static cn.ac.iie.storm.utils.hos.HosService.httpPostObject;
import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost; import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*;
public class GkUnstructuredSendBolt extends BaseBasicBolt { public class GkUnstructuredSendBolt extends BaseBasicBolt {
String bucketName = null; String hosBucketName = null;
String filename = null; String fileName = null;
String bucketname = null; String s3BucketName = null;
byte[] bytes; byte[] bytes;
private final static Logger logger = Logger.getLogger(GkUnstructuredSendBolt.class); private final static Logger logger = Logger.getLogger(GkUnstructuredSendBolt.class);
@Override @Override
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
try { try {
bucketName = tuple.getStringByField("bucketName"); hosBucketName = tuple.getStringByField("hosBucketName");
filename = tuple.getStringByField("filename"); fileName = tuple.getStringByField("fileName");
bucketname = tuple.getStringByField("bucketname"); s3BucketName = tuple.getStringByField("s3BucketName");
bytes = httpPostObject(bucketName, filename); bytes = httpPostObject(hosBucketName, fileName);
int connAnsyHttpPost =0; int connAnsyHttpPost =0;
String s3Url; String s3Url;
if(bytes!=null) { if(bytes!=null) {
int i = 0; int i = 0;
String ip; String ip;
while (i < StreamAggregateConfig.S3_SENT_COUNT) { while (i < StreamAggregateConfig.S3_SENT_COUNT) {
s3Url = getS3Url(bucketname, filename); s3Url = getS3Url(s3BucketName, fileName);
connAnsyHttpPost = getConnAnsyHttpPost(s3Url, bytes); connAnsyHttpPost = getConnAnsyHttpPost(s3Url, bytes);
if (connAnsyHttpPost == 200) { if (connAnsyHttpPost == 200) {
break; break;
@@ -42,11 +41,11 @@ public class GkUnstructuredSendBolt extends BaseBasicBolt {
i++; i++;
} }
if (connAnsyHttpPost != 200) { if (connAnsyHttpPost != 200) {
logger.error("桶：" + bucketName + "下的文件" + filename + "发送失败"); logger.error("桶：" + hosBucketName + "下的文件" + fileName + "发送失败");
} }
} }
} catch (IOException e) { } catch (IOException e) {
logger.error("桶："+bucketName+"下的文件"+filename+"发送失败原因为："+e); logger.error("桶："+hosBucketName+"下的文件"+fileName+"发送失败原因为："+e);
} }
} }

71
src/main/java/cn/ac/iie/storm/bolt/index/ConnCountBolt.java
View File

@@ -46,10 +46,21 @@ public class ConnCountBolt extends BaseBasicBolt {
private long conn_ssl_success = 0L; private long conn_ssl_success = 0L;
private long conn_ssl_fail = 0L; private long conn_ssl_fail = 0L;
InetAddress ip=null;
private String localip;
private float inputrate; private float inputrate;
private float outputrate; private float outputrate;
private long yys_error=0L;
private long ydy_inside=0L;
private long mddy_inside=0L;
private long ydy_outside=0L;
private long mddy_outside=0L;
private long ydy_unknown=0L;
private long mddy_unknown=0L;
static final Gauge InputRate = Gauge.build() static final Gauge InputRate = Gauge.build()
.name("InputRate").labelNames("ServName","Duration").help("Program receiving flow rate, the unit is Mbps").register(); .name("InputRate").labelNames("ServName","Duration").help("Program receiving flow rate, the unit is Mbps").register();
@@ -119,6 +130,30 @@ public class ConnCountBolt extends BaseBasicBolt {
.name("connParseFail").labelNames("ServName","Duration").help("Program output flow rate, the unit is Mbps").register(); .name("connParseFail").labelNames("ServName","Duration").help("Program output flow rate, the unit is Mbps").register();
static final Gauge ConnYysError = Gauge.build()
.name("ConnYysError").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register();
static final Gauge ConnYdyInside = Gauge.build()
.name("ConnYdyInside").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register();
static final Gauge ConnYdyOutside = Gauge.build()
.name("ConnYdyOutside").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register();
static final Gauge ConnYdyUnknown = Gauge.build()
.name("ConnYdyUnknown").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register();
static final Gauge ConnMddyInside = Gauge.build()
.name("ConnMddyInside").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register();
static final Gauge ConnMddyOutside = Gauge.build()
.name("ConnmddyOutside").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register();
static final Gauge ConnMddyUnknown = Gauge.build()
.name("ConnMddyUnknown").labelNames("ServName","Duration").help("The general intput log volume, the unit is slip").register();
@Override @Override
public void execute(Tuple input, BasicOutputCollector collector) { public void execute(Tuple input, BasicOutputCollector collector) {
@@ -162,6 +197,16 @@ public class ConnCountBolt extends BaseBasicBolt {
connFtpSuccess.labels("conn-ftp-success",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(conn_ftp_success); connFtpSuccess.labels("conn-ftp-success",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(conn_ftp_success);
connFtpFail.labels("conn-ftp-fail",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(conn_ftp_fail); connFtpFail.labels("conn-ftp-fail",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(conn_ftp_fail);
ConnYysError.labels("conn-yys-error",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(yys_error);
ConnYdyInside.labels("conn-ydy-inside",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(ydy_inside);
ConnMddyInside.labels("conn-mddy-inside",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(mddy_inside);
ConnYdyOutside.labels("conn-ydy-outside",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(ydy_outside);
ConnMddyOutside.labels("conn-mddy-outside",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(mddy_outside);
ConnYdyUnknown.labels("conn-ydy-unknow",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(ydy_unknown);
ConnMddyUnknown.labels("conn-mddy-unknow",String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(mddy_unknown);
try { try {
@@ -169,6 +214,9 @@ public class ConnCountBolt extends BaseBasicBolt {
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
inputrate = 0L;
outputrate = 0L;
connCount= 0L; connCount= 0L;
ConnSendCount = 0L; ConnSendCount = 0L;
InputSize = 0L; InputSize = 0L;
@@ -191,17 +239,31 @@ public class ConnCountBolt extends BaseBasicBolt {
conn_ftp_fail=0L; conn_ftp_fail=0L;
conn_ssl_success = 0L; conn_ssl_success = 0L;
conn_ssl_fail = 0L; conn_ssl_fail = 0L;
yys_error=0L;
ydy_inside=0L;
mddy_inside=0L;
ydy_outside=0L;
mddy_outside=0L;
ydy_unknown=0L;
mddy_unknown=0L;
}else { }else {
String sourceComponent = input.getSourceComponent(); String sourceComponent = input.getSourceComponent();
switch (sourceComponent){ switch (sourceComponent){
case "CONN-PARSE-COMPONENTID": case "CONN-PARSE-COMPONENTID":
if (input.getSourceStreamId().equals(StreamAggregateConfig.CONN_YBInputSize_STREAMID)){ if (input.getSourceStreamId().equals(StreamAggregateConfig.CONN_YBInputSize_STREAMID)){
InputSize = InputSize + input.getLongByField("connInputSize"); InputSize = InputSize + input.getLongByField("connInputSize");
connCount = connCount + input.getLongByField("count"); connCount = connCount + input.getLongByField("count");
connParseFail = connParseFail + input.getLongByField("loadFail"); connParseFail = connParseFail + input.getLongByField("loadFail");
OutputSize = OutputSize + input.getLongByField("connOutputSize"); OutputSize = OutputSize + input.getLongByField("connOutputSize");
yys_error = yys_error + input.getLongByField("yys_error");
ydy_inside = ydy_inside + input.getLongByField("ydy_inside");
ydy_outside = ydy_outside + input.getLongByField("ydy_outside");
ydy_unknown = ydy_unknown + input.getLongByField("ydy_unknown");
mddy_inside = mddy_inside + input.getLongByField("mddy_inside");
mddy_outside = mddy_outside + input.getLongByField("mddy_outside");
mddy_unknown = mddy_unknown + input.getLongByField("mddy_unknown");
} }
break; break;
case "CONN-SEND": case "CONN-SEND":
@@ -217,9 +279,6 @@ public class ConnCountBolt extends BaseBasicBolt {
conn_jstx_success =conn_jstx_success + input.getLongByField("conn_jstx_success"); conn_jstx_success =conn_jstx_success + input.getLongByField("conn_jstx_success");
conn_jstx_fail =conn_jstx_fail + input.getLongByField("conn_jstx_fail"); conn_jstx_fail =conn_jstx_fail + input.getLongByField("conn_jstx_fail");
conn_jstx_success =conn_jstx_success + input.getLongByField("conn_jstx_success");
conn_jstx_fail =conn_jstx_fail + input.getLongByField("conn_jstx_fail");
conn_ftp_success =conn_ftp_success + input.getLongByField("conn_ftp_success"); conn_ftp_success =conn_ftp_success + input.getLongByField("conn_ftp_success");
conn_ftp_fail =conn_ftp_fail + input.getLongByField("conn_ftp_fail"); conn_ftp_fail =conn_ftp_fail + input.getLongByField("conn_ftp_fail");
break; break;

2
src/main/java/cn/ac/iie/storm/bolt/index/GkCountBolt.java
View File

@@ -213,8 +213,6 @@ public class GkCountBolt extends BaseBasicBolt {
gk_jstx_success = gk_jstx_success + tuple.getLongByField("gk_jstx_success"); gk_jstx_success = gk_jstx_success + tuple.getLongByField("gk_jstx_success");
gk_jstx_fail = gk_jstx_fail + tuple.getLongByField("gk_jstx_fail"); gk_jstx_fail = gk_jstx_fail + tuple.getLongByField("gk_jstx_fail");
gk_jstx_success = gk_jstx_success + tuple.getLongByField("gk_jstx_success");
gk_jstx_fail = gk_jstx_fail + tuple.getLongByField("gk_jstx_fail");
gk_ftp_success = gk_ftp_success + tuple.getLongByField("gk_ftp_success"); gk_ftp_success = gk_ftp_success + tuple.getLongByField("gk_ftp_success");
gk_ftp_fail = gk_ftp_fail + tuple.getLongByField("gk_ftp_fail"); gk_ftp_fail = gk_ftp_fail + tuple.getLongByField("gk_ftp_fail");

3
src/main/java/cn/ac/iie/storm/bolt/jc/JCHTTPSendBolt.java
View File

@@ -1,6 +1,6 @@
package cn.ac.iie.storm.bolt.jc; package cn.ac.iie.storm.bolt.jc;
import cn.ac.iie.storm.bolt.gk.GKHTTPSendBolt;
import cn.ac.iie.storm.utils.config.StreamAggregateConfig; import cn.ac.iie.storm.utils.config.StreamAggregateConfig;
import cn.ac.iie.storm.utils.http.HttpClientService; import cn.ac.iie.storm.utils.http.HttpClientService;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
@@ -13,7 +13,6 @@ import org.apache.storm.tuple.Values;
import org.apache.storm.utils.TupleUtils; import org.apache.storm.utils.TupleUtils;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*;

294
src/main/java/cn/ac/iie/storm/bolt/jc/JCParseBolt.java
View File

@@ -26,7 +26,6 @@ import java.util.Map;
import static cn.ac.iie.storm.utils.general.JCAssignment.*; import static cn.ac.iie.storm.utils.general.JCAssignment.*;
import static cn.ac.iie.storm.utils.general.TransFormUtils.generate; import static cn.ac.iie.storm.utils.general.TransFormUtils.generate;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.getS3Url;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.getYL5; import static cn.ac.iie.storm.utils.http.HttpGetUrl.getYL5;
public class JCParseBolt extends BaseBasicBolt { public class JCParseBolt extends BaseBasicBolt {
@@ -38,17 +37,17 @@ public class JCParseBolt extends BaseBasicBolt {
private List<GenericRecord> IPList = new ArrayList<GenericRecord>(); private List<GenericRecord> ipList = new ArrayList<GenericRecord>();
private List<GenericRecord> HTTPList = new ArrayList<GenericRecord>(); private List<GenericRecord> httpList = new ArrayList<GenericRecord>();
private List<GenericRecord> FTPList = new ArrayList<GenericRecord>(); private List<GenericRecord> ftpList = new ArrayList<GenericRecord>();
private List<GenericRecord> SSLList = new ArrayList<GenericRecord>(); private List<GenericRecord> sslList = new ArrayList<GenericRecord>();
private List<GenericRecord> VPNList = new ArrayList<GenericRecord>(); private List<GenericRecord> vpnList = new ArrayList<GenericRecord>();
private List<GenericRecord> DNSList = new ArrayList<GenericRecord>(); private List<GenericRecord> dnsList = new ArrayList<GenericRecord>();
private List<GenericRecord> MAILList = new ArrayList<GenericRecord>(); private List<GenericRecord> mailList = new ArrayList<GenericRecord>();
private List<GenericRecord> JSTXList = new ArrayList<GenericRecord>(); private List<GenericRecord> jstxList = new ArrayList<GenericRecord>();
private List<GenericRecord> WYGJCList = new ArrayList<GenericRecord>(); private List<GenericRecord> wygjcList = new ArrayList<GenericRecord>();
private List<GenericRecord> PICList = new ArrayList<GenericRecord>(); private List<GenericRecord> picList = new ArrayList<GenericRecord>();
private List<GenericRecord> MEDIAList = new ArrayList<GenericRecord>(); private List<GenericRecord> mediaList = new ArrayList<GenericRecord>();
private long count = 0L; private long count = 0L;
private long loadfail = 0L; private long loadfail = 0L;
private Schema vpnSchema; private Schema vpnSchema;
@@ -62,9 +61,12 @@ public class JCParseBolt extends BaseBasicBolt {
private Schema wygjcSchema; private Schema wygjcSchema;
private Schema picSchema; private Schema picSchema;
private Schema mediaSchema; private Schema mediaSchema;
private String line;
String bucketName = null;
String filename = null;
String hosBucketName = null;
String fileName = null;
@@ -88,61 +90,61 @@ public class JCParseBolt extends BaseBasicBolt {
//首先获取一条数据 //首先获取一条数据
if (TupleUtils.isTick(tuple)) { if (TupleUtils.isTick(tuple)) {
try { try {
if (!IPList.isEmpty()){ if (!ipList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.JC_IP_STREAMID, new Values((Object) generate(ipSchema, IPList),IPList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_IP_STREAMID, new Values(generate(ipSchema, ipList),ipList.size()));
IPList.clear(); ipList.clear();
} }
if (!HTTPList.isEmpty()){ if (!httpList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.JC_HTTP_STREAMID, new Values((Object) generate(httpSchema, HTTPList),HTTPList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_HTTP_STREAMID, new Values(generate(httpSchema, httpList),httpList.size()));
HTTPList.clear(); httpList.clear();
} }
if (!SSLList.isEmpty()){ if (!sslList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.JC_SSL_STREAMID, new Values((Object) generate(sslSchema, SSLList),SSLList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_SSL_STREAMID, new Values(generate(sslSchema, sslList),sslList.size()));
SSLList.clear(); sslList.clear();
} }
if (!DNSList.isEmpty()){ if (!dnsList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.JC_DNS_STREAMID, new Values((Object) generate(dnsSchema, DNSList),DNSList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_DNS_STREAMID, new Values(generate(dnsSchema, dnsList),dnsList.size()));
DNSList.clear(); dnsList.clear();
} }
if (!(WYGJCList.isEmpty())) { if (!(wygjcList.isEmpty())) {
basicOutputCollector.emit(StreamAggregateConfig.JC_WYGJC_STREAMID, new Values((Object) generate(wygjcSchema,WYGJCList),WYGJCList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_WYGJC_STREAMID, new Values(generate(wygjcSchema,wygjcList),wygjcList.size()));
WYGJCList.clear(); wygjcList.clear();
} }
if (!(JSTXList.isEmpty())) { if (!(jstxList.isEmpty())) {
basicOutputCollector.emit(StreamAggregateConfig.JC_JSTX_STREAMID, new Values((Object) generate(jstxSchema,JSTXList),JSTXList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_JSTX_STREAMID, new Values(generate(jstxSchema,jstxList),jstxList.size()));
JSTXList.clear(); jstxList.clear();
} }
if (!(FTPList.isEmpty())) { if (!(ftpList.isEmpty())) {
basicOutputCollector.emit(StreamAggregateConfig.JC_FTP_STREAMID, new Values((Object) generate(ftpSchema,FTPList),FTPList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_FTP_STREAMID, new Values(generate(ftpSchema,ftpList),ftpList.size()));
FTPList.clear(); ftpList.clear();
} }
if (!(PICList.isEmpty())) { if (!(picList.isEmpty())) {
basicOutputCollector.emit(StreamAggregateConfig.JC_PIC_STREAMID, new Values((Object) generate(picSchema,PICList),PICList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_PIC_STREAMID, new Values(generate(picSchema,picList),picList.size()));
PICList.clear(); picList.clear();
} }
if (!(MEDIAList.isEmpty())) { if (!(mediaList.isEmpty())) {
basicOutputCollector.emit(StreamAggregateConfig.JC_MEDIA_STREAMID, new Values((Object) generate(mediaSchema,MEDIAList),MEDIAList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_MEDIA_STREAMID, new Values(generate(mediaSchema,mediaList),mediaList.size()));
MEDIAList.clear(); mediaList.clear();
} }
if(!(MAILList.isEmpty())) { if(!(mailList.isEmpty())) {
basicOutputCollector.emit(StreamAggregateConfig.JC_MAIL_STREAMID, new Values((Object) generate(mailSchema,MAILList),MAILList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_MAIL_STREAMID, new Values(generate(mailSchema,mailList),mailList.size()));
MAILList.clear(); mailList.clear();
} }
if (!VPNList.isEmpty()){ if (!vpnList.isEmpty()){
basicOutputCollector.emit(StreamAggregateConfig.JC_VPN_STREAMID, new Values((Object) generate(vpnSchema, VPNList),VPNList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_VPN_STREAMID, new Values(generate(vpnSchema, vpnList),vpnList.size()));
VPNList.clear(); vpnList.clear();
} }
basicOutputCollector.emit(StreamAggregateConfig.JC_COUNT_STREAMID, new Values(count,loadfail)); basicOutputCollector.emit(StreamAggregateConfig.JC_COUNT_STREAMID, new Values(count,loadfail));
}catch (Exception e) { }catch (Exception e) {
logger.error("----------批数据发送异常-----------" +e); logger.error("----------批数据发送异常-----------" +e);
} }
count = 0; count = 0L;
loadfail = 0; loadfail = 0L;
} else { } else {
try { try {
String line = tuple.getString(0); line = tuple.getString(0);
sessionRecordLog = JSONObject.parseObject(line, SessionRecordLog.class); sessionRecordLog = JSONObject.parseObject(line, SessionRecordLog.class);
UserRegion userRegion = JSONObject.parseObject(sessionRecordLog.getCommon_user_region(), UserRegion.class); UserRegion userRegion = JSONObject.parseObject(sessionRecordLog.getCommon_user_region(), UserRegion.class);
Traffic_Forward traffic_forward = JSONObject.parseObject(userRegion.getTraffic_forward(), Traffic_Forward.class); Traffic_Forward traffic_forward = JSONObject.parseObject(userRegion.getTraffic_forward(), Traffic_Forward.class);
@@ -153,28 +155,27 @@ public class JCParseBolt extends BaseBasicBolt {
sessionRecordLog.setPzlx(traffic_forward.getPzlx()); sessionRecordLog.setPzlx(traffic_forward.getPzlx());
sessionRecordLog.setGklb(traffic_forward.getGklb()); sessionRecordLog.setGklb(traffic_forward.getGklb());
sessionRecordLog.setLabels(traffic_forward.getLabels()); sessionRecordLog.setLabels(traffic_forward.getLabels());
sessionRecordLog.setYyzl(traffic_forward.getYyzl()); //弃用
sessionRecordLog.setGjc(traffic_forward.getGjc()); // sessionRecordLog.setYyzl(traffic_forward.getYyzl());
sessionRecordLog.setZsmc(traffic_forward.getZsmc()); // sessionRecordLog.setGjc(traffic_forward.getGjc());
sessionRecordLog.setYjxy(traffic_forward.getYjxy()); // sessionRecordLog.setZsmc(traffic_forward.getZsmc());
sessionRecordLog.setProtocol(userRegion.getProtocol()); // sessionRecordLog.setYjxy(traffic_forward.getYjxy());
// sessionRecordLog.setProtocol(userRegion.getProtocol());
int pzlx = traffic_forward.getPzlx(); int pzlx = traffic_forward.getPzlx();
if (sessionRecordLog.getHttp_response_body_key()!=null) {
System.out.println("--------------" + sessionRecordLog.getHttp_response_body_key() + "--------------");
}
//按pzlx的值对数据进行分类发送
//按pzlx的值对数据进行分类发送 //按pzlx的值对数据进行分类发送
switch (pzlx) { switch (pzlx) {
case 501: //IP监测 case 501: //IP监测
count++;
try { try {
count ++;
GenericRecord ip_jcrz = new GenericData.Record(ipSchema); GenericRecord ip_jcrz = new GenericData.Record(ipSchema);
sessionRecordLog.setCommon_schema_type("IP"); sessionRecordLog.setCommon_schema_type("IP");
setIP_JCRZ(ip_jcrz, sessionRecordLog); setIP_JCRZ(ip_jcrz, sessionRecordLog);
IPList.add(ip_jcrz); ipList.add(ip_jcrz);
if (IPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (ipList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.JC_IP_STREAMID, new Values((Object) generate(ipSchema, IPList),IPList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_IP_STREAMID, new Values(generate(ipSchema, ipList),ipList.size()));
IPList.clear(); ipList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -183,19 +184,19 @@ public class JCParseBolt extends BaseBasicBolt {
} }
break; break;
case 502: //DNS监测 case 502: //DNS监测
count++;
try { try {
count++;
GenericRecord dns_jczb = new GenericData.Record(dnsSchema); GenericRecord dns_jczb = new GenericData.Record(dnsSchema);
sessionRecordLog.setCommon_schema_type("DNS"); sessionRecordLog.setCommon_schema_type("DNS");
setDNS_JCZB(dns_jczb, sessionRecordLog); setDNS_JCZB(dns_jczb, sessionRecordLog);
if (dns_jczb.get("DNS_NAME")!=("")){ if (dns_jczb.get("DNS_NAME")!=("")){
DNSList.add(dns_jczb); dnsList.add(dns_jczb);
}else { }else {
loadfail++; loadfail++;
} }
if (DNSList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (dnsList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.JC_DNS_STREAMID, new Values((Object) generate(dnsSchema, DNSList),DNSList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_DNS_STREAMID, new Values(generate(dnsSchema, dnsList),dnsList.size()));
DNSList.clear(); dnsList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -205,22 +206,23 @@ public class JCParseBolt extends BaseBasicBolt {
break; break;
case 503: //URL监测 case 503: //URL监测
case 504: //网站监测 case 504: //网站监测
count++;
try { try {
count++;
sessionRecordLog.setYyzl(traffic_forward.getYyzl());
if (sessionRecordLog.getHttp_response_body()!=null){ if (sessionRecordLog.getHttp_response_body()!=null){
String[] split = sessionRecordLog.getHttp_response_body().split("/"); String[] split = sessionRecordLog.getHttp_response_body().split("/");
bucketName = split[split.length-2]; hosBucketName = split[split.length-2];
filename = split[split.length-1]; fileName = split[split.length-1];
basicOutputCollector.emit("jc-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_HTTP_JCZB)); basicOutputCollector.emit("jc-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_HTTP_JCZB));
sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_HTTP_JCZB, filename)); sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_HTTP_JCZB, fileName));
} }
GenericRecord http_jczb = new GenericData.Record(httpSchema); GenericRecord http_jczb = new GenericData.Record(httpSchema);
sessionRecordLog.setCommon_schema_type("HTTP"); sessionRecordLog.setCommon_schema_type("HTTP");
setHTTP_JCZB(http_jczb, sessionRecordLog); setHTTP_JCZB(http_jczb, sessionRecordLog);
HTTPList.add(http_jczb); httpList.add(http_jczb);
if (HTTPList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (httpList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.JC_HTTP_STREAMID, new Values((Object) generate(httpSchema, HTTPList),HTTPList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_HTTP_STREAMID, new Values(generate(httpSchema, httpList),httpList.size()));
HTTPList.clear(); httpList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -229,15 +231,16 @@ public class JCParseBolt extends BaseBasicBolt {
} }
break; break;
case 505: //特定证书监测 case 505: //特定证书监测
count++;
try { try {
count++;
sessionRecordLog.setZsmc(traffic_forward.getZsmc());
GenericRecord ssl_jczb = new GenericData.Record(sslSchema); GenericRecord ssl_jczb = new GenericData.Record(sslSchema);
sessionRecordLog.setCommon_schema_type("SSL"); sessionRecordLog.setCommon_schema_type("SSL");
setSSL_JCZB(ssl_jczb, sessionRecordLog); setSSL_JCZB(ssl_jczb, sessionRecordLog);
SSLList.add(ssl_jczb); sslList.add(ssl_jczb);
if (SSLList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (sslList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.JC_SSL_STREAMID, new Values((Object) generate(sslSchema, SSLList),SSLList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_SSL_STREAMID, new Values(generate(sslSchema, sslList),sslList.size()));
SSLList.clear(); sslList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -247,22 +250,24 @@ public class JCParseBolt extends BaseBasicBolt {
break; break;
case 506: //网站关键词监测 case 506: //网站关键词监测
case 509: //搜索词监测 case 509: //搜索词监测
count++;
try { try {
count++;
sessionRecordLog.setYyzl(traffic_forward.getYyzl());
sessionRecordLog.setGjc(traffic_forward.getGjc());
if (sessionRecordLog.getHttp_response_body()!=null){ if (sessionRecordLog.getHttp_response_body()!=null){
String[] split = sessionRecordLog.getHttp_response_body().split("/"); String[] split = sessionRecordLog.getHttp_response_body().split("/");
bucketName = split[split.length-2]; hosBucketName = split[split.length-2];
filename = split[split.length-1]; fileName = split[split.length-1];
basicOutputCollector.emit("jc-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_WYGJC_JCRZ)); basicOutputCollector.emit("jc-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_WYGJC_JCRZ));
sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_WYGJC_JCRZ, filename)); sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_WYGJC_JCRZ, fileName));
} }
GenericRecord wygjc_jczb = new GenericData.Record(wygjcSchema); GenericRecord wygjc_jczb = new GenericData.Record(wygjcSchema);
sessionRecordLog.setCommon_schema_type("HTTP"); sessionRecordLog.setCommon_schema_type("HTTP");
setWYGJC_JCZB(wygjc_jczb, sessionRecordLog); setWYGJC_JCZB(wygjc_jczb, sessionRecordLog);
WYGJCList.add(wygjc_jczb); wygjcList.add(wygjc_jczb);
if (WYGJCList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (wygjcList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.JC_WYGJC_STREAMID, new Values((Object) generate(wygjcSchema,WYGJCList),WYGJCList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_WYGJC_STREAMID, new Values(generate(wygjcSchema,wygjcList),wygjcList.size()));
WYGJCList.clear(); wygjcList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -272,22 +277,25 @@ public class JCParseBolt extends BaseBasicBolt {
break; break;
case 507: //邮件监测 case 507: //邮件监测
case 510: //邮件监测 case 510: //邮件监测
count++;
try { try {
count++;
sessionRecordLog.setYyzl(traffic_forward.getYyzl());
sessionRecordLog.setGjc(traffic_forward.getGjc());
sessionRecordLog.setYjxy(traffic_forward.getYjxy());
if (sessionRecordLog.getHttp_response_body()!=null){ if (sessionRecordLog.getHttp_response_body()!=null){
String[] split = sessionRecordLog.getHttp_response_body().split("/"); String[] split = sessionRecordLog.getHttp_response_body().split("/");
bucketName = split[split.length-2]; hosBucketName = split[split.length-2];
filename = split[split.length-1]; fileName = split[split.length-1];
basicOutputCollector.emit("jc-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_MAIL_JCZB)); basicOutputCollector.emit("jc-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_MAIL_JCZB));
sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MAIL_JCZB, filename)); sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MAIL_JCZB, fileName));
} }
GenericRecord mail_jczb = new GenericData.Record(mailSchema); GenericRecord mail_jczb = new GenericData.Record(mailSchema);
sessionRecordLog.setCommon_schema_type("MAIL"); sessionRecordLog.setCommon_schema_type("MAIL");
setMAIL_JCZB(mail_jczb, sessionRecordLog); setMAIL_JCZB(mail_jczb, sessionRecordLog);
MAILList.add(mail_jczb); mailList.add(mail_jczb);
if (MAILList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (mailList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.JC_MAIL_STREAMID, new Values((Object) generate(mailSchema,MAILList),MAILList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_MAIL_STREAMID, new Values(generate(mailSchema,mailList),mailList.size()));
MAILList.clear(); mailList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -296,15 +304,17 @@ public class JCParseBolt extends BaseBasicBolt {
} }
break; break;
case 508: //FTP关键词监测 case 508: //FTP关键词监测
count++;
try { try {
count++;
sessionRecordLog.setYyzl(traffic_forward.getYyzl());
sessionRecordLog.setGjc(traffic_forward.getGjc());
GenericRecord ftp_jczb = new GenericData.Record(ftpSchema); GenericRecord ftp_jczb = new GenericData.Record(ftpSchema);
sessionRecordLog.setCommon_schema_type("FTP"); sessionRecordLog.setCommon_schema_type("FTP");
setFTP_JCZB(ftp_jczb,sessionRecordLog); setFTP_JCZB(ftp_jczb,sessionRecordLog);
FTPList.add(ftp_jczb); ftpList.add(ftp_jczb);
if (MAILList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (mailList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.JC_FTP_STREAMID, new Values((Object) generate(ftpSchema,MAILList),FTPList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_FTP_STREAMID, new Values(generate(ftpSchema,mailList),ftpList.size()));
FTPList.clear(); ftpList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -313,18 +323,21 @@ public class JCParseBolt extends BaseBasicBolt {
} }
break; break;
case 511: //VPN监测 case 511: //VPN监测
count++;
try { try {
count++;
GenericRecord vpn_jczb = new GenericData.Record(vpnSchema); GenericRecord vpn_jczb = new GenericData.Record(vpnSchema);
sessionRecordLog.setCommon_schema_type("VPN"); sessionRecordLog.setCommon_schema_type("VPN");
sessionRecordLog.setLx(traffic_forward.getLx());
setVPN_JCZB(vpn_jczb, sessionRecordLog); setVPN_JCZB(vpn_jczb, sessionRecordLog);
if(StringUtil.isNotEmpty(vpn_jczb.get("DETAIL_ID"))) { if(StringUtil.isNotEmpty(vpn_jczb.get("DETAIL_ID"))) {
VPNList.add(vpn_jczb); vpnList.add(vpn_jczb);
}else {
logger.error("vpnjc流量DETAIL_ID为空具体日志为:"+line);
loadfail++;
} }
VPNList.add(vpn_jczb); if (vpnList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
if (VPNList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { basicOutputCollector.emit(StreamAggregateConfig.JC_VPN_STREAMID, new Values(generate(vpnSchema,vpnList),vpnList.size()));
basicOutputCollector.emit(StreamAggregateConfig.JC_VPN_STREAMID, new Values((Object) generate(vpnSchema,VPNList),VPNList.size())); vpnList.clear();
VPNList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -334,15 +347,18 @@ public class JCParseBolt extends BaseBasicBolt {
break; break;
case 512: //特定应用监测 case 512: //特定应用监测
case 513: //社交应用监测 case 513: //社交应用监测
count++;
try { try {
count++;
sessionRecordLog.setYyzl(traffic_forward.getYyzl());
sessionRecordLog.setProtocol(userRegion.getProtocol());
GenericRecord jstx_jczb = new GenericData.Record(jstxSchema); GenericRecord jstx_jczb = new GenericData.Record(jstxSchema);
sessionRecordLog.setProtocol(userRegion.getProtocol());
sessionRecordLog.setCommon_schema_type("JSTX"); sessionRecordLog.setCommon_schema_type("JSTX");
setJSTX_JCZB(jstx_jczb, sessionRecordLog); setJSTX_JCZB(jstx_jczb, sessionRecordLog);
JSTXList.add(jstx_jczb); jstxList.add(jstx_jczb);
if (JSTXList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (jstxList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.JC_JSTX_STREAMID, new Values((Object) generate(jstxSchema,JSTXList),JSTXList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_JSTX_STREAMID, new Values(generate(jstxSchema,jstxList),jstxList.size()));
JSTXList.clear(); jstxList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -351,23 +367,23 @@ public class JCParseBolt extends BaseBasicBolt {
} }
break; break;
case 514: //图片监测 case 514: //图片监测
count++;
try { try {
count++;
sessionRecordLog.setYyzl(traffic_forward.getYyzl());
GenericRecord pic_jczb = new GenericData.Record(picSchema); GenericRecord pic_jczb = new GenericData.Record(picSchema);
if (sessionRecordLog.getHttp_response_body()!=null){ if (sessionRecordLog.getHttp_response_body()!=null){
String[] split = sessionRecordLog.getHttp_response_body().split("/"); String[] split = sessionRecordLog.getHttp_response_body().split("/");
bucketName = split[split.length-2]; hosBucketName = split[split.length-2];
filename = split[split.length-1]; fileName = split[split.length-1];
basicOutputCollector.emit("jc-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_PIC_JCZB)); basicOutputCollector.emit("jc-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_PIC_JCZB));
sessionRecordLog.setHttp_response_body(getYL5( StreamAggregateConfig.T_PIC_JCZB,filename)); sessionRecordLog.setHttp_response_body(getYL5( StreamAggregateConfig.T_PIC_JCZB,fileName));
} }
sessionRecordLog.setCommon_schema_type("HTTP"); sessionRecordLog.setCommon_schema_type("HTTP");
setPIC_JCZB(pic_jczb, sessionRecordLog); setPIC_JCZB(pic_jczb, sessionRecordLog);
PICList.add(pic_jczb); picList.add(pic_jczb);
if (PICList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (picList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.JC_PIC_STREAMID, new Values((Object) generate(picSchema,PICList),PICList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_PIC_STREAMID, new Values(generate(picSchema,picList),picList.size()));
PICList.clear(); picList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -376,22 +392,23 @@ public class JCParseBolt extends BaseBasicBolt {
} }
break; break;
case 515: //多媒体监测 case 515: //多媒体监测
count++;
try { try {
count++;
sessionRecordLog.setYyzl(traffic_forward.getYyzl());
if (sessionRecordLog.getHttp_response_body()!=null){ if (sessionRecordLog.getHttp_response_body()!=null){
String[] split = sessionRecordLog.getHttp_response_body().split("/"); String[] split = sessionRecordLog.getHttp_response_body().split("/");
bucketName = split[split.length-2]; hosBucketName = split[split.length-2];
filename = split[split.length-1]; fileName = split[split.length-1];
basicOutputCollector.emit("jc-unstructured",new Values(bucketName,filename,StreamAggregateConfig.T_MEDIA_JCZB)); basicOutputCollector.emit("jc-unstructured",new Values(hosBucketName,fileName,StreamAggregateConfig.T_MEDIA_JCZB));
sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MEDIA_JCZB, filename)); sessionRecordLog.setHttp_response_body(getYL5(StreamAggregateConfig.T_MEDIA_JCZB, fileName));
} }
GenericRecord media_jczb = new GenericData.Record(mediaSchema); GenericRecord media_jczb = new GenericData.Record(mediaSchema);
sessionRecordLog.setCommon_schema_type("HTTP"); sessionRecordLog.setCommon_schema_type("HTTP");
setMEDIA_JCZB(media_jczb, sessionRecordLog); setMEDIA_JCZB(media_jczb, sessionRecordLog);
MEDIAList.add(media_jczb); mediaList.add(media_jczb);
if (MEDIAList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) { if (mediaList.size() >= StreamAggregateConfig.BATCH_INTER_NUM) {
basicOutputCollector.emit(StreamAggregateConfig.JC_MEDIA_STREAMID, new Values((Object) generate(mediaSchema,MEDIAList),MEDIAList.size())); basicOutputCollector.emit(StreamAggregateConfig.JC_MEDIA_STREAMID, new Values(generate(mediaSchema,mediaList),mediaList.size()));
MEDIAList.clear(); mediaList.clear();
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@@ -403,7 +420,8 @@ public class JCParseBolt extends BaseBasicBolt {
break; break;
} }
} catch(Exception e){ } catch(Exception e){
logger.error("JC数据处理异常"+e); logger.error("JC数据处理异常"+e+"异常数据为:"+line);
loadfail++;
} }
} }
} }
@@ -422,7 +440,7 @@ public class JCParseBolt extends BaseBasicBolt {
outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_PIC_STREAMID,new Fields(StreamAggregateConfig.JC_PIC_STREAMID,"piccount")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_PIC_STREAMID,new Fields(StreamAggregateConfig.JC_PIC_STREAMID,"piccount"));
outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_MEDIA_STREAMID,new Fields(StreamAggregateConfig.JC_MEDIA_STREAMID,"mediacount")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_MEDIA_STREAMID,new Fields(StreamAggregateConfig.JC_MEDIA_STREAMID,"mediacount"));
outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_COUNT_STREAMID, new Fields("count","loadfail")); outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_COUNT_STREAMID, new Fields("count","loadfail"));
outputFieldsDeclarer.declareStream("jc-unstructured", new Fields("bucketName","filename","bucketname")); outputFieldsDeclarer.declareStream("jc-unstructured", new Fields("hosBucketName","fileName","s3BucketName"));
} }
@Override @Override

1
src/main/java/cn/ac/iie/storm/bolt/jc/JCSendBolt.java
View File

@@ -13,7 +13,6 @@ import org.apache.storm.tuple.Values;
import org.apache.storm.utils.TupleUtils; import org.apache.storm.utils.TupleUtils;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*;

25
src/main/java/cn/ac/iie/storm/bolt/jc/JcUnstructuredSendBolt.java
View File

@@ -1,6 +1,5 @@
package cn.ac.iie.storm.bolt.jc; package cn.ac.iie.storm.bolt.jc;
import cn.ac.iie.storm.bolt.conn.ConnUnstructuredSendBolt;
import cn.ac.iie.storm.utils.config.StreamAggregateConfig; import cn.ac.iie.storm.utils.config.StreamAggregateConfig;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.apache.storm.topology.BasicOutputCollector; import org.apache.storm.topology.BasicOutputCollector;
@@ -9,32 +8,30 @@ import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Tuple;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import static cn.ac.iie.storm.utils.hos.HosService.httpPostObject; import static cn.ac.iie.storm.utils.hos.HosService.httpPostObject;
import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost; import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost;
import static cn.ac.iie.storm.utils.http.HttpGetUrl.*; import static cn.ac.iie.storm.utils.http.HttpGetUrl.*;
public class JcUnstructuredSendBolt extends BaseBasicBolt { public class JcUnstructuredSendBolt extends BaseBasicBolt {
String bucketName = null;
String filename = null;
String bucketname = null;
private final static Logger logger = Logger.getLogger(JcUnstructuredSendBolt.class); private final static Logger logger = Logger.getLogger(JcUnstructuredSendBolt.class);
String hosBucketName = null;
String fileName = null;
String s3BucketName = null;
@Override @Override
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
try { try {
bucketName = tuple.getStringByField("bucketName"); hosBucketName = tuple.getStringByField("hosBucketName");
filename = tuple.getStringByField("filename"); fileName = tuple.getStringByField("fileName");
bucketname = tuple.getStringByField("bucketname"); s3BucketName = tuple.getStringByField("s3BucketName");
byte[] bytes = httpPostObject(bucketName, filename); byte[] bytes = httpPostObject(hosBucketName, fileName);
int connAnsyHttpPost =0; int connAnsyHttpPost =0;
String s3Url; String s3Url;
if(bytes!=null) { if(bytes!=null) {
int i = 0; int i = 0;
String ip;
while (i < StreamAggregateConfig.S3_SENT_COUNT) { while (i < StreamAggregateConfig.S3_SENT_COUNT) {
s3Url = getS3Url(bucketname, filename ); s3Url = getS3Url(s3BucketName, fileName );
connAnsyHttpPost = getConnAnsyHttpPost(s3Url, bytes); connAnsyHttpPost = getConnAnsyHttpPost(s3Url, bytes);
if (connAnsyHttpPost == 200) { if (connAnsyHttpPost == 200) {
break; break;
@@ -42,11 +39,11 @@ public class JcUnstructuredSendBolt extends BaseBasicBolt {
i++; i++;
} }
if (connAnsyHttpPost != 200) { if (connAnsyHttpPost != 200) {
logger.error("桶：" + bucketName + "下的文件" + filename + "发送失败"); logger.error("桶：" + hosBucketName + "下的文件" + fileName + "发送失败");
} }
} }
} catch (IOException e) { } catch (IOException e) {
logger.error("桶："+bucketName+"下的文件"+filename+"发送失败原因为："+e); logger.error("桶："+hosBucketName+"下的文件"+fileName+"发送失败原因为："+e);
} }
} }
@Override @Override

74
src/main/java/cn/ac/iie/storm/spout/FloodingKafkaSpout.java
View File

@@ -1,74 +0,0 @@
package cn.ac.iie.storm.spout;
import cn.ac.iie.storm.utils.config.StreamAggregateConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.log4j.Logger;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import java.util.Collections;
import java.util.Map;
import java.util.Properties;
/**
* kafkaSpout
*/
public class FloodingKafkaSpout extends BaseRichSpout {
private static final long serialVersionUID = -3363788553406229592L;
private KafkaConsumer<String, String> consumer;
private SpoutOutputCollector collector = null;
private TopologyContext context = null;
private final static Logger logger = Logger.getLogger(FloodingKafkaSpout.class);
private static Properties createConsumerConfig() {
Properties props = new Properties();
props.put("bootstrap.servers", StreamAggregateConfig.BOOTSTRAP_SERVERS);
props.put("group.id", StreamAggregateConfig.GROUP_ID);
props.put("session.timeout.ms", "60000");
props.put("max.poll.records", 3000);
props.put("max.partition.fetch.bytes", 31457280);
props.put("auto.offset.reset", StreamAggregateConfig.AUTO_OFFSET_RESET);
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
logger.info("链接kafka"+StreamAggregateConfig.BOOTSTRAP_SERVERS+"成功");
return props;
}
@Override
public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
this.collector=spoutOutputCollector;
this.context=topologyContext;
Properties prop = createConsumerConfig();
this.consumer = new KafkaConsumer<>(prop);
this.consumer.subscribe(Collections.singletonList(StreamAggregateConfig.KAFKA_TOPIC));
}
@Override
public void close() {
consumer.close();
}
@Override
public void nextTuple() {
try {
ConsumerRecords<String, String> records = consumer.poll(StreamAggregateConfig.KAFKA_COMSUMER_POLL);
Thread.sleep(StreamAggregateConfig.TOPOLOGY_SPOUT_SLEEP_TIME);
for (ConsumerRecord<String, String> record : records) {
this.collector.emit(new Values(record.value()));
}
} catch (Exception e) {
logger.error("KafkaSpout发送消息出现异常!", e);
e.printStackTrace();
}
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields("flooding-source"));
}
}

2
src/main/java/cn/ac/iie/storm/spout/SafeKafkaSpout.java
View File

@@ -49,6 +49,8 @@ public class SafeKafkaSpout extends BaseRichSpout {
try { try {
ConsumerRecords<String, String> records = consumer.poll(StreamAggregateConfig.KAFKA_COMSUMER_POLL); ConsumerRecords<String, String> records = consumer.poll(StreamAggregateConfig.KAFKA_COMSUMER_POLL);
Thread.sleep(StreamAggregateConfig.TOPOLOGY_SPOUT_SLEEP_TIME); Thread.sleep(StreamAggregateConfig.TOPOLOGY_SPOUT_SLEEP_TIME);
for (ConsumerRecord<String, String> record : records) { for (ConsumerRecord<String, String> record : records) {
this.collector.emit(new Values(record.value())); this.collector.emit(new Values(record.value()));
} }

43
src/main/java/cn/ac/iie/storm/topology/StreamAggregateTopology.java
View File

@@ -8,8 +8,9 @@ import cn.ac.iie.storm.bolt.index.ConnCountBolt;
import cn.ac.iie.storm.bolt.index.GkCountBolt; import cn.ac.iie.storm.bolt.index.GkCountBolt;
import cn.ac.iie.storm.bolt.index.JCCountBolt; import cn.ac.iie.storm.bolt.index.JCCountBolt;
import cn.ac.iie.storm.bolt.jc.*; import cn.ac.iie.storm.bolt.jc.*;
import cn.ac.iie.storm.spout.FloodingKafkaSpout; import cn.ac.iie.storm.spout.ConnectionKafkaSpout;
import cn.ac.iie.storm.spout.SafeKafkaSpout; import cn.ac.iie.storm.spout.SafeKafkaSpout;
import cn.ac.iie.storm.spout.TransactionKafkaSpout;
import cn.ac.iie.storm.utils.config.StreamAggregateConfig; import cn.ac.iie.storm.utils.config.StreamAggregateConfig;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.apache.storm.Config; import org.apache.storm.Config;
@@ -61,7 +62,7 @@ public class StreamAggregateTopology {
//设置拓扑关系(Bolt) --切分行 ---分发bolt //设置拓扑关系(Bolt) --切分行 ---分发bolt
switch (StreamAggregateConfig.TOPOLOGY_LOGO) { switch (StreamAggregateConfig.TOPOLOGY_LOGO) {
case "conn": case "conn":
topologyBuilder.setSpout(StreamAggregateConfig.KAFKA_SPOUT, new FloodingKafkaSpout(), StreamAggregateConfig.SPOUT_PARALLELISM); topologyBuilder.setSpout(StreamAggregateConfig.KAFKA_SPOUT, new ConnectionKafkaSpout(), StreamAggregateConfig.SPOUT_PARALLELISM);
topologyBuilder.setBolt(StreamAggregateConfig.CONN_PARSE_COMPONENTID, new ConnParseBolt(), StreamAggregateConfig.PARSE_PARALLELISM) topologyBuilder.setBolt(StreamAggregateConfig.CONN_PARSE_COMPONENTID, new ConnParseBolt(), StreamAggregateConfig.PARSE_PARALLELISM)
.shuffleGrouping(StreamAggregateConfig.KAFKA_SPOUT); .shuffleGrouping(StreamAggregateConfig.KAFKA_SPOUT);
@@ -148,6 +149,44 @@ public class StreamAggregateTopology {
.shuffleGrouping("JC-HTTP-SEND") .shuffleGrouping("JC-HTTP-SEND")
.shuffleGrouping("JC-SSL-SEND"); .shuffleGrouping("JC-SSL-SEND");
break; break;
case "conn1":
topologyBuilder.setSpout(StreamAggregateConfig.KAFKA_SPOUT, new ConnectionKafkaSpout(), StreamAggregateConfig.SPOUT_PARALLELISM);
topologyBuilder.setSpout(StreamAggregateConfig.TRANSACTION_SPOUT, new TransactionKafkaSpout(), StreamAggregateConfig.SPOUT_PARALLELISM);
// topologyBuilder.setSpout(StreamAggregateConfig.SESSION_SPOUT, new TransactionKafkaSpout(), StreamAggregateConfig.SPOUT_PARALLELISM);
topologyBuilder.setBolt(StreamAggregateConfig.CONN_PARSE_COMPONENTID, new ConnParseBolt(), StreamAggregateConfig.PARSE_PARALLELISM)
.shuffleGrouping(StreamAggregateConfig.TRANSACTION_SPOUT)
.shuffleGrouping(StreamAggregateConfig.KAFKA_SPOUT);
// .shuffleGrouping(StreamAggregateConfig.SESSION_SPOUT);
topologyBuilder.setBolt("CONN-SEND", new ConnSendBolt(),StreamAggregateConfig.SEND_PARALLELISM)
.shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_DNS_STREAMID)
.shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_VPN_STREAMID)
.shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_MAIL_STREAMID)
.shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_JSTX_STREAMID)
.shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_FTP_STREAMID);
topologyBuilder.setBolt("CONN-IP-SEND",new ConnIPSendBolt(),StreamAggregateConfig.IP_SEND_PARALLELISM)
.shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_IP_STREAMID);
topologyBuilder.setBolt("CONN-HTTP-SEND",new ConnHTTPSendBolt(),StreamAggregateConfig.HTTP_SEND_PARALLELISM)
.shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_HTTP_STREAMID);
topologyBuilder.setBolt("CONN-SSL-SEND",new ConnSSLSendBolt(),StreamAggregateConfig.SSL_SEND_PARALLELISM)
.shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_SSL_STREAMID);
topologyBuilder.setBolt("CONN-UNSTRUCTURED-SEND",new ConnUnstructuredSendBolt(),StreamAggregateConfig.UNSTRUCTURED_SEND_PARALLELISM)
.shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,"conn-unstructured");
topologyBuilder.setBolt("CONN-COUNT", new ConnCountBolt(),StreamAggregateConfig.COUNT_SEND_PARALLELISM)
.shuffleGrouping(StreamAggregateConfig.CONN_PARSE_COMPONENTID,StreamAggregateConfig.CONN_YBInputSize_STREAMID)
.shuffleGrouping("CONN-SEND")
.shuffleGrouping("CONN-IP-SEND")
.shuffleGrouping("CONN-HTTP-SEND")
.shuffleGrouping("CONN-SSL-SEND");
break;
default: default:
break; break;
} }

3
src/main/java/cn/ac/iie/storm/utils/combine/httpurl/Http_URL_process.java
View File

@@ -2,9 +2,6 @@ package cn.ac.iie.storm.utils.combine.httpurl;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Http_URL_process { public class Http_URL_process {
private final static Logger logger = Logger.getLogger(Http_URL_process.class); private final static Logger logger = Logger.getLogger(Http_URL_process.class);

192
src/main/java/cn/ac/iie/storm/utils/config/GetMap.java
View File

@@ -1,192 +0,0 @@
package cn.ac.iie.storm.utils.config;
import org.apache.logging.log4j.util.PropertiesUtil;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
public class GetMap {
private static Properties propService = new Properties();
private static Properties propService1 = new Properties();
private static Properties propService_city = new Properties();
private static Properties propService_province = new Properties();
private static Properties propService_country = new Properties();
private static Properties propService_jstx = new Properties();
private static Map<String, String> labelMap;
private static Map<String, String> appMap;
private static Map<String, String> cityMap;
private static Map<String, String> provinceMap;
private static Map<String, String> countryMap;
private static Map<String, String> jstxIdMap;
public static String getJstxDetail_id(int common_app_id){
String appId = String.valueOf(common_app_id);
if (appMap.get(appId) != null) {
return appMap.get(appId);
} else {
return appId;
}
}
public static String getProto_id(String common_schema_type){
return labelMap.get(common_schema_type);
}
public static String getDy(String location){
String[] split;
if(location==null){
return "0";
} else if (location.contains(",,")){
split = location.split(",,");
if (split.length == 2) {
String s = split[0];
if (s.length()<1) {
s = split[1];
if (s == null) {
s = "0";
}
}
return s;
}else {
return "0";
}
} else {
String loc = location.replace(" ","");
String str = cityMap.get(loc);
split = loc.split(",");
if ((str == null&&split.length==3)) {
str = provinceMap.get(split[1]);
if (str == null) {
String ss = split[2];
str = countryMap.get(ss);
if (str == null) {
str = "0";
}
}
return str;
}else if(str !=null){
return str;
}else {
return "0";
}
}
}
public static String getFjSzd(String location,String mail_from,String mail_protocol_type) {
if (mail_from != null&& mail_protocol_type.equals("SMTP")) {
String[] split;
if (location == null) {
return String.valueOf(3);
} else if (location.contains(",,")) {
return String.valueOf(1);
} else {
split = location.split(",");
if (split.length == 3) {
String country = split[2].replace(" ", "");
String s = countryMap.get(country);
if (country.equals("China")) {
return String.valueOf(1);
} else if (s != null) {
return String.valueOf(2);
} else {
return String.valueOf(3);
}
} else {
return String.valueOf(3);
}
}
}else {
return String.valueOf(3);
}
}
public static String getSjSzd(String location,String mail_to,String mail_protocol_type) {
if (mail_to != null&&(mail_protocol_type.equals("POP3")||mail_protocol_type.equals("IMAP"))) {
String[] split;
if (location == null) {
return String.valueOf(3);
} else if (location.contains(",,")) {
return String.valueOf(1);
} else {
split = location.split(",");
if (split.length == 3) {
String country = split[2].replace(" ", "");
String s = countryMap.get(country);
if (country.equals("China")) {
return String.valueOf(1);
} else if (s != null) {
return String.valueOf(2);
} else {
return String.valueOf(3);
}
} else {
return String.valueOf(3);
}
}
}else {
return String.valueOf(3);
}
}
public static String getDns_name(String dns_qname){
if (dns_qname==null){
return "";
}else if (dns_qname.contains(" ")){
return "";
}else {
return dns_qname;
}
}
static {
try {
InputStream applabel = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_app_label_coding.properties");
InputStream app = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_app_id_coding.properties");
InputStream jstxid = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_jstx_id_coding.properties");
InputStream cityNumbering = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_city_coding.properties");
InputStream provinceNumbering = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_province_coding.properties");
InputStream countryNumbering = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_country_coding.properties");
propService1.load(app);
propService.load(applabel);
propService_jstx.load(jstxid);
propService_city.load(cityNumbering);
propService_province.load(provinceNumbering);
propService_country.load(countryNumbering);
appMap = new HashMap<String, String>((Map)propService1);
labelMap = new HashMap<String, String>((Map) propService);
jstxIdMap = new HashMap<String, String>((Map) propService_jstx);
cityMap = new HashMap<String, String>((Map) propService_city);
provinceMap = new HashMap<String, String>((Map) propService_province);
countryMap = new HashMap<String, String>((Map) propService_country);
} catch (Exception e) {
// propCommon = null;
propService = null;
propService1 = null;
propService_jstx = null;
propService_city = null;
propService_province = null;
propService_country = null;
}
}
}

13
src/main/java/cn/ac/iie/storm/utils/config/StreamAggregateConfig.java
View File

@@ -10,6 +10,10 @@ public class StreamAggregateConfig {
public static final String TOPOLOGY_LOGO = StreamAggregateConfigurations.getStringProperty(0, "topology.logo"); public static final String TOPOLOGY_LOGO = StreamAggregateConfigurations.getStringProperty(0, "topology.logo");
/**
* yys字段
*/
public static final Integer YYS_NUM = StreamAggregateConfigurations.getIntProperty(0,"yys_num");
/** /**
* System * System
@@ -30,6 +34,9 @@ public class StreamAggregateConfig {
public static final String BOOTSTRAP_SERVERS = StreamAggregateConfigurations.getStringProperty(0, "bootstrap.servers"); public static final String BOOTSTRAP_SERVERS = StreamAggregateConfigurations.getStringProperty(0, "bootstrap.servers");
public static final String GROUP_ID = StreamAggregateConfigurations.getStringProperty(0, "group.id"); public static final String GROUP_ID = StreamAggregateConfigurations.getStringProperty(0, "group.id");
public static final String KAFKA_TOPIC_SESSION = StreamAggregateConfigurations.getStringProperty(0, "kafka.topic.session");
public static final String KAFKA_TOPIC_TRANSACTION = StreamAggregateConfigurations.getStringProperty(0, "kafka.topic.transaction");
public static final String KAFKA_TOPIC = StreamAggregateConfigurations.getStringProperty(0, "kafka.topic"); public static final String KAFKA_TOPIC = StreamAggregateConfigurations.getStringProperty(0, "kafka.topic");
public static final String KAFKA_SAFE_TOPIC = StreamAggregateConfigurations.getStringProperty(0, "kafka.safe.topic"); public static final String KAFKA_SAFE_TOPIC = StreamAggregateConfigurations.getStringProperty(0, "kafka.safe.topic");
public static final String AUTO_OFFSET_RESET = StreamAggregateConfigurations.getStringProperty(0, "auto.offset.reset"); public static final String AUTO_OFFSET_RESET = StreamAggregateConfigurations.getStringProperty(0, "auto.offset.reset");
@@ -43,6 +50,12 @@ public class StreamAggregateConfig {
//conn-ComponentId-spout //conn-ComponentId-spout
public static final String KAFKA_SPOUT = StreamAggregateConfigurations.getStringProperty(0, "kafka.spout"); public static final String KAFKA_SPOUT = StreamAggregateConfigurations.getStringProperty(0, "kafka.spout");
public static final String TRANSACTION_SPOUT = StreamAggregateConfigurations.getStringProperty(0, "transaction.spout");
public static final String SESSION_SPOUT = StreamAggregateConfigurations.getStringProperty(0, "session.spout");
//#conn-bolt-parallelism_hint //#conn-bolt-parallelism_hint
public static final int PARSE_PARALLELISM = StreamAggregateConfigurations.getIntProperty(0,"parse.parallelism"); public static final int PARSE_PARALLELISM = StreamAggregateConfigurations.getIntProperty(0,"parse.parallelism");
public static final int SEND_PARALLELISM = StreamAggregateConfigurations.getIntProperty(0,"send.parallelism"); public static final int SEND_PARALLELISM = StreamAggregateConfigurations.getIntProperty(0,"send.parallelism");

54
src/main/java/cn/ac/iie/storm/utils/general/CONNAssignment.java
View File

@@ -1,7 +1,6 @@
package cn.ac.iie.storm.utils.general; package cn.ac.iie.storm.utils.general;
import cn.ac.iie.storm.bean.SessionRecordLog; import cn.ac.iie.storm.bean.SessionRecordLog;
import cn.ac.iie.storm.utils.config.GetMap;
import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecord;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
@@ -10,7 +9,7 @@ import java.io.IOException;
import static cn.ac.iie.storm.utils.combine.httpurl.Http_URL_process.getFile_Name; import static cn.ac.iie.storm.utils.combine.httpurl.Http_URL_process.getFile_Name;
import static cn.ac.iie.storm.utils.config.GetMap.*; import static cn.ac.iie.storm.utils.general.GetMap.*;
import static cn.ac.iie.storm.utils.general.MailTrans.*; import static cn.ac.iie.storm.utils.general.MailTrans.*;
import static cn.ac.iie.storm.utils.general.TransFormUtils.*; import static cn.ac.iie.storm.utils.general.TransFormUtils.*;
@@ -19,8 +18,8 @@ public class CONNAssignment {
private final static Logger logger = Logger.getLogger(CONNAssignment.class); private final static Logger logger = Logger.getLogger(CONNAssignment.class);
public static void setDNS_RZZB(GenericRecord dns_rzzb, SessionRecordLog sessionRecordLog) throws IOException { public static void setDNS_RZZB(GenericRecord dns_rzzb, SessionRecordLog sessionRecordLog) throws IOException {
dns_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); dns_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time());
dns_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); dns_rzzb.put("YDY",sessionRecordLog.getYdy());
dns_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); dns_rzzb.put("MDDY",sessionRecordLog.getMddy());
dns_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); dns_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip()));
dns_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); dns_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip()));
dns_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); dns_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip()));
@@ -29,7 +28,7 @@ public class CONNAssignment {
dns_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); dns_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type()));
dns_rzzb.put("DETAIL_ID","02999"); dns_rzzb.put("DETAIL_ID","02999");
dns_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); dns_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol()));
dns_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); dns_rzzb.put("YYS",sessionRecordLog.getYys());
dns_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); dns_rzzb.put("YL1",sessionRecordLog.getCommon_address_type());
//dns特有 //dns特有
dns_rzzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num()); dns_rzzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num());
@@ -42,8 +41,8 @@ public class CONNAssignment {
public static void setIP_RZZB(GenericRecord IP_rzzb, SessionRecordLog sessionRecordLog) throws IOException { public static void setIP_RZZB(GenericRecord IP_rzzb, SessionRecordLog sessionRecordLog) throws IOException {
IP_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); IP_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time());
IP_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); IP_rzzb.put("YDY",sessionRecordLog.getYdy());
IP_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); IP_rzzb.put("MDDY",sessionRecordLog.getMddy());
IP_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); IP_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip()));
IP_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); IP_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip()));
IP_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); IP_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip()));
@@ -52,7 +51,7 @@ public class CONNAssignment {
IP_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); IP_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type()));
IP_rzzb.put("DETAIL_ID","08003"); IP_rzzb.put("DETAIL_ID","08003");
IP_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); IP_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol()));
IP_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); IP_rzzb.put("YYS",sessionRecordLog.getYys());
IP_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); IP_rzzb.put("YL1",sessionRecordLog.getCommon_address_type());
//dns特有 //dns特有
IP_rzzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num()); IP_rzzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num());
@@ -64,8 +63,8 @@ public class CONNAssignment {
public static void setHTTP_RZZB(GenericRecord http_rzzb, SessionRecordLog sessionRecordLog) throws IOException { public static void setHTTP_RZZB(GenericRecord http_rzzb, SessionRecordLog sessionRecordLog) throws IOException {
http_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); http_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time());
http_rzzb.put("YDY", getDy(sessionRecordLog.getCommon_client_location())); http_rzzb.put("YDY",sessionRecordLog.getYdy());
http_rzzb.put("MDDY", getDy(sessionRecordLog.getCommon_server_location())); http_rzzb.put("MDDY", sessionRecordLog.getMddy());
http_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); http_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip()));
http_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(), sessionRecordLog.getCommon_client_ip())); http_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(), sessionRecordLog.getCommon_client_ip()));
http_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(), sessionRecordLog.getCommon_server_ip())); http_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(), sessionRecordLog.getCommon_server_ip()));
@@ -74,7 +73,7 @@ public class CONNAssignment {
http_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); http_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type()));
http_rzzb.put("DETAIL_ID","01999"); http_rzzb.put("DETAIL_ID","01999");
http_rzzb.put("PROTOCOL", getProtocol(sessionRecordLog.getCommon_l4_protocol())); http_rzzb.put("PROTOCOL", getProtocol(sessionRecordLog.getCommon_l4_protocol()));
http_rzzb.put("YYS", getYys(sessionRecordLog.getCommon_data_center())); http_rzzb.put("YYS", sessionRecordLog.getYys());
http_rzzb.put("YL1", sessionRecordLog.getCommon_address_type()); http_rzzb.put("YL1", sessionRecordLog.getCommon_address_type());
http_rzzb.put("YL5",sessionRecordLog.getHttp_response_body()); http_rzzb.put("YL5",sessionRecordLog.getHttp_response_body());
//http特有 //http特有
@@ -92,8 +91,8 @@ public class CONNAssignment {
} }
public static void setVPN_RZZB(GenericRecord vpn_rzzb, SessionRecordLog sessionRecordLog){ public static void setVPN_RZZB(GenericRecord vpn_rzzb, SessionRecordLog sessionRecordLog){
vpn_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); vpn_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time());
vpn_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); vpn_rzzb.put("YDY",sessionRecordLog.getYdy());
vpn_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); vpn_rzzb.put("MDDY",sessionRecordLog.getMddy());
vpn_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); vpn_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip()));
vpn_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); vpn_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip()));
vpn_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); vpn_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip()));
@@ -102,7 +101,7 @@ public class CONNAssignment {
vpn_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); vpn_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type()));
vpn_rzzb.put("DETAIL_ID",getVpnDetail_id(sessionRecordLog.getCommon_l7_protocol(),sessionRecordLog.getCommon_protocol_label())); vpn_rzzb.put("DETAIL_ID",getVpnDetail_id(sessionRecordLog.getCommon_l7_protocol(),sessionRecordLog.getCommon_protocol_label()));
vpn_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); vpn_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol()));
vpn_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); vpn_rzzb.put("YYS",sessionRecordLog.getYys());
vpn_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); vpn_rzzb.put("YL1",sessionRecordLog.getCommon_address_type());
//vpn特有 //vpn特有
vpn_rzzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num()); vpn_rzzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num());
@@ -113,8 +112,8 @@ public class CONNAssignment {
public static void setSSL_RZZB(GenericRecord ssl_rzzb, SessionRecordLog sessionRecordLog) throws IOException { public static void setSSL_RZZB(GenericRecord ssl_rzzb, SessionRecordLog sessionRecordLog) throws IOException {
ssl_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); ssl_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time());
ssl_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); ssl_rzzb.put("YDY",sessionRecordLog.getYdy());
ssl_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); ssl_rzzb.put("MDDY",sessionRecordLog.getMddy());
ssl_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); ssl_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip()));
ssl_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); ssl_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip()));
ssl_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); ssl_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip()));
@@ -123,7 +122,7 @@ public class CONNAssignment {
ssl_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); ssl_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type()));
ssl_rzzb.put("DETAIL_ID","07999"); //同上 ssl_rzzb.put("DETAIL_ID","07999"); //同上
ssl_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); ssl_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol()));
ssl_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); ssl_rzzb.put("YYS",sessionRecordLog.getYys());
ssl_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); ssl_rzzb.put("YL1",sessionRecordLog.getCommon_address_type());
//ssl特有 //ssl特有
@@ -131,14 +130,13 @@ public class CONNAssignment {
ssl_rzzb.put("CS_BYTES",sessionRecordLog.getCommon_c2s_byte_num()); ssl_rzzb.put("CS_BYTES",sessionRecordLog.getCommon_c2s_byte_num());
ssl_rzzb.put("SC_PKTS",sessionRecordLog.getCommon_s2c_pkt_num()); ssl_rzzb.put("SC_PKTS",sessionRecordLog.getCommon_s2c_pkt_num());
ssl_rzzb.put("SC_BYTES",sessionRecordLog.getCommon_s2c_byte_num()); ssl_rzzb.put("SC_BYTES",sessionRecordLog.getCommon_s2c_byte_num());
ssl_rzzb.put("FWQMC",sessionRecordLog.getSsl_sni()); ssl_rzzb.put("FWQMC",sessionRecordLog.getSsl_sni());
} }
public static void setFTP_RZZB(GenericRecord ftp_rzzb, SessionRecordLog sessionRecordLog) throws IOException { public static void setFTP_RZZB(GenericRecord ftp_rzzb, SessionRecordLog sessionRecordLog) throws IOException {
ftp_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); ftp_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time());
ftp_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); ftp_rzzb.put("YDY",sessionRecordLog.getYdy());
ftp_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); ftp_rzzb.put("MDDY",sessionRecordLog.getMddy());
ftp_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); ftp_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip()));
ftp_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); ftp_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip()));
ftp_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); ftp_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip()));
@@ -147,7 +145,7 @@ public class CONNAssignment {
ftp_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); ftp_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type()));
ftp_rzzb.put("DETAIL_ID","06999"); //同上 ftp_rzzb.put("DETAIL_ID","06999"); //同上
ftp_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); ftp_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol()));
ftp_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); ftp_rzzb.put("YYS",sessionRecordLog.getYys());
ftp_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); ftp_rzzb.put("YL1",sessionRecordLog.getCommon_address_type());
//ftp特有 //ftp特有
@@ -161,8 +159,8 @@ public class CONNAssignment {
public static void setMAIL_RZZB(GenericRecord mail_rzzb, SessionRecordLog sessionRecordLog) throws IOException { public static void setMAIL_RZZB(GenericRecord mail_rzzb, SessionRecordLog sessionRecordLog) throws IOException {
mail_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); mail_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time());
mail_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); mail_rzzb.put("YDY",sessionRecordLog.getYdy());
mail_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); mail_rzzb.put("MDDY",sessionRecordLog.getMddy());
mail_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); mail_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip()));
mail_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); mail_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip()));
mail_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); mail_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip()));
@@ -171,7 +169,7 @@ public class CONNAssignment {
mail_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); mail_rzzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type()));
mail_rzzb.put("DETAIL_ID",getMailDetail_id(sessionRecordLog.getMail_protocol_type())); mail_rzzb.put("DETAIL_ID",getMailDetail_id(sessionRecordLog.getMail_protocol_type()));
mail_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); mail_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol()));
mail_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); mail_rzzb.put("YYS",sessionRecordLog.getYys());
mail_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); mail_rzzb.put("YL1",sessionRecordLog.getCommon_address_type());
mail_rzzb.put("YL5",sessionRecordLog.getHttp_response_body()); mail_rzzb.put("YL5",sessionRecordLog.getHttp_response_body());
//根据mail_bcc填充 //根据mail_bcc填充
@@ -203,17 +201,17 @@ public class CONNAssignment {
public static void setJSTX_RZZB(GenericRecord jstx_rzzb, SessionRecordLog sessionRecordLog) throws IOException { public static void setJSTX_RZZB(GenericRecord jstx_rzzb, SessionRecordLog sessionRecordLog) throws IOException {
jstx_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); jstx_rzzb.put("BHSJ",sessionRecordLog.getCommon_recv_time());
jstx_rzzb.put("YDY",getDy(sessionRecordLog.getCommon_client_location())); jstx_rzzb.put("YDY",sessionRecordLog.getYdy());
jstx_rzzb.put("MDDY",getDy(sessionRecordLog.getCommon_server_location())); jstx_rzzb.put("MDDY",sessionRecordLog.getMddy());
jstx_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip())); jstx_rzzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_sled_ip()));
jstx_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip())); jstx_rzzb.put("SIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_client_ip()));
jstx_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip())); jstx_rzzb.put("DIP",getIP(sessionRecordLog.getCommon_address_type(),sessionRecordLog.getCommon_server_ip()));
jstx_rzzb.put("SPORT",sessionRecordLog.getCommon_client_port()); jstx_rzzb.put("SPORT",sessionRecordLog.getCommon_client_port());
jstx_rzzb.put("DPORT",sessionRecordLog.getCommon_server_port()); jstx_rzzb.put("DPORT",sessionRecordLog.getCommon_server_port());
jstx_rzzb.put("PROTO_ID", GetMap.getProto_id(sessionRecordLog.getCommon_schema_type())); jstx_rzzb.put("PROTO_ID", GetMap.getProto_id(sessionRecordLog.getCommon_schema_type()));
jstx_rzzb.put("DETAIL_ID", GetMap.getJstxDetail_id(sessionRecordLog.getCommon_app_id())); jstx_rzzb.put("DETAIL_ID",sessionRecordLog.getCommon_app_label());
jstx_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); jstx_rzzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol()));
jstx_rzzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); jstx_rzzb.put("YYS",sessionRecordLog.getYys());
jstx_rzzb.put("YL1",sessionRecordLog.getCommon_address_type()); jstx_rzzb.put("YL1",sessionRecordLog.getCommon_address_type());
//ftp特有 //ftp特有
jstx_rzzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num()); jstx_rzzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num());

6
src/main/java/cn/ac/iie/storm/utils/general/GKAssignment.java
View File

@@ -1,13 +1,12 @@
package cn.ac.iie.storm.utils.general; package cn.ac.iie.storm.utils.general;
import cn.ac.iie.storm.bean.SessionRecordLog; import cn.ac.iie.storm.bean.SessionRecordLog;
import cn.ac.iie.storm.utils.config.GetMap;
import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecord;
import java.io.IOException; import java.io.IOException;
import static cn.ac.iie.storm.utils.combine.httpurl.Http_URL_process.getFile_Name; import static cn.ac.iie.storm.utils.combine.httpurl.Http_URL_process.getFile_Name;
import static cn.ac.iie.storm.utils.config.GetMap.*; import static cn.ac.iie.storm.utils.general.GetMap.*;
import static cn.ac.iie.storm.utils.general.MailTrans.*; import static cn.ac.iie.storm.utils.general.MailTrans.*;
import static cn.ac.iie.storm.utils.general.TransFormUtils.*; import static cn.ac.iie.storm.utils.general.TransFormUtils.*;
import static cn.ac.iie.storm.utils.general.TransFormUtils.getYys; import static cn.ac.iie.storm.utils.general.TransFormUtils.getYys;
@@ -259,7 +258,8 @@ public class GKAssignment {
vpn_gkzb.put("DPORT",sessionRecordLog.getCommon_server_port()); vpn_gkzb.put("DPORT",sessionRecordLog.getCommon_server_port());
vpn_gkzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); ///--转化表 还没弄 vpn_gkzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); ///--转化表 还没弄
vpn_gkzb.put("DETAIL_ID",getVpnDetail_id(sessionRecordLog.getCommon_l7_protocol(),sessionRecordLog.getCommon_protocol_label())); //同上 // vpn_gkzb.put("DETAIL_ID",getVpnDetail_id(sessionRecordLog.getCommon_l7_protocol(),sessionRecordLog.getCommon_protocol_label())); //同上
vpn_gkzb.put("DETAIL_ID",sessionRecordLog.getLx());
vpn_gkzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); //-common_l4_protocol其实是这个字段 vpn_gkzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); //-common_l4_protocol其实是这个字段
vpn_gkzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); //-**common_isp这个字段 vpn_gkzb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); //-**common_isp这个字段
vpn_gkzb.put("YL1",sessionRecordLog.getCommon_address_type()); vpn_gkzb.put("YL1",sessionRecordLog.getCommon_address_type());

6
src/main/java/cn/ac/iie/storm/utils/general/JCAssignment.java
View File

@@ -1,14 +1,13 @@
package cn.ac.iie.storm.utils.general; package cn.ac.iie.storm.utils.general;
import cn.ac.iie.storm.bean.SessionRecordLog; import cn.ac.iie.storm.bean.SessionRecordLog;
import cn.ac.iie.storm.utils.config.GetMap;
import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecord;
import java.io.IOException; import java.io.IOException;
import static cn.ac.iie.storm.utils.combine.httpurl.Http_URL_process.getFile_Name; import static cn.ac.iie.storm.utils.combine.httpurl.Http_URL_process.getFile_Name;
import static cn.ac.iie.storm.utils.config.GetMap.*; import static cn.ac.iie.storm.utils.general.GetMap.*;
import static cn.ac.iie.storm.utils.general.MailTrans.*; import static cn.ac.iie.storm.utils.general.MailTrans.*;
import static cn.ac.iie.storm.utils.general.TransFormUtils.*; import static cn.ac.iie.storm.utils.general.TransFormUtils.*;
@@ -284,7 +283,8 @@ public class JCAssignment {
vpn_jczb.put("SPORT",sessionRecordLog.getCommon_client_port()); vpn_jczb.put("SPORT",sessionRecordLog.getCommon_client_port());
vpn_jczb.put("DPORT",sessionRecordLog.getCommon_server_port()); vpn_jczb.put("DPORT",sessionRecordLog.getCommon_server_port());
vpn_jczb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); vpn_jczb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type()));
vpn_jczb.put("DETAIL_ID",getVpnDetail_id(sessionRecordLog.getCommon_l7_protocol(),sessionRecordLog.getCommon_protocol_label())); // vpn_jczb.put("DETAIL_ID",getVpnDetail_id(sessionRecordLog.getCommon_l7_protocol(),sessionRecordLog.getCommon_protocol_label()));
vpn_jczb.put("DETAIL_ID",sessionRecordLog.getLx());
vpn_jczb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); vpn_jczb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol()));
vpn_jczb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); //-**common_isp这个字段 vpn_jczb.put("YYS",getYys(sessionRecordLog.getCommon_data_center())); //-**common_isp这个字段
vpn_jczb.put("YL1",sessionRecordLog.getCommon_address_type()); vpn_jczb.put("YL1",sessionRecordLog.getCommon_address_type());

4
src/main/java/cn/ac/iie/storm/utils/general/TransFormUtils.java
View File

@@ -203,10 +203,6 @@ public class TransFormUtils {
/** /**
* avro对象的序列化方式 * avro对象的序列化方式
* @param schema * @param schema

1
src/resources/common_app_id_coding.properties
View File

@@ -1 +0,0 @@
#common_app_id_conding

1
src/resources/common_app_label_coding.properties
View File

@@ -1 +0,0 @@
全部=00

359
src/resources/common_city_coding.properties
View File

@@ -1,359 +0,0 @@
#河北
Shijiazhuang,HeBei,China=130100
Tangshan,Hebei,China=130200
Qinhuangdao,Hebei,China=130300
Handan,Hebei,China=130400
Xingtai,Hebei,China=130500
Baoding,Hebei,China=130600
Zhangjiakou,Hebei,China=130700
Chengde,Hebei,China=130800
Cangzhou,Hebei,China=130900
Langfang,Hebei,China=131000
Hengshui,Hebei,China=131100
#山西
Taiyuan,Shanxi,China=140100
Datong,Shanxi,China=140200
Yangquan,Shanxi,China=140300
Changzhi,Shanxi,China=140400
Jincheng,Shanxi,China=140500
Shuozhou,Shanxi,China=140600
Jinzhong,Shanxi,China=140700
Yuncheng,Shanxi,China=140800
Xinzhou,Shanxi,China=140900
Linfen,Shanxi,China=141000
Lvliang,Shanxi,China=141100
#内蒙古
Other,InnerMongoliaAutonomousRegion,China=150000
Huhehaote,InnerMongoliaAutonomousRegion,China=150100
Baotou,InnerMongoliaAutonomousRegion,China=150200
Wuhai,InnerMongoliaAutonomousRegion,China=150300
Chifeng,InnerMongoliaAutonomousRegion,China=150400
Tongliao,InnerMongoliaAutonomousRegion,China=150500
Eerduosi,InnerMongoliaAutonomousRegion,China=150600
Hulunbeier,InnerMongoliaAutonomousRegion,China=150700
Bayannaoer,InnerMongoliaAutonomousRegion,China=150800
Wulanchabu,InnerMongoliaAutonomousRegion,China=150900
Xingan,InnerMongoliaAutonomousRegion,China=152200
Xilinguolei,InnerMongoliaAutonomousRegion,China=152500
Alashan,InnerMongoliaAutonomousRegion,China=152900
#辽宁
Shenyang,Liaoning,China=210100
Dalian,Liaoning,China=210200
Anshan,Liaoning,China=210300
Funshun,Liaoning,China=210400
Benxi,Liaoning,China=210500
Dandong,Liaoning,China=210600
Jinzhou,Liaoning,China=210700
Yingkou,Liaoning,China=210800
Fuxin,Liaoning,China=210900
LiaoYang,Liaoning,China=211000
Panjin,Liaoning,China=211100
Tieling,Liaoning,China=211200
Chaoyang,Liaoning,China=211300
Huludao,Liaoning,China=211400
#吉林
Changchun,Jilin,China=220100
Jilin,Jilin,China=220200
Siping,Jilin,China=220300
Liaoyuan,Jilin,China=220400
Tonghua,Jilin,China=220500
Baishan,Jilin,China=220600
Songyuan,Jilin,China=220700
Baicheng,Jilin,China=220800
#黑龙江
Ha'erbin,Heilongjiang,China=230100
Qiqiha'er,Heilongjiang,China=230200
Jixi,Heilongjiang,China=230300
Hegang,Heilongjiang,China=230400
Shuangyashan,Heilongjiang,China=230500
Daqing,Heilongjiang,China=230600
Yichun,Heilongjiang,China=230700
Jiamusi,Heilongjiang,China=230800
Qitaihe,Heilongjiang,China=230900
Mudanjiang,Heilongjiang,China=231000
Heihe,Heilongjiang,China=231100
Suihua,Heilongjiang,China=231200
#江苏省
Nanjing,Jiangsu,China=320100
Wuxi,Jiangsu,China=320200
Xuzhou,Jiangsu,China=320300
Changzhou,Jiangsu,China=320400
Suzhou,Jiangsu,China=320500
Nantong,Jiangsu,China=320600
Lianyungang,Jiangsu,China=320700
Huai'an,Jiangsu,China=320800
Yancheng,Jiangsu,China=320900
Yangzhou,Jiangsu,China=321000
Zhenjiang,Jiangsu,China=321100
Suqian,Jiangsu,China=321300
#浙江省
Hangzhou,Zhejiang,China=330100
Ningbo,Zhejiang,China=330200
Wenzhou,Zhejiang,China=330300
Jiaxing,Zhejiang,China=330400
Huzhou,Zhejiang,China=330500
Shaoxing,Zhejiang,China=330600
Jinhua,Zhejiang,China=330700
Quzhou,Zhejiang,China=330800
Zhoushan,Zhejiang,China=330900
Taizhou,Zhejiang,China=331000
Lishui,Zhejiang,China=331100
#安徽省
Hefei,Anhui,China=340100
Wuhu,Anhui,China=340200
Bangbu,Anhui,China=340300
Huainan,Anhui,China=340400
Ma'anshan,Anhui,China=340500
Huaibei,Anhui,China=340600
Tongling,Anhui,China=340700
Anqing,Anhui,China=340800
Huangshan,Anhui,China=341000
Chuzhou,Anhui,China=341100
Fuyang,Anhui,China=341200
Suzhou,Anhui,China=341300
Liu'an,Anhui,China=341500
Haozhou,Anhui,China=341600
Chizhou,Anhui,China=341700
Xuancheng,Anhui,China=341800
#福建省
Fuzhou,Fujian,China=350100
Xiamen,Fujian,China=350200
Putian,Fujian,China=350300
Sanming,Fujian,China=350400
Quanzhou,Fujian,China=350500
Zhangzhou,Fujian,China=350600
Nanping,Fujian,China=350700
Longyan,Fujian,China=350800
Ningde,Fujian,China=350900
#江西省
Nanchang,Jiangxi,China=360100
Jingdezhen,Jiangxi,China=360200
Pingxiang,Jiangxi,China=360300
Jiujiang,Jiangxi,China=360400
Xinyu,Jiangxi,China=360500
Yingtan,Jiangxi,China=360600
Ganzhou,Jiangxi,China=360700
Ji'an,Jiangxi,China=360800
Yichun,Jiangxi,China=360900
Fuzhou,Jiangxi,China=361000
Shang'rao,Jiangxi,China=361100
#山东省
Jinan,Shandong,China=370100
Qingdao,Shandong,China=370200
Zibo,Shandong,China=370300
Zaozhuang,Shandong,China=370400
Dongying,Shandong,China=370500
Yantai,Shandong,China=370600
Weifang,Shandong,China=370700
Jining,Shandong,China=370800
Tai'an,Shandong,China=370900
Weihai,Shandong,China=371000
RiZhao,Shandong,China=371100
Laiwu,Shandong,China=371200
Linyi,Shandong,China=371300
Dezhou,Shandong,China=371400
Liaocheng,Shandong,China=371500
Binzhou,Shandong,China=371600
Heze,Shandong,China=371700
#河南省
Zhengzhou,Henan,China=410100
Kaifeng,Henan,China=410200
Luoyang,Henan,China=410300
Pingdingshan,Henan,China=410400
Anyang,Henan,China=410500
Hebi,Henan,China=410600
Xinxiang,Henan,China=410700
Jiaozuo,Henan,China=410800
Puyang,Henan,China=410900
Xuchang,Henan,China=411000
Luohe,Henan,China=411100
Sanmenxia,Henan,China=411200
Nanyang,Henan,China=411300
Shangqiu,Henan,China=411400
Xinyang,Henan,China=411500
Zhoukou,Henan,China=411600
Zhumadian,Henan,China=411700
#湖北省
Wuhan,Hubei,China=420100
Huangshi,Hubei,China=420200
Shiyan,Hubei,China=420300
Yichang,Hubei,China=420500
Xiangyang,Hubei,China=420600
Ezhou,Hubei,China=420700
Xingmen,Hubei,China=420800
Xiaogan,Hubei,China=420900
Jingzhou,Hubei,China=420100
Huanggang,Hubei,China=421100
Xianning,Hubei,China=421200
Suizhou,Hubei,China=421300
Enshi,Hubei,China=422800
#湖南省
Changsha,Hunan,China=430100
Zhuzhou,Hunan,China=430200
Xiangtan,Hunan,China=430300
Hengyang,Hunan,China=430400
Shaoyang,Hunan,China=430500
Yueyang,Hunan,China=430600
Changde,Hunan,China=430700
Zhangjiajie,Hunan,China=430800
Yiyang,Hunan,China=430900
Chenzhou,Hunan,China=431000
Yongzhou,Hunan,China=431100
Huaihua,Hunan,China=431200
Loudi,Hunan,China=431300
#广东省
Guangzhou,Guangdong,China=440100
Shaoguan,Guangdong,China=440200
Shenzhen,Guangdong,China=440300
Zhuhai,Guangdong,China=440400
Shantou,Guangdong,China=440500
Foshan,Guangdong,China=440600
Jiangmen,Guangdong,China=440700
Zhanjiang,Guangdong,China=440800
Maoming,Guangdong,China=440900
Zhaoqing,Guangdong,China=441200
Huizhou,Guangdong,China=441300
Meizhou,Guangdong,China=441400
Shanwei,Guangdong,China=441500
Heyuan,Guangdong,China=441600
Yangjiang,Guangdong,China=441700
Qingyuan,Guangdong,China=441800
Dongguan,Guangdong,China=441900
Zhongshan,Guangdong,China=442000
Chaozhou,Guangdong,China=445100
Jieyang,Guangdong,China=445200
Yunfu,Guangdong,China=445300
#广西壮族自治区
Nanning,GuangxiZhuangAutonomousRegion,China=450100
Liuzhou,GuangxiZhuangAutonomousRegion,China=450200
Guilin,GuangxiZhuangAutonomousRegion,China=450300
Wuzhou,GuangxiZhuangAutonomousRegion,China=450400
Beihai,GuangxiZhuangAutonomousRegion,China=450500
Fangchenggang,GuangxiZhuangAutonomousRegion,China=450600
Qinzhou,GuangxiZhuangAutonomousRegion,China=450700
Guigang,GuangxiZhuangAutonomousRegion,China=450800
Yulin,GuangxiZhuangAutonomousRegion,China=450900
Baise,GuangxiZhuangAutonomousRegion,China=451000
Hezhou,GuangxiZhuangAutonomousRegion,China=451100
Hechi,GuangxiZhuangAutonomousRegion,China=451200
Laibin,GuangxiZhuangAutonomousRegion,China=451300
Chongzuo,GuangxiZhuangAutonomousRegion,China=451400
#海南省
Haikou,Hainan,China=460100
Sanya,Hainan,China=460200
Sansha,Hainan,China=460300
Danzhou,Hainan,China=460400
#四川
Chengdou,Sichuan,China=510100
Zigong,Sichuan,China=510300
Panzhihua,Sichuan,China=510400
Luzhou,Sichuan,China=510500
Deyang,Sichuan,China=510600
Mianyang,Sichuan,China=510700
Guangyuan,Sichuan,China=510800
Suining,Sichuan,China=510900
Neijiang,Sichuan,China=511000
Leshan,Sichuan,China=511100
Nanchong,Sichuan,China=511300
Meishan,Sichuan,China=511400
Yibin,Sichuan,China=511500
Guang'an,Sichuan,China=511600
Dazhou,Sichuan,China=511700
Ya'an,Sichuan,China=511800
Bazhong,Sichuan,China=511900
Ziyang,Sichuan,China=512000
#贵州省
Guiyang,Guizhou,China=520100
Liupanshui,Sichuan,China=520200
Zunyi,Sichuan,China=520300
Anshun,Sichuan,China=520400
Bijie,Sichuan,China=520500
Tongren,Sichuan,China=520600
#云南省
Kunming,Yunnan,China=530100
Qujing,Yunnan,China=530300
Yuxi,Yunnan,China=530400
Baoshan,Yunnan,China=530500
Zhaotong,Yunnan,China=530600
Lijiang,Yunnan,China=530700
Pu'er,Yunnan,China=530800
Lincang,Yunnan,China=530900
XishuangbannaDaiAutonomousPrefecture,Yunnan,China=532800
Dali,Yunnan,China=532900
#陕西省
xi'an,Shaanxi,China=610100
Tongchuan,Shaanxi,China=610200
Baoji,Shaanxi,China=610300
Xianyang,Shaanxi,China=610400
Weinan,Shaanxi,China=610500
Yan'an,Shaanxi,China=610600
Hanzhong,Shaanxi,China=610700
Yulin,Shaanxi,China=610800
Ankang,Shaanxi,China=610900
Shangluo,Shaanxi,China=611000
#甘肃省
Lanzhou,Gansu,China=620100
Jiayuguan,Gansu,China=620200
Jinchang,Gansu,China=620300
Baiyin,Gansu,China=620400
Tianshui,Gansu,China=620500
Wuwei,Gansu,China=620600
Zhangye,Gansu,China=620700
Pingliang,Gansu,China=620800
Jiuquan,Gansu,China=620900
Qingyang,Gansu,China=621000
Dingxi,Gansu,China=621100
Longnan,Gansu,China=621200
#青海省
Xining,Qinghai,China=630100
Haidong,Qinghai,China=630200
#宁夏回族自治区
Yinchuan,NingxiaHuiAutonomousRegion,China=640100
Zuishan,NingxiaHuiAutonomousRegion,China=640200
Wuzhong,NingxiaHuiAutonomousRegion,China=640300
Guyuan,NingxiaHuiAutonomousRegion,China=640400
Zhongwei,NingxiaHuiAutonomousRegion,China=640500
#台湾
#西藏
Lhasa,Tibet,China=540100
PrivateIP=000000

253
src/resources/common_country_coding.properties
View File

@@ -1,253 +0,0 @@
UnitedStates=1
Canada=1
Russia=7
Egypt=20
SouthAfrica=27
Greece=30
Netherlands=31
Belgium=32
France=33
Spain=34
Hungary=36
EastGermany=37
Yugoslavia=38
Italy=39
Romania=40
Switzerland=41
Czechoslovakia=42
Austria=43
UnitedKingdom=44
Denmark=44
Sweden=46
Norway=47
Poland=48
Germany=49
Peru=51
Mexico=52
Cuba=53
Argentina=54
Brazil=55
Chile=56
Colombia=57
Venezuela=58
Malaysia=60
Australia=61
Indonesia=62
Philippines=63
newZealand=64
Singapore=65
Thailand=66
Japa=81
Korea=82
Vietnam=84
China=86
Turkey=90
India=91
Pakistan=92
Afghanistan=93
SriLanka=94
Myanmar=95
Iran=98
SouthSudan=211
Morocco=212
Algeria=213
Tunisia=216
Libya=218
Gambia=220
Senegal=221
Mauritania=222
Mali=223
Guinea=224
Coted'Ivoire=225
BurkinaFaso=226
Niger=227
Togo=228
Benin=229
Mauritius=230
Liberia=231
SierraLeone=232
Ghana=233
Nigeria=234
Chad=235
CentralAfricanRepublic=236
Cameroon=237
CapeVerde=238
SaoTomeandPrincipe=239
EquatorialGuinea=240
Gabon=241
therepublicofCongo=242
DemocraticRepublicofCongo=243
Angola=244
Guinea-Bissau=245
DiegoGarcia=246
AscensionIsland=247
Seychelles=248
Sudan=249
Rwanda=250
Ethiopia=251
Somalia=252
Djibouti=253
Kenya=254
Tanzania=255
Uganda=256
Burundi=257
Mozambique=258
Zanzibar=259
Zambia=260
Madagascar=261
Reunion=262
Zimbabwe=263
Namibia=264
Malawi=265
Lesotho=266
Botswana=267
Swaziland=268
ComorosandMayotte=269
St.Helena=290
Eritrea=291
Aruba=297
FaroeIslands=298
Greenland=299
Gibraltar=350
Portugal=351
Luxembourg=352
Ireland=353
Iceland=354
Albania=355
Malta=356
Cyprus=357
Finland=358
Bulgaria=359
Lithuania=370
Latvia=371
Estonia=372
Moldova=373
Armenia=374
Belarus=375
Andorra=376
Monaco=377
SanMarino=378
Vatican=379
Ukraine=380
Serbia=381
Montenegro=382
Croatia=385
Slovenia=386
BosniaandHerzegovina=387
Europeantelephonenumberspace-ringEuropeservice=388
Macedonia=389
CzechRepublic=420
Slovakia=421
Liechtenstein=423
FalklandIslands=500
Belize=501
Guatemala=502
Salvador=503
Honduras=504
Nicaragua=505
CostaRica=506
Panama=507
SaintPierreandMiquelon=508
Haiti=509
Guadeloupe(includingFrenchSaintMartinandSaintBarthelemy)=590
Bolivia=591
Guyana=592
Ecuador=593
FrenchGuiana=594
Paraguay=595
Martinique=596
Suriname=597
Uruguay=598
NetherlandsAntilles=599
EastTimor=670
Australianoverseasterritories:Antarctica,ChristmasIsland,CocosIslands,NorfolkIsland=672
Brunei=673
Nauru=674
PapuaNewGuinea=675
Tonga=676
SolomonIslands=677
Vanuatu=678
Fiji=679
Palau=680
WallisandFutunaIslands=681
CookIslands=682
Niue=683
Samoa=685
Kiribati,GilbertIslands=686
NewCaledonia=687
Tuvalu,EllisIslands=688
FrenchPolynesia=689
Tokelau=690
FederatedStatesofMicronesia=691
MarshallIslands=692
NorthKorea=850
Macao,China=853
Cambodia=855
Laos=856
InternationalMaritimeSatelliteOrganization=870
GlobalPersonalCommunicationService=878
Bangladesh=880
Mobilesatellitesystem=881
Internet=882
Maldives=960
Lebanon=961
Jordan=962
Syria=963
Iraq=964
Kuwait=965
SaudiArabia=966
Yemen=967
Oman=968
YemenDemocraticRepublic=969
Palestine=970
UnitedArabEmirates=971
Israel=972
Bahrain=973
Qatar=974
Bhutan=975
Mongolia=976
Nepal=977
Internationalrateservice=979
InternationalTelecommunicationsPublicCommunicationServiceTest=991
Tajikistan=992
Turkmenistan=993
Azerbaijan=994
Georgia=995
Kyrgyzstan=996
Uzbekistan=998
Bahamas=1242
Barbados=1246
Anguilla=1264
AntiguaandBarbuda=1268
BritishVirginIslands=1284
U.S.VirginIslands=1340
CaymanIslands=1345
Bermuda=1441
Grenada=1473
TurksandCaicosIslands=1649
Montserrat=1664
NorthernMarianaIslands=1670
Guam=1671
AmericanSamoa=1684
SaintLucia=1758
Dominica=1767
SaintVincentandheGrenadines=1784
PuertoRico=1787
DominicanRepublic=1809
TrinidadandTobago=1868
SaintKittsandNevis=1869
Jamaica=1876
Curacao=5999
Macao=820000
HongKong=810000
Taiwan=710000

194
src/resources/common_jstx_id_coding.properties
View File

@@ -1,194 +0,0 @@
100=Unknown_TCP
101=Unknown_UDP
102=Unknown_Other
103=DNS
104=FTP
105=FTPS
106=HTTP
107=HTTPS
108=ICMP
109=IKE
110=MAIL
111=IMAPS
112=IPSEC
113=XMPP
114=L2TP
115=NTP
117=POP3S
118=PPTP
119=QUIC
120=SIP
121=SMB
123=SMTPS
124=SPDY
125=SSH
126=SSL
127=SOCKS
128=TELNET
129=DHCP
130=RADIUS
131=OPENVPN
132=STUN
133=TEREDO
134=DTLS
135=DoH
136=ISAKMP
137=MDNS
138=NETBIOS
139=NETFLOW
140=RDP
141=RTCP
142=RTP
143=SLP
144=SNMP
145=SSDP
146=TFTP
147=BJNP
148=LDAP
149=RTMP
150=RTSP
200=05001
201=05002
202=05003
203=05004
204=05005
205=05006
206=05007
207=05008
208=05009
209=05010
210=05011
211=05012
212=05013
213=05014
214=05015
215=05016
216=05017
217=05018
218=05019
219=05020
220=05021
221=05022
222=05023
223=05024
224=05025
225=05026
226=05027
227=05028
228=05029
229=05030
230=05031
231=05032
232=05033
233=05034
234=05035
235=05036
236=05037
237=05038
238=05039
239=05040
240=05041
241=05042
242=05043
243=05044
244=05045
245=05046
246=05047
247=05048
248=05049
249=05050
250=05051
251=05052
252=05053
253=05054
254=05055
255=05056
256=05057
257=05058
258=05059
259=05060
260=05061
261=05062
262=05063
263=05064
264=05065
265=05066
266=05067
267=05068
268=05069
269=05070
270=05071
271=05072
272=05073
273=05074
274=05075
275=05076
276=05077
277=05078
278=05079
279=05080
280=05081
281=05082
282=05083
283=05084
284=05085
285=05086
286=05087
287=05088
288=05089
289=05090
290=05091
291=05092
292=05093
293=05094
294=05095
295=05096
296=05097
297=05098
298=05099
299=05100
300=05101
301=05102
302=05103
303=05104
304=05105
305=05106
306=05107
307=05108
308=05109
309=05110
310=05111
311=05112
312=05113
313=05114
314=05115
315=05116
316=05117
317=05118
318=05119
319=05120
320=05121
321=05122
322=05123
323=05124
324=05125
325=05126
326=05127
327=05128
328=05129
329=05130
330=05131
331=05132
332=05133
333=05134
334=05135
335=05136
336=05137
337=05138
338=05139
339=05140
340=05141
341=05142
342=05143
343=05144
344=05145

55
src/resources/common_province_coding.properties
View File

@@ -1,55 +0,0 @@
Beijing=110000
Tianjin=120000
Hebei=130000
Shanxi=140000
InnerMongoliaAutonomousRegion=150000
Liaoning=210000
Jilin=220000
Heilongjiang=230000
Shanghai=310000
Jiangsu=320000
Zhejiang=330000
Anhui=340000
Fujian=350000
Jiangxi=360000
Shandong=370000
Henan=410000
Hubei=420000
Hunan=430000
Guangdong=440000
GuangxiZhuangAutonomousRegion=450000
Hainan=460000
Chongqing=500000
Sichuan=510000
Guizhou=520000
Yunnan=530000
TibetAutonomousRegion=540000
Shaanxi=610000
Gansu=620000
Qinghai=630000
NingxiaHuiAutonomousRegion=640000
Taiwan=710000
HongKong=810000
Macao=820000
Tibet=540000
TaipeiCity=710100
Kaohsiung=710200
Tainan=710300
TaichungCity=710400
Nantou=710600
Keelung=710700
Hsinchu=710800
Chiayi=710900
NewTaipei=711100
Yilan=711200
HsinchuCounty=711300
Taoyuan=711400
Miaoli=711500
Changhua=711700
ChiayiCounty=711900
Yunlin=712100
Pingtung=712400
Taitung=712500
Hualien=712600
PenghuCounty=712700

68
src/resources/service_bolt_config.properties
View File

@@ -1,68 +0,0 @@
#Flooding-spout-field
#conn-bolt-StreamID
conn.dns.streamid=CONN-DNS-SOURCE
conn.ftp.streamid=CONN-FTP-SOURCE
conn.http.streamid=CONN-HTTP-SOURCE
conn.ip.streamid=CONN-IP-SOURCE
conn.jstx.streamid=CONN-JSTX-SOURCE
conn.mail.streamid=CONN-MAIL-SOURCE
conn.ssl.streamid=CONN-SSL-SOURCE
conn.vpn.streamid=CONN-VPN-SOURCE
#conn-ComponentId-spout
kafka.spout=KAFKA-SPOUT
#conn-ComponentId-bolt
conn.parse.componentid=CONN-PARSE-COMPONENTID
conn.ybinputsize.streamid=CONN-YBInputSize-STREAMID
#safe-spout-field
safe.spout.field=SAFE-SOURCE
#GK-SOURCE-STREAMID
gk.ip.streamid=GK-IP-STREAMID
gk.dns.streamid=GK-DNS-STREAMID
gk.http.streamid=GK-HTTP-STREAMID
gk.ssl.streamid=GK-SSL-STREAMID
gk.wygjc.streamid=GK-WYGJC-STREAMID
gk.mail.streamid=GK-MAIL-STREAMID
gk.ftp.streamid=GK-FTP-STREAMID
gk.vpn.streamid=GK-VPN-STREAMID
gk.jstx.streamid=GK-JSTX-STREAMID
gk.count.streamid=GK-COUNT-STREAMID
#JC-SOURCE-STREAMID
jc.ip.streamid=JC-IP-STREAMID
jc.dns.streamid=JC-DNS-STREAMID
jc.http.streamid=JC-HTTP-STREAMID
jc.ssl.streamid=JC-SSL-STREAMID
jc.wygjc.streamid=JC-WYGJC-STREAMID
jc.mail.streamid=JC-MAIL-STREAMID
jc.ftp.streamid=JC-FTP-STREAMID
jc.vpn.streamid=JC-VPN-STREAMID
jc.jstx.streamid=JC-JSTX-STREAMID
jc.pic.streamid=JC-PIC-STREAMID
jc.media.streamid=JC-MEDIA-STREAMID
jc.count.streamid=JC-COUNT-STREAMID

200
src/resources/service_flow_config.properties
View File

@@ -1,200 +0,0 @@
#管理kafka地址
bootstrap.servers=10.111.72.5:9092,10.111.72.6:9092,10.111.72.7:9092,10.111.72.8:9092,10.111.72.9:9092,10.111.72.16:9092,10.111.72.17:9092,10.111.72.18:9092
#latest/earliest
auto.offset.reset=earliest
#压缩模式 none or snappy
kafka.compression.type=none
topology.logo=jc
#通联日志kafka主题
kafka.topic=CONNECTION-RECORD-COMPLETED-LOG
#策略日志kafka主题
kafka.safe.topic=SECURITY-EVENT-COMPLETED-LOG
#消费者组id-读取topic,存储该spout id的消费offset信息，每次启动前需要更改
group.id=xj-jc-group20210709
#kafka消费者每次拉取数据的时间
kafka.consumer.poll=1000
#storm topology workers
topology.workers=8
#spout并行度，建议与kafka分区一致
max.spout.parallelism=8
spout.parallelism=8
#ack设置 1启动ack 0不启动ack
topology.num.acks=0
#spou睡眠间隔
topology.spout.sleep.time=1
#通联程序批处理的时间
topology.tick.tuple.freq.secs=60
#通联程序没批处理的条数
batch.inter.num=100000
parse.parallelism=100
send.parallelism=20
ip.send.parallelism=50
ssl.send.parallelism=50
http.send.parallelism=50
count.send.parallelism=1
unstructured.send.parallelism=50
#count指标数据时间批次（单位为秒）
count.topology.tick.tuple.freq.secs=300
#prometheus-httpserver
prometheus.pushgateway.address=10.111.143.240:9274
#http-post请求超时时间
http.post.socketTimeOut=60
#http-post发送设置
http.connectionRequestTimeout=1000
http.connectTimeout=1000
http.socketTimeout=1000
http.sent.count=10
http.retry.count=2
http.heard.content.type=utf-8
http.hearder.user=yd_xgs_source
http.hearder.password=yd_xgs_source@2021
http.hearder.format=avro
http.hearder.conection=keep-alilve
#http-post后端接受地址
http.post.ip=10.111.151.249,10.111.151.250,10.111.151.251,10.111.151.252,10.111.151.253,10.111.153.248,10.111.153.249,10.111.153.250,10.111.153.251,10.111.153.252,10.111.153.253
http.post.port=10080
hos.url=http://10.111.143.200:9098/hos/
hos.pool.maxconnections=400
hos.pool.maxerrorretry=3
hos.pool.requesttimeout=0
hos.pool.connectiontimeout=60
hos.pool.connectionttl=-1
hos.pool.sockettimeout=50
s3.connectionRequestTimeout=1000
s3.connectTimeout=1000
s3.socketTimeout=1000
s3.sent.count=10
s3.hearder.username=yd_xgs_s3
s3.hearder.password=yd_xgs_s3@2021
s3.hearder.filetype=pcm
s3.hearder.filetitle=ydtest-file
s3.hearder.isfulltex=false
s3.hearder.contenttype=application/octet-stream
s3.post.ip=10.111.144.4,10.111.144.5,10.111.144.6
s3.post.port=8888
t.http.rzzb=t-http-rzzb
t.http.jczb=t-http-jczb
t.http.gkzb=t-http-gkzb
t.mail.rzzb=t-mail-rzzb
t.mail.jczb=t-mail-jczb
t.mail.gkzb=t-mail-gkzb
t.wygjc.jcrz=t-wygjc-jcrz
t.wygjc.gkrz=t-wygjc-gkrz
t.pic.jczb=t-pic-jczb
t.media.jczb=t-media-jczb
#通联数据的schema
ftp.rzzb.schema={"type":"record","name":"FTP_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
dns.rzzb.schema={"type":"record","name":"DNS_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DNS_NAME","type":["string","null"]},{"name":"DNS_VALUE","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
http.rzzb.schema={"type":"record","name":"HTTP_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"COOKIE","type":["string","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
ip.rzzb.schema={"type":"record","name":"IP_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
jstx.rzzb.schema={"type":"record","name":"JSTX_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DZ_ID","type":["string","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"URL_AGENT","type":["string","null"]},{"name":"USER_ID","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"RESERVED","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
ssl.rzzb.schema={"type":"record","name":"SSL_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"ZSMC","type":["string","null"]},{"name":"BFZ","type":["string","null"]},{"name":"SYZ","type":["string","null"]},{"name":"FWQMC","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
vpn.rzzb.schema={"type":"record","name":"VPN_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"URL","type":["int","null"]},{"name":"USERNAME","type":["int","null"]},{"name":"PWD","type":["int","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
mail.rzzb.schema={"type":"record","name":"MAIL_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"MAIL_SENDER","type":["string","null"]},{"name":"MAIL_RECEVER","type":["string","null"]},{"name":"RECEVER_NUM","type":["int","null"]},{"name":"MAIL_CC","type":["string","null"]},{"name":"CC_NUM","type":["int","null"]},{"name":"MAIL_SUBJECT","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"ACCESSORY","type":["int","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"FJFWS","type":["string","null"]},{"name":"FJFWS_SZD","type":["string","null"]},{"name":"SJFWS","type":["string","null"]},{"name":"SJFWS_SZD","type":["string","null"]},{"name":"XYLX","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
conn.post.dns.topic=t_dns_rzzb
conn.post.ftp.topic=t_ftp_rzzb
conn.post.http.topic=t_http_rzzb
conn.post.ip.topic=t_ip_rzzb
conn.post.jstx.topic=t_jstx_rzzb
conn.post.mail.topic=t_mail_rzzb
conn.post.ssl.topic=t_ssl_rzzb
conn.post.vpn.topic=t_vpn_rzzb
gk.post.dns.topic=t_dns_gkzb
gk.post.ftp.topic=t_ftp_gkzb
gk.post.http.topic=t_http_gkzb
gk.post.ip.topic=t_ip_gkzb
gk.post.jstx.topic=t_jstx_gkzb
gk.post.mail.topic=t_mail_gkzb
gk.post.ssl.topic=t_ssl_gkzb
gk.post.vpn.topic=t_vpn_gkzb
gk.post.wygjc.topic=t_wygjc_gkrz
jc.post.dns.topic=t_dns_jczb
jc.post.ftp.topic=t_ftp_jczb
jc.post.http.topic=t_http_jczb
jc.post.ip.topic=t_ip_jczb
jc.post.jstx.topic=t_jstx_jczb
jc.post.mail.topic=t_mail_jczb
jc.post.ssl.topic=t_ssl_jczb
jc.post.vpn.topic=t_vpn_jczb
jc.post.wygjc.topic=t_wygjc_jcrz
jc.post.pic.topic=t_pic_jczb
jc.post.media.topic=t_media_jczb
#gk-schema
ip.gkrz.schema={"type":"record","name":"IP_GKRZ","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
dns.gkzb.schema={"type":"record","name":"DNS_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"DNS_NAME","type":["string","null"]},{"name":"DNS_VALUE","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":["int","null"]},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
http.gkzb.schema={"type":"record","name":"HTTP_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"COOKIE","type":["string","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
ssl.gkzb.schema={"type":"record","name":"SSL_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"ZSMC","type":["string","null"]},{"name":"BFZ","type":["string","null"]},{"name":"SYZ","type":["string","null"]},{"name":"FWQMC","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
wygjc.gkrz.schema={"type":"record","name":"WYGJC_GKRZ","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"JC_URL","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"SSYQ","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
ftp.gkzb.schema={"type":"record","name":"FTPGJZ_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
vpn.gkzb.schema={"type":"record","name":"VPN_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"URL","type":["string","null"]},{"name":"USERNAME","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
jstx.gkzb.schema={"type":"record","name":"JSTX_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"DZ_ID","type":["string","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"USER_ID","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"RESERVED","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
mail.gkzb.schema={"type":"record","name":"MAIL_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"MAIL_SENDER","type":["string","null"]},{"name":"MAIL_RECEVER","type":["string","null"]},{"name":"RECEVER_NUM","type":["int","null"]},{"name":"MAIL_CC","type":["string","null"]},{"name":"CC_NUM","type":["int","null"]},{"name":"MAIL_SUBJECT","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"ACCESSORY","type":["int","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"FJFWS","type":["string","null"]},{"name":"FJFWS_SZD","type":["string","null"]},{"name":"SJFWS","type":["string","null"]},{"name":"SJFWS_SZD","type":["string","null"]},{"name":"XYLX","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
#jc-schema
dns.jczb.schema={"type":"record","name":"DNS_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DNS_NAME","type":["string","null"]},{"name":"DNS_VALUE","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
ftp.jczb.schema={"type":"record","name":"FTPGJZ_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
http.jczb.schema={"type":"record","name":"HTTP_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"COOKIE","type":["string","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
ip.jczb.schema={"type":"record","name":"IP_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
jstx.jczb.schema={"type":"record","name":"JSTX_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DZ_ID","type":["string","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"USER_ID","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"RESERVED","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
ssl.jczb.schema={"type":"record","name":"TDZS_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"ZSMC","type":["string","null"]},{"name":"BFZ","type":["string","null"]},{"name":"SYZ","type":["string","null"]},{"name":"FWQMC","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
vpn.jczb.schema={"type":"record","name":"VPN_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"URL","type":["string","null"]},{"name":"USERNAME","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
wygjz.jczb.schema={"type":"record","name":"WYGJC_JCRZ","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"JC_URL","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"SSYQ","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
pic.jczb.schema={"type":"record","name":"PIC_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"YYZL","type":["int","null"]},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"PIC_NAME","type":["string","null"]},{"name":"PIC_URL","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"PIC_TYPE","type":["string","null"]},{"name":"PIC_CONTENT","type":["bytes","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
media.jczb.schema={"type":"record","name":"MEDIA_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YYZL","type":["int","null"]},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"MEDIA_NAME","type":["string","null"]},{"name":"MEDIA_URL","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"MEDIA_TYPE","type":["string","null"]},{"name":"MEDIA_CONTENT","type":["bytes","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}
mail.jczb.schema={"type":"record","name":"MAILGJZ_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"MAIL_SENDER","type":["string","null"]},{"name":"MAIL_RECEVER","type":["string","null"]},{"name":"RECEVER_NUM","type":["int","null"]},{"name":"MAIL_CC","type":["string","null"]},{"name":"CC_NUM","type":["int","null"]},{"name":"MAIL_SUBJECT","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"ACCESSORY","type":["int","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"FJFWS","type":["string","null"]},{"name":"FJFWS_SZD","type":["string","null"]},{"name":"SJFWS","type":["string","null"]},{"name":"SJFWS_SZD","type":["string","null"]},{"name":"XYLX","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]}

29
src/test/java/People.java
View File

@@ -1,29 +0,0 @@
public class People {
private String name;
private int age;
private String weight;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public int getAge() {
return age;
}
public void setAge(int age) {
this.age = age;
}
public String getWeight() {
return weight;
}
public void setWeight(String weight) {
this.weight = weight;
}
}

8
src/test/java/Test.java
View File

@@ -15,12 +15,4 @@ public class Test {
} }
return sb.toString(); return sb.toString();
} }
public static void main(String[] args) {
People people = new People();
if (2>1){
people.setAge(2);
}
System.out.println(people.getAge());
}
} }

34
src/test/java/TestDy.java
View File

@@ -1,16 +1,30 @@
import com.amazonaws.services.s3.model.CSVOutput;
import org.apache.logging.log4j.util.PropertiesUtil;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import static cn.ac.iie.storm.utils.config.GetMap.*; import static cn.ac.iie.storm.utils.general.GetMap.getDy;
import static cn.ac.iie.storm.utils.general.MailTrans.*;
import static cn.ac.iie.storm.utils.general.TransFormUtils.*; import static cn.ac.iie.storm.utils.general.TransFormUtils.*;
public class TestDy { public class TestDy {
public static void main(String[] args) { private static Properties propService_city = new Properties();
// String mail = getSecurityMailDetail_id(410, "0300231", "WEBMAIL"); InputStream cityNumbering = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_city_coding.properties");
// System.out.println(mail); // propService_city.load(cityNumbering);
// Map<String, String> cityMap = new HashMap<String, String>((Map) propService_city);
String imap = getYjxy(510, "03000", "IMAP"); // for (String key : cityMap.keySet()) {
System.out.println(imap); // String str = key.replace(",", ".");
} // String[] split = str.split("\\.");
// if (split.length != 3) {
// System.out.println(key);
//// System.out.println(split[2] + "." + split[1] + "." + split[0] + "=" + cityMap.get(key));
// }
// }
public static void main(String[] args) throws IOException {
String dy = getDy("China.Fujian.Other");
System.out.println(dy);
}
} }

2
src/test/java/TestHos.java
View File

@@ -1,7 +1,5 @@
import java.io.*; import java.io.*;
import static cn.ac.iie.storm.utils.config.GetMap.getDy;
import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost; import static cn.ac.iie.storm.utils.http.HttpAsncClient.getConnAnsyHttpPost;
public class TestHos { public class TestHos {

2
src/test/java/testAppId.java
View File

@@ -1,6 +1,6 @@
import java.io.IOException; import java.io.IOException;
import static cn.ac.iie.storm.utils.config.GetMap.getJstxDetail_id; import static cn.ac.iie.storm.utils.general.GetMap.getJstxDetail_id;
public class testAppId { public class testAppId {
public static void main(String[] args) throws IOException { public static void main(String[] args) throws IOException {