gfwleak/zx-dataserker-yb-flume-cus-sink-file
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mail,http,ftp三个表新增若干字段

Browse Source
This commit is contained in:
姜鹏辉
2022-06-02 15:13:14 +08:00
parent 37d3d2c60c
commit ecb608f94f
5 changed files with 184 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/avroUtils/avroRecord/GetAvroRecordByTopicUtils.java
View File

@@ -10,6 +10,7 @@ import cn.ac.iie.cusflume.sink.bean.fileBean.NTC_COLLECT_MAIL_LOG;
import cn.ac.iie.cusflume.sink.bean.tagBean.LOG_D_TAG_BEAN;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
//import com.sun.deploy.util.StringUtils;
import com.zdjizhi.utils.StringUtil;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
@@ -71,6 +72,23 @@ public class GetAvroRecordByTopicUtils {
}
private static GenericRecord getCollFtpDocAvroRecord(GenericRecord recordAvro, String dataJson, Schema schemaStr) {
// NTC_COLLECT_MAIL_LOG ntcCollectMailLog = JSONObject.parseObject(dataJson, NTC_COLLECT_MAIL_LOG.class);
//
// if (StringUtil.isNotBlank(ntcCollectMailLog.getSubject())) {
// String subjectCharset = JSONObject.parseObject(dataJson).getString("subject_charset");
// ntcCollectMailLog.setSubject(DecodeUtils.base64Str(ntcCollectMailLog.getSubject(), subjectCharset));
// }
// recordAvro = converter.convertToGenericDataRecord(JSONObject.toJSONString(ntcCollectMailLog).getBytes(), schemaStr);
// NTC_COLLECT_FTP_DOC_LOG ntcCollectFtpDocLog = JSONObject.parseObject(dataJson, NTC_COLLECT_FTP_DOC_LOG.class);
//
// if (StringUtil.isNotBlank(ntcCollectFtpDocLog.getFtp_file_name())) {
// ntcCollectFtpDocLog.
// }
//
recordAvro = converter.convertToGenericDataRecord(dataJson.getBytes(), schemaStr);
return recordAvro;
}

50
yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_FTP_DOC_LOG.java
View File

@@ -13,6 +13,11 @@ public class NTC_COLLECT_FTP_DOC_LOG extends CommonLog_File {
// private String ftp_content_id;//旧-20200506废弃
private String content_path;//新-20200506新修改
private String ftp_username; // v4.1新增
private String ftp_password; // 20220314新增,v4.2
private int ftp_dir;
private String ftp_info;
private String ftp_cmd;
private String ftp_file_name;
//20200211新增
private LOG_D_TAG_BEAN d_tag;//json版本
@@ -63,6 +68,46 @@ public class NTC_COLLECT_FTP_DOC_LOG extends CommonLog_File {
public void setFtp_username(String ftp_username) { this.ftp_username = ftp_username; }
public String getFtp_password() {
return ftp_password;
}
public void setFtp_password(String ftp_password) {
this.ftp_password = ftp_password;
}
public int getFtp_dir() {
return ftp_dir;
}
public void setFtp_dir(int ftp_dir) {
this.ftp_dir = ftp_dir;
}
public String getFtp_info() {
return ftp_info;
}
public void setFtp_info(String ftp_info) {
this.ftp_info = ftp_info;
}
public String getFtp_cmd() {
return ftp_cmd;
}
public void setFtp_cmd(String ftp_cmd) {
this.ftp_cmd = ftp_cmd;
}
public String getFtp_file_name() {
return ftp_file_name;
}
public void setFtp_file_name(String ftp_file_name) {
this.ftp_file_name = ftp_file_name;
}
@Override
public String toString() {
return "NTC_COLLECT_FTP_DOC_LOG{" +
@@ -70,6 +115,11 @@ public class NTC_COLLECT_FTP_DOC_LOG extends CommonLog_File {
", ftp_content='" + ftp_content + '\'' +
", content_path='" + content_path + '\'' +
", ftp_username='" + ftp_username + '\'' +
", ftp_password='" + ftp_password + '\'' +
", ftp_dir='" + ftp_dir + '\'' +
", ftp_info='" + ftp_info + '\'' +
", ftp_cmd='" + ftp_cmd + '\'' +
", ftp_file_name='" + ftp_file_name + '\'' +
", d_tag=" + d_tag +
", x_tag='" + x_tag + '\'' +
'}';

46
yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_HTTP_DOC_LOG.java
View File

@@ -20,6 +20,13 @@ public class NTC_COLLECT_HTTP_DOC_LOG extends CommonLog_File {
private String res_body_file_path;//新增-20191216
private String x_forward_for; // 20210401新增
private String req_head; // 20220314新增,v4.2
private String res_head;
private String http_charset;
private String http_file_name;
//20191209新增
// private String d_tag;//json字符串版本
private LOG_D_TAG_BEAN d_tag;//json版本-20191216
@@ -224,6 +231,39 @@ public class NTC_COLLECT_HTTP_DOC_LOG extends CommonLog_File {
this.res_body_file_id = res_body_file_id;
}
public String getReq_head() {
return req_head;
}
public void setReq_head(String req_head) {
this.req_head = req_head;
}
public String getRes_head() {
return res_head;
}
public void setRes_head(String res_head) {
this.res_head = res_head;
}
public String getHttp_charset() {
return http_charset;
}
public void setHttp_charset(String http_charset) {
this.http_charset = http_charset;
}
public String getHttp_file_name() {
return http_file_name;
}
public void setHttp_file_name(String http_file_name) {
this.http_file_name = http_file_name;
}
@Override
public String toString() {
return "NTC_COLLECT_HTTP_DOC_LOG{" +
@@ -239,9 +279,15 @@ public class NTC_COLLECT_HTTP_DOC_LOG extends CommonLog_File {
", doc_type='" + doc_type + '\'' +
", req_body_file_path='" + req_body_file_path + '\'' +
", res_body_file_path='" + res_body_file_path + '\'' +
", req_head='" + req_head + '\'' +
", res_head='" + res_head + '\'' +
", http_charset='" + http_charset + '\'' +
", http_file_name='" + http_file_name + '\'' +
", d_tag=" + d_tag +
", x_tag='" + x_tag + '\'' +
", x_forward_for='" + x_forward_for + '\'' +
", res_body_file='" + res_body_file + '\'' +
", res_body_file_id='" + res_body_file_id + '\'' +
'}';
}
}

67
yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_MAIL_LOG.java
View File

@@ -40,7 +40,16 @@ public class NTC_COLLECT_MAIL_LOG extends CommonLog_File {
private String resent_to;
private String resent_date;
private String message_id;
private String x_originating_ip;// 20210401新增
private String x_originating_ip;// 20210401新增,v4.1
private String mail_from_name; // 20220314新增,v4.2
private String mail_to_name;
private String mail_username;
private String mail_password;
private String attachments_name;
private String eml_charset;
// private String subject_charset; // 该字段在4.2版本之前就已经包含在代码中
public String getMail_proto() {
return mail_proto;
@@ -174,7 +183,7 @@ public class NTC_COLLECT_MAIL_LOG extends CommonLog_File {
return received;
}
public void setRecevied(String received) {
public void setReceived(String received) {
this.received = received;
}
@@ -246,6 +255,54 @@ public class NTC_COLLECT_MAIL_LOG extends CommonLog_File {
public void setX_originating_ip(String x_originating_ip) { this.x_originating_ip = x_originating_ip; }
public String getMail_from_name() {
return mail_from_name;
}
public void setMail_from_name(String mail_from_name) {
this.mail_from_name = mail_from_name;
}
public String getMail_to_name() {
return mail_to_name;
}
public void setMail_to_name(String mail_to_name) {
this.mail_to_name = mail_to_name;
}
public String getMail_username() {
return mail_username;
}
public void setMail_username(String mail_username) {
this.mail_username = mail_username;
}
public String getMail_password() {
return mail_password;
}
public void setMail_password(String mail_password) {
this.mail_password = mail_password;
}
public String getAttachments_name() {
return attachments_name;
}
public void setAttachments_name(String attachments_name) {
this.attachments_name = attachments_name;
}
public String getEml_charset() {
return eml_charset;
}
public void setEml_charset(String eml_charset) {
this.eml_charset = eml_charset;
}
@Override
public String toString() {
return "NTC_COLLECT_MAIL_LOG{" +
@@ -275,6 +332,12 @@ public class NTC_COLLECT_MAIL_LOG extends CommonLog_File {
", resent_date='" + resent_date + '\'' +
", message_id='" + message_id + '\'' +
", x_originating_ip='" + x_originating_ip + '\'' +
", mail_from_name='" + mail_from_name + '\'' +
", mail_to_name='" + mail_to_name + '\'' +
", mail_username='" + mail_username + '\'' +
", mail_password='" + mail_password + '\'' +
", attachments_name='" + attachments_name + '\'' +
", eml_charset='" + eml_charset + '\'' +
'}';
}
}

10
yb_http_avro_sink_file/src/main/resources/flume_config.properties
View File

@@ -8,7 +8,7 @@ http.async.socketTimeout=60000
http.async.connectTimeout=30000
#异步Http客户端-连接池最大连接数
http.async.poolSize=4cdcd
http.async.poolSize=4
#异步Http客户端-每个主机的并发最多只有1500
http.async.maxPerRoute=4
@@ -24,10 +24,10 @@ schema.ntc-collect-ssl-log={"type":"record","name":"collssl","fields":[{"name":"
schema.ntc-collect-dns-log={"type":"record","name":"colldns","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"dns_sub","type":"long","doc":"label:DNS种类","default":0},{"name":"hdr","type":"string","doc":"label:hdr信息","default":""},{"name":"flags","type":"string","doc":"label:标志","default":""},{"name":"question","type":"string","doc":"label:问题计数","default":""},{"name":"rr","type":"string","doc":"label:资源记录","default":""}]}
schema.ntc-collect-http-hdr-log={"type":"record","name":"collhttphdr","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"url","type":"string","doc":"label:HTTP url地址","default":""},{"name":"referer","type":"string","doc":"label:HTTP访问referer","default":""},{"name":"user_agent","type":"string","doc":"label:HTTP访问user agent","default":""},{"name":"http_seq","type":"int","doc":"label:HTTP请求会话序列号","default":0},{"name":"req_line","type":"string","doc":"label:请求行","default":""},{"name":"res_line","type":"string","doc":"label:应答行","default":""},{"name":"cookie","type":"string","doc":"label:cookiez值","default":""},{"name":"content_type","type":"string","doc":"label:content_type 值","default":""},{"name":"content_len","type":"string","doc":"label:content_len 值","default":""},{"name":"host","type":"string","doc":"label:host值","default":""},{"name":"transfer_encoding","type":"string","doc":"label:transfer_encoding","default":""},{"name":"transfer_length","type":"string","doc":"label:transfer_length","default":""},{"name":"cont_encoding","type":"string","doc":"label:cont_encoding","default":""},{"name":"cont_disposition","type":"string","doc":"label:cont_disposition","default":""},{"name":"charset","type":"string","doc":"label:编码方式","default":""},{"name":"isn","type":"long","doc":"label:","default":0},{"name":"proxy_flag","type":"int","doc":"label:","default":0},{"name":"x_forward_for","type":"string","doc":"label:","default":""},{"name":"http_c2s_head","type":"string","doc":"label:客户端到服务端负载前100字节","default":""},{"name":"http_s2c_head","type":"string","doc":"label:服务端到客户端负载前100字节","default":""}]}
#文件消息-和前端定义相关联
schema.ntc-collect-mail-log={"type":"record","name":"collmail","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"mail_proto","type":"string","doc":"label:邮件协议","default":""},{"name":"mail_from","type":"string","doc":"label:邮件发件人","default":""},{"name":"mail_to","type":"string","doc":"label:邮件收件人","default":""},{"name":"mail_cc","type":"string","doc":"label:邮件抄送人","default":""},{"name":"mail_date","type":"string","doc":"label:邮件日期","default":""},{"name":"subject","type":"string","doc":"label:邮件主题","default":""},{"name":"eml_file_path","type":"string","doc":"label:邮件EML文件转储地址","default":""},{"name":"attachments_path","type":"string","doc":"label:附件转储地址列表","default":""},{"name":"bcc","type":"string","doc":"label:暗抄送","default":""},{"name":"received","type":"string","doc":"label:系统添加部分","default":""},{"name":"return_path","type":"string","doc":"label:电子邮件地址","default":""},{"name":"reply_to","type":"string","doc":"label:回复","default":""},{"name":"mail_from_cmd","type":"string","doc":"label:mail_from_cc","default":""},{"name":"rcpt_to_cmd","type":"string","doc":"label:mail_to_cc","default":""},{"name":"resent_from","type":"string","doc":"label:转发邮件发件人","default":""},{"name":"resent_to","type":"string","doc":"label:转发邮件收件人","default":""},{"name":"resent_date","type":"string","doc":"label:转发时间","default":""},{"name":"message_id","type":"string","doc":"label:","default":""},{"name":"x_originating_ip","type":"string","doc":"label:邮件经过的服务器IP","default":""}]}
schema.ntc-collect-http-doc-log={"type":"record","name":"collhttpav","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"url","type":"string","doc":"label:HTTP_url地址","default":""},{"name":"referer","type":"string","doc":"label:HTTP访问referer","default":""},{"name":"user_agent","type":"string","doc":"label:HTTP访问user_agent","default":""},{"name":"http_seq","type":"int","doc":"label:HTTP请求会话序列号","default":0},{"name":"req_line","type":"string","doc":"label:请求行","default":""},{"name":"res_line","type":"string","doc":"label:应答行","default":""},{"name":"cookie","type":"string","doc":"label:cookie值","default":""},{"name":"content_type","type":"string","doc":"label:content_type值","default":""},{"name":"content_len","type":"long","doc":"label:content_len值","default":0},{"name":"doc_type","type":"string","doc":"label:文件类型","default":""},{"name":"req_body_file_path","type":"string","doc":"label:请求体转储文件地址","default":""},{"name":"res_body_file_path","type":"string","doc":"label:应答体转储文件地址","default":""},{"name":"x_forward_for","type":"string","doc":"label:","default":""}]}
schema.ntc-collect-http-av-log={"type":"record","name":"collhttpav","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"url","type":"string","doc":"label:HTTP_url地址","default":""},{"name":"referer","type":"string","doc":"label:HTTP访问referer","default":""},{"name":"user_agent","type":"string","doc":"label:HTTP访问user_agent","default":""},{"name":"http_seq","type":"int","doc":"label:HTTP请求会话序列号","default":0},{"name":"req_line","type":"string","doc":"label:请求行","default":""},{"name":"res_line","type":"string","doc":"label:应答行","default":""},{"name":"cookie","type":"string","doc":"label:cookie值","default":""},{"name":"content_type","type":"string","doc":"label:content_type值","default":""},{"name":"content_len","type":"long","doc":"label:content_len值","default":0},{"name":"doc_type","type":"string","doc":"label:文件类型","default":""},{"name":"req_body_file_path","type":"string","doc":"label:请求体转储文件地址","default":""},{"name":"res_body_file_path","type":"string","doc":"label:应答体转储文件地址","default":""},{"name":"x_forward_for","type":"string","doc":"label:","default":""}]}
schema.ntc-collect-ftp-doc-log={"type":"record","name":"collftpdoc","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"ftp_url","type":"string","doc":"label:ftp_url地址","default":""},{"name":"content_path","type":"string","doc":"label:FTP文件转储文件地址","default":""},{"name":"ftp_username","type":"string","doc":"label:FTP登录用户名","default":""}]}
schema.ntc-collect-mail-log={"type":"record","name":"collmail","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"mail_proto","type":"string","doc":"label:邮件协议","default":""},{"name":"mail_from","type":"string","doc":"label:邮件发件人","default":""},{"name":"mail_to","type":"string","doc":"label:邮件收件人","default":""},{"name":"mail_cc","type":"string","doc":"label:邮件抄送人","default":""},{"name":"mail_date","type":"string","doc":"label:邮件日期","default":""},{"name":"subject","type":"string","doc":"label:邮件主题","default":""},{"name":"eml_file_path","type":"string","doc":"label:邮件EML文件转储地址","default":""},{"name":"attachments_path","type":"string","doc":"label:附件转储地址列表","default":""},{"name":"bcc","type":"string","doc":"label:暗抄送","default":""},{"name":"received","type":"string","doc":"label:系统添加部分","default":""},{"name":"return_path","type":"string","doc":"label:电子邮件地址","default":""},{"name":"reply_to","type":"string","doc":"label:回复","default":""},{"name":"mail_from_cmd","type":"string","doc":"label:mail_from_cc","default":""},{"name":"rcpt_to_cmd","type":"string","doc":"label:mail_to_cc","default":""},{"name":"resent_from","type":"string","doc":"label:转发邮件发件人","default":""},{"name":"resent_to","type":"string","doc":"label:转发邮件收件人","default":""},{"name":"resent_date","type":"string","doc":"label:转发时间","default":""},{"name":"message_id","type":"string","doc":"label:","default":""},{"name":"x_originating_ip","type":"string","doc":"label:邮件经过的服务器IP","default":""},{"name":"mail_from_name","type":"string","doc":"label:发件人名","default":""},{"name":"mail_to_name","type":"string","doc":"label:收件人名","default":""},{"name":"mail_username","type":"string","doc":"label:登录用户名","default":""},{"name":"mail_password","type":"string","doc":"label:用户密码","default":""},{"name":"attachments_name","type":"string","doc":"label:附件名列表","default":""},{"name":"eml_charset","type":"string","doc":"label:邮件编码","default":""},{"name":"subject_charset","type":"string","doc":"label:邮件主题编码","default":""}]}
schema.ntc-collect-http-doc-log={"type":"record","name":"collhttpdoc","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"url","type":"string","doc":"label:HTTP_url地址","default":""},{"name":"referer","type":"string","doc":"label:HTTP访问referer","default":""},{"name":"user_agent","type":"string","doc":"label:HTTP访问user_agent","default":""},{"name":"http_seq","type":"int","doc":"label:HTTP请求会话序列号","default":0},{"name":"req_line","type":"string","doc":"label:请求行","default":""},{"name":"res_line","type":"string","doc":"label:应答行","default":""},{"name":"cookie","type":"string","doc":"label:cookie值","default":""},{"name":"content_type","type":"string","doc":"label:content_type值","default":""},{"name":"content_len","type":"long","doc":"label:content_len值","default":0},{"name":"doc_type","type":"string","doc":"label:文件类型","default":""},{"name":"req_body_file_path","type":"string","doc":"label:请求体转储文件地址","default":""},{"name":"res_body_file_path","type":"string","doc":"label:应答体转储文件地址","default":""},{"name":"x_forward_for","type":"string","doc":"label:","default":""},{"name":"req_head","type":"string","doc":"label:http请求头","default":""},{"name":"res_head","type":"string","doc":"label:HTTP应答头","default":""},{"name":"http_charset","type":"string","doc":"label:HTTP编码方式","default":""},{"name":"http_file_name","type":"string","doc":"label:HTTP文件名","default":""}]}
schema.ntc-collect-http-av-log={"type":"record","name":"collhttpav","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"url","type":"string","doc":"label:HTTP_url地址","default":""},{"name":"referer","type":"string","doc":"label:HTTP访问referer","default":""},{"name":"user_agent","type":"string","doc":"label:HTTP访问user_agent","default":""},{"name":"http_seq","type":"int","doc":"label:HTTP请求会话序列号","default":0},{"name":"req_line","type":"string","doc":"label:请求行","default":""},{"name":"res_line","type":"string","doc":"label:应答行","default":""},{"name":"cookie","type":"string","doc":"label:cookie值","default":""},{"name":"content_type","type":"string","doc":"label:content_type值","default":""},{"name":"content_len","type":"long","doc":"label:content_len值","default":0},{"name":"doc_type","type":"string","doc":"label:文件类型","default":""},{"name":"req_body_file_path","type":"string","doc":"label:请求体转储文件地址","default":""},{"name":"res_body_file_path","type":"string","doc":"label:应答体转储文件地址","default":""},{"name":"x_forward_for","type":"string","doc":"label:","default":""},{"name":"req_head","type":"string","doc":"label:http请求头","default":""},{"name":"res_head","type":"string","doc":"label:HTTP应答头","default":""},{"name":"http_charset","type":"string","doc":"label:HTTP编码方式","default":""},{"name":"http_file_name","type":"string","doc":"label:HTTP文件名","default":""}]}
schema.ntc-collect-ftp-doc-log={"type":"record","name":"collftpdoc","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"ftp_url","type":"string","doc":"label:ftp_url地址","default":""},{"name":"content_path","type":"string","doc":"label:FTP文件转储文件地址","default":""},{"name":"ftp_username","type":"string","doc":"label:FTP登录用户名","default":""},{"name":"ftp_password","type":"string","doc":"label:FTP登录密码","default":""},{"name":"ftp_dir","type":"int","doc":"label:FTP传输方向","default":""},{"name":"ftp_info","type":"string","doc":"label:FTP登录信息","default":""},{"name":"ftp_cmd","type":"string","doc":"label:FTP交互命令","default":""},{"name":"ftp_file_name","type":"string","doc":"label:","default":""}]}
schema.ntc-collect-telnet-log={"type":"record","name":"colltelnet","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"username","type":"string","doc":"label:用户名","default":""},{"name":"password","type":"string","doc":"label:密码","default":""},{"name":"content_path","type":"string","doc":"label:telnet内容转储文件地址","default":""}]}
#特殊-和前端定义相关联
schema.ntc-collect-file-log={"type":"record","name":"file_tag","fields":[{"type":"string","name":"tag_version","default":""},{"type":"int","name":"data_type","default":0},{"type":"int","name":"data_subtype","default":0},{"type":"int","name":"producer_id","default":0},{"type":"int","name":"data_source","default":0},{"type":{"type":"array","items":"int"},"name":"task_id","default":[]},{"type":"long","name":"file_id","default":0},{"type":"string","name":"file_path","default":""},{"type":"string","name":"file_name","default":""},{"type":"float","name":"file_size","default":0.0},{"type":"string","name":"md5","default":""},{"type":"string","name":"encrypted","default":""},{"type":"string","name":"timestamp","default":""},{"type":"long","name":"parent_id","default":0},{"type":"long","name":"flow_id","default":0}]}