diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..dfb9496 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,10 @@ + + + + \ No newline at end of file diff --git a/.idea/jarRepositories.xml b/.idea/jarRepositories.xml new file mode 100644 index 0000000..1cea22d --- /dev/null +++ b/.idea/jarRepositories.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_101tec_zkclient_0_10.xml b/.idea/libraries/Maven__com_101tec_zkclient_0_10.xml new file mode 100644 index 0000000..eec48fe --- /dev/null +++ b/.idea/libraries/Maven__com_101tec_zkclient_0_10.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_alibaba_fastjson_1_2_47.xml b/.idea/libraries/Maven__com_alibaba_fastjson_1_2_47.xml new file mode 100644 index 0000000..6d9450f --- /dev/null +++ b/.idea/libraries/Maven__com_alibaba_fastjson_1_2_47.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_5.xml b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_5.xml new file mode 100644 index 0000000..ac0efbc --- /dev/null +++ b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_9_5.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_9_5.xml b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_9_5.xml new file mode 100644 index 0000000..afeaac4 --- /dev/null +++ b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_9_5.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_9_1.xml b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_9_1.xml new file mode 100644 index 0000000..95ffc15 --- /dev/null +++ b/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_9_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_code_gson_gson_2_2_2.xml b/.idea/libraries/Maven__com_google_code_gson_gson_2_2_2.xml new file mode 100644 index 0000000..218c71f --- /dev/null +++ b/.idea/libraries/Maven__com_google_code_gson_gson_2_2_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_guava_guava_18_0.xml b/.idea/libraries/Maven__com_google_guava_guava_18_0.xml new file mode 100644 index 0000000..5b17142 --- /dev/null +++ b/.idea/libraries/Maven__com_google_guava_guava_18_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_maxmind_db_maxmind_db_1_2_2.xml b/.idea/libraries/Maven__com_maxmind_db_maxmind_db_1_2_2.xml new file mode 100644 index 0000000..43ae22b --- /dev/null +++ b/.idea/libraries/Maven__com_maxmind_db_maxmind_db_1_2_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_maxmind_geoip2_geoip2_2_12_0.xml b/.idea/libraries/Maven__com_maxmind_geoip2_geoip2_2_12_0.xml new file mode 100644 index 0000000..55c2a46 --- /dev/null +++ b/.idea/libraries/Maven__com_maxmind_geoip2_geoip2_2_12_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_maxmind_geoip_geoip_api_1_3_1.xml b/.idea/libraries/Maven__com_maxmind_geoip_geoip_api_1_3_1.xml new file mode 100644 index 0000000..44f6111 --- /dev/null +++ b/.idea/libraries/Maven__com_maxmind_geoip_geoip_api_1_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_squareup_okhttp3_okhttp_3_9_0.xml b/.idea/libraries/Maven__com_squareup_okhttp3_okhttp_3_9_0.xml new file mode 100644 index 0000000..fd37651 --- /dev/null +++ b/.idea/libraries/Maven__com_squareup_okhttp3_okhttp_3_9_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_squareup_okhttp_okhttp_2_4_0.xml b/.idea/libraries/Maven__com_squareup_okhttp_okhttp_2_4_0.xml new file mode 100644 index 0000000..991b26a --- /dev/null +++ b/.idea/libraries/Maven__com_squareup_okhttp_okhttp_2_4_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_squareup_okio_okio_1_13_0.xml b/.idea/libraries/Maven__com_squareup_okio_okio_1_13_0.xml new file mode 100644 index 0000000..e9b0db6 --- /dev/null +++ b/.idea/libraries/Maven__com_squareup_okio_okio_1_13_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_squareup_retrofit_retrofit_1_9_0.xml b/.idea/libraries/Maven__com_squareup_retrofit_retrofit_1_9_0.xml new file mode 100644 index 0000000..175e01f --- /dev/null +++ b/.idea/libraries/Maven__com_squareup_retrofit_retrofit_1_9_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml b/.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml new file mode 100644 index 0000000..9764314 --- /dev/null +++ b/.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml b/.idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml new file mode 100644 index 0000000..0c58157 --- /dev/null +++ b/.idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_zdjizhi_galaxy_1_0_1.xml b/.idea/libraries/Maven__com_zdjizhi_galaxy_1_0_1.xml new file mode 100644 index 0000000..d811259 --- /dev/null +++ b/.idea/libraries/Maven__com_zdjizhi_galaxy_1_0_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml b/.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml new file mode 100644 index 0000000..e6493ac --- /dev/null +++ b/.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_codec_commons_codec_1_8.xml b/.idea/libraries/Maven__commons_codec_commons_codec_1_8.xml new file mode 100644 index 0000000..e479388 --- /dev/null +++ b/.idea/libraries/Maven__commons_codec_commons_codec_1_8.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml b/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml new file mode 100644 index 0000000..4555dca --- /dev/null +++ b/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_io_commons_io_2_4.xml b/.idea/libraries/Maven__commons_io_commons_io_2_4.xml new file mode 100644 index 0000000..34bca9e --- /dev/null +++ b/.idea/libraries/Maven__commons_io_commons_io_2_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml b/.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml new file mode 100644 index 0000000..d79c563 --- /dev/null +++ b/.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_logging_commons_logging_1_2.xml b/.idea/libraries/Maven__commons_logging_commons_logging_1_2.xml new file mode 100644 index 0000000..18024a6 --- /dev/null +++ b/.idea/libraries/Maven__commons_logging_commons_logging_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__io_netty_netty_3_10_6_Final.xml b/.idea/libraries/Maven__io_netty_netty_3_10_6_Final.xml new file mode 100644 index 0000000..0ec5e08 --- /dev/null +++ b/.idea/libraries/Maven__io_netty_netty_3_10_6_Final.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_servlet_javax_servlet_api_3_1_0.xml b/.idea/libraries/Maven__javax_servlet_javax_servlet_api_3_1_0.xml new file mode 100644 index 0000000..0e89b29 --- /dev/null +++ b/.idea/libraries/Maven__javax_servlet_javax_servlet_api_3_1_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__joda_time_joda_time_2_9_9.xml b/.idea/libraries/Maven__joda_time_joda_time_2_9_9.xml new file mode 100644 index 0000000..b0f17b5 --- /dev/null +++ b/.idea/libraries/Maven__joda_time_joda_time_2_9_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__junit_junit_4_12.xml b/.idea/libraries/Maven__junit_junit_4_12.xml new file mode 100644 index 0000000..04f012d --- /dev/null +++ b/.idea/libraries/Maven__junit_junit_4_12.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__log4j_log4j_1_2_14.xml b/.idea/libraries/Maven__log4j_log4j_1_2_14.xml new file mode 100644 index 0000000..0fd9042 --- /dev/null +++ b/.idea/libraries/Maven__log4j_log4j_1_2_14.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml b/.idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml new file mode 100644 index 0000000..29b0bd3 --- /dev/null +++ b/.idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_5_0_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml b/.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml new file mode 100644 index 0000000..896fdf4 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_avro_avro_ipc_1_7_4.xml b/.idea/libraries/Maven__org_apache_avro_avro_ipc_1_7_4.xml new file mode 100644 index 0000000..fc3162a --- /dev/null +++ b/.idea/libraries/Maven__org_apache_avro_avro_ipc_1_7_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_avro_avro_tools_1_7_4.xml b/.idea/libraries/Maven__org_apache_avro_avro_tools_1_7_4.xml new file mode 100644 index 0000000..6f77ae5 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_avro_avro_tools_1_7_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml b/.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml new file mode 100644 index 0000000..d29f8b7 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_4.xml b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_4.xml new file mode 100644 index 0000000..edfe108 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_flume_flume_ng_auth_1_9_0.xml b/.idea/libraries/Maven__org_apache_flume_flume_ng_auth_1_9_0.xml new file mode 100644 index 0000000..e107c0a --- /dev/null +++ b/.idea/libraries/Maven__org_apache_flume_flume_ng_auth_1_9_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_flume_flume_ng_configfilters_flume_ng_config_filter_api_1_9_0.xml b/.idea/libraries/Maven__org_apache_flume_flume_ng_configfilters_flume_ng_config_filter_api_1_9_0.xml new file mode 100644 index 0000000..327975d --- /dev/null +++ b/.idea/libraries/Maven__org_apache_flume_flume_ng_configfilters_flume_ng_config_filter_api_1_9_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_flume_flume_ng_configuration_1_9_0.xml b/.idea/libraries/Maven__org_apache_flume_flume_ng_configuration_1_9_0.xml new file mode 100644 index 0000000..f7cf402 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_flume_flume_ng_configuration_1_9_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_flume_flume_ng_core_1_9_0.xml b/.idea/libraries/Maven__org_apache_flume_flume_ng_core_1_9_0.xml new file mode 100644 index 0000000..eddd03e --- /dev/null +++ b/.idea/libraries/Maven__org_apache_flume_flume_ng_core_1_9_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_flume_flume_ng_sdk_1_9_0.xml b/.idea/libraries/Maven__org_apache_flume_flume_ng_sdk_1_9_0.xml new file mode 100644 index 0000000..983f6c2 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_flume_flume_ng_sdk_1_9_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_httpcomponents_httpasyncclient_4_1_3.xml b/.idea/libraries/Maven__org_apache_httpcomponents_httpasyncclient_4_1_3.xml new file mode 100644 index 0000000..a738abf --- /dev/null +++ b/.idea/libraries/Maven__org_apache_httpcomponents_httpasyncclient_4_1_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_5_2.xml b/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_5_2.xml new file mode 100644 index 0000000..778f58c --- /dev/null +++ b/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_5_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_4_6.xml b/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_4_6.xml new file mode 100644 index 0000000..6a9e309 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_4_6.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_nio_4_4_6.xml b/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_nio_4_4_6.xml new file mode 100644 index 0000000..1631446 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_nio_4_4_6.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_httpcomponents_httpmime_4_3_1.xml b/.idea/libraries/Maven__org_apache_httpcomponents_httpmime_4_3_1.xml new file mode 100644 index 0000000..3f73ed0 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_httpcomponents_httpmime_4_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_kafka_kafka_2_11_1_0_0.xml b/.idea/libraries/Maven__org_apache_kafka_kafka_2_11_1_0_0.xml new file mode 100644 index 0000000..4d908b3 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_kafka_kafka_2_11_1_0_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_kafka_kafka_clients_1_0_0.xml b/.idea/libraries/Maven__org_apache_kafka_kafka_clients_1_0_0.xml new file mode 100644 index 0000000..ebff5f4 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_kafka_kafka_clients_1_0_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_mina_mina_core_2_0_4.xml b/.idea/libraries/Maven__org_apache_mina_mina_core_2_0_4.xml new file mode 100644 index 0000000..d5818ef --- /dev/null +++ b/.idea/libraries/Maven__org_apache_mina_mina_core_2_0_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_thrift_libthrift_0_9_3.xml b/.idea/libraries/Maven__org_apache_thrift_libthrift_0_9_3.xml new file mode 100644 index 0000000..53a5217 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_thrift_libthrift_0_9_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_velocity_velocity_1_7.xml b/.idea/libraries/Maven__org_apache_velocity_velocity_1_7.xml new file mode 100644 index 0000000..04826b4 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_velocity_velocity_1_7.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_10.xml b/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_10.xml new file mode 100644 index 0000000..18b8c7b --- /dev/null +++ b/.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_10.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_8_8.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_8_8.xml new file mode 100644 index 0000000..bdd522f --- /dev/null +++ b/.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_8_8.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_8_8.xml b/.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_8_8.xml new file mode 100644 index 0000000..1735bb2 --- /dev/null +++ b/.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_8_8.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_http_9_4_6_v20170531.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_http_9_4_6_v20170531.xml new file mode 100644 index 0000000..a96826b --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_http_9_4_6_v20170531.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_io_9_4_6_v20170531.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_io_9_4_6_v20170531.xml new file mode 100644 index 0000000..49c861c --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_io_9_4_6_v20170531.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_jmx_9_4_6_v20170531.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_jmx_9_4_6_v20170531.xml new file mode 100644 index 0000000..b067a90 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_jmx_9_4_6_v20170531.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_security_9_4_6_v20170531.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_security_9_4_6_v20170531.xml new file mode 100644 index 0000000..200c371 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_security_9_4_6_v20170531.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_server_9_4_6_v20170531.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_server_9_4_6_v20170531.xml new file mode 100644 index 0000000..bec9a67 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_server_9_4_6_v20170531.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_servlet_9_4_6_v20170531.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_servlet_9_4_6_v20170531.xml new file mode 100644 index 0000000..d8cf76a --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_servlet_9_4_6_v20170531.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_util_9_4_6_v20170531.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_util_9_4_6_v20170531.xml new file mode 100644 index 0000000..9c8e9a2 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_util_9_4_6_v20170531.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml b/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml new file mode 100644 index 0000000..1d7ef14 --- /dev/null +++ b/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_influxdb_influxdb_java_2_1.xml b/.idea/libraries/Maven__org_influxdb_influxdb_java_2_1.xml new file mode 100644 index 0000000..7fb6ddf --- /dev/null +++ b/.idea/libraries/Maven__org_influxdb_influxdb_java_2_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_lz4_lz4_java_1_4.xml b/.idea/libraries/Maven__org_lz4_lz4_java_1_4.xml new file mode 100644 index 0000000..99c386b --- /dev/null +++ b/.idea/libraries/Maven__org_lz4_lz4_java_1_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mockito_mockito_all_1_10_19.xml b/.idea/libraries/Maven__org_mockito_mockito_all_1_10_19.xml new file mode 100644 index 0000000..48ab8cd --- /dev/null +++ b/.idea/libraries/Maven__org_mockito_mockito_all_1_10_19.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml b/.idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml new file mode 100644 index 0000000..045b100 --- /dev/null +++ b/.idea/libraries/Maven__org_mortbay_jetty_jetty_6_1_26.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml b/.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml new file mode 100644 index 0000000..bbb8024 --- /dev/null +++ b/.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_11.xml b/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_11.xml new file mode 100644 index 0000000..db77adc --- /dev/null +++ b/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_11.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_25.xml b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_25.xml new file mode 100644 index 0000000..ee0efb9 --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_25.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_tukaani_xz_1_0.xml b/.idea/libraries/Maven__org_tukaani_xz_1_0.xml new file mode 100644 index 0000000..0f72b33 --- /dev/null +++ b/.idea/libraries/Maven__org_tukaani_xz_1_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_4.xml b/.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_4.xml new file mode 100644 index 0000000..99ca185 --- /dev/null +++ b/.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__tech_allegro_schema_json2avro_converter_0_2_5.xml b/.idea/libraries/Maven__tech_allegro_schema_json2avro_converter_0_2_5.xml new file mode 100644 index 0000000..dc976f4 --- /dev/null +++ b/.idea/libraries/Maven__tech_allegro_schema_json2avro_converter_0_2_5.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/CommonUtils/GetDataDictionaryCodeByTopicUtils.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/CommonUtils/GetDataDictionaryCodeByTopicUtils.java index 7211567..99e4a95 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/CommonUtils/GetDataDictionaryCodeByTopicUtils.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/CommonUtils/GetDataDictionaryCodeByTopicUtils.java @@ -22,6 +22,7 @@ public class GetDataDictionaryCodeByTopicUtils { case "NTC-COLLECT-HTTP-LOG": case "NTC-COLLECT-SSL-LOG": case "NTC-COLLECT-DNS-LOG": + case "NTC-COLLECT-HTTP-HDR-LOG": case "NTC-COLLECT-FILE-LOG"://此时没有数据对象,数据对象类型选择1-log日志 return 1; /** @@ -69,6 +70,7 @@ public class GetDataDictionaryCodeByTopicUtils { case "NTC-COLLECT-HTTP-LOG": case "NTC-COLLECT-SSL-LOG": case "NTC-COLLECT-DNS-LOG": + case "NTC-COLLECT-HTTP-HDR-LOG": case "NTC-COLLECT-FTP-DOC-LOG": case "NTC-COLLECT-HTTP-DOC-LOG": case "NTC-COLLECT-HTTP-AV-LOG"://20200810新增，等同于NTC-COLLECT-HTTP-DOC-LOG @@ -109,6 +111,8 @@ public class GetDataDictionaryCodeByTopicUtils { return 4; case "NTC-COLLECT-DNS-LOG": return 5; + case "NTC-COLLECT-HTTP-HDR-LOG": + return 6; /** * 文件消息-file */ diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/CommonUtils/GetFilePathByTopicUtils.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/CommonUtils/GetFilePathByTopicUtils.java index 57690d9..79e54cb 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/CommonUtils/GetFilePathByTopicUtils.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/CommonUtils/GetFilePathByTopicUtils.java @@ -25,6 +25,8 @@ public class GetFilePathByTopicUtils { return "-"; case "NTC-COLLECT-DNS-LOG": return "-"; + case "NTC-COLLECT-HTTP-HDR-LOG": + return "-"; /** * 文件消息 */ diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/HttpAsyncUtils/HttpClientUtil.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/HttpAsyncUtils/HttpClientUtil.java index 02a1d17..a9a4ee4 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/HttpAsyncUtils/HttpClientUtil.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/HttpAsyncUtils/HttpClientUtil.java @@ -55,19 +55,16 @@ import java.util.concurrent.Executors; * 执行请求发送 */ public class HttpClientUtil { - private static HashMap schemaHashMap = new HashMap();//用于存放Schma + private static HashMap schemaHashMap = new HashMap();//用于存放Schema private static Logger logger = Logger.getLogger(HttpClientUtil.class); protected static ExecutorService pool = Executors.newFixedThreadPool(RealtimeCountConfig.HTTP_ASYNC_PARALLELISM);//线程池 private final static BlockingQueue retryQueue = new ArrayBlockingQueue(100000); - private static String utf8Charset = "utf-8"; - - private static final MediaType JSON - = MediaType.parse("application/json; charset=utf-8"); - + //private static String utf8Charset = "utf-8"; + //private static final MediaType JSON = MediaType.parse("application/json; charset=utf-8"); public static BlockingQueue getRetryQueue() { return retryQueue; @@ -79,7 +76,6 @@ public class HttpClientUtil { * * @param baseUrl 请求地址 * @param callback 回调方法,格式是FutureCallback - * @return 返回结果, 请求失败时返回null * @apiNote http接口处用 @RequestParam接收参数 */ public static void httpAsyncPostFileToZx(String baseUrl, byte[] fileIs, AsyncHttpClientPostFileCallback callback) throws Exception { @@ -124,7 +120,6 @@ public class HttpClientUtil { * * @param baseUrl 请求地址 * @param callback 回调方法,格式是FutureCallback - * @return 返回结果, 请求失败时返回null * @apiNote http接口处用 @RequestParam接收参数 */ public static void httpAsyncPostFileToZx(String baseUrl, byte[] fileIs, AsyncPostMailFilesCallback callback) throws Exception { @@ -170,7 +165,6 @@ public class HttpClientUtil { * * @param baseUrl 请求地址 * @param callback 回调方法,格式是FutureCallback - * @return 返回结果, 请求失败时返回null * @apiNote http接口处用 @RequestParam接收参数 */ public static void httpAsyncGetFile(String baseUrl, FutureCallback callback) throws Exception { @@ -258,7 +252,6 @@ public class HttpClientUtil { * @param baseUrl 请求地址 * @param urlParams 请求参数,格式是List * @param callback 回调方法,格式是FutureCallback - * @return 返回结果, 请求失败时返回null * @apiNote http接口处用 @RequestParam接收参数 */ public static void httpAsyncGet(String baseUrl, List urlParams, FutureCallback callback) throws Exception { @@ -440,12 +433,11 @@ public class HttpClientUtil { * 生产AVRO数据入ZX(单条)--数据不包含schema * 静态,适用于异步与多线程的版本 * - * @param singleProduceUrl - * @param topic - * @param dataJson - * @param userAgent - * @param msgSessionCookie - * @return + * @param singleProduceUrl singleProduceUrl + * @param topic topic + * @param dataJson dataJson + * @param userAgent userAgent + * @param msgSessionCookie msgSessionCookie */ public static void singleAsyncProduceMessage(String singleProduceUrl, String topic, String dataJson, String userAgent, @@ -738,7 +730,7 @@ public class HttpClientUtil { * 获取X-Tag用于设置请求头 * * @param dataJson - * @return + * @return xTag */ public static String getXTAG(String dataJson, String topic) { if ("monitor-msg".equals(topic) || "INFLUX-SAPP-BPS-STAT-LOG".equals(topic)) { diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/YbHttpAvroSinkFile.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/YbHttpAvroSinkFile.java index 8af28d2..ac32085 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/YbHttpAvroSinkFile.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/YbHttpAvroSinkFile.java @@ -214,6 +214,7 @@ public class YbHttpAvroSinkFile extends AbstractSink implements Configurable { case "NTC-COLLECT-FILE-LOG": case "NTC-COLLECT-HTTP-DOC-LOG": case "NTC-COLLECT-HTTP-AV-LOG": + case "NTC-COLLECT-HTTP-HDR-LOG": case "NTC-COLLECT-FTP-DOC-LOG": case "NTC-COLLECT-MAIL-LOG": case "NTC-COLLECT-TELNET-LOG": @@ -403,6 +404,7 @@ public class YbHttpAvroSinkFile extends AbstractSink implements Configurable { || topicName.equals("NTC-COLLECT-FTP-DOC-LOG") || topicName.equals("NTC-COLLECT-MAIL-LOG") || topicName.equals("NTC-COLLECT-TELNET-LOG") + || topicName.equals(("NTC-COLLECT-HTTP-HDR-LOG")) || topicName.equals("INFLUX-SAPP-BPS-STAT-LOG")) //监控消息，几分钟1条 return true; diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/avroUtils/avroRecord/GetAvroRecordByTopicUtils.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/avroUtils/avroRecord/GetAvroRecordByTopicUtils.java index bb07c72..ddf5ab4 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/avroUtils/avroRecord/GetAvroRecordByTopicUtils.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/avroUtils/avroRecord/GetAvroRecordByTopicUtils.java @@ -2,6 +2,7 @@ package cn.ac.iie.cusflume.sink.avroUtils.avroRecord; import cn.ac.iie.cusflume.sink.CommonUtils.DecodeUtils; import cn.ac.iie.cusflume.sink.bean.dataBean.NTC_COLLECT_DNS_LOG; +import cn.ac.iie.cusflume.sink.bean.dataBean.NTC_COLLECT_HTTP_HDR_LOG; import cn.ac.iie.cusflume.sink.bean.dataBean.NTC_COLLECT_HTTP_LOG; import cn.ac.iie.cusflume.sink.bean.dataBean.NTC_COLLECT_SSL_LOG; import cn.ac.iie.cusflume.sink.bean.fileBean.FILE_TAG_BEAN; @@ -30,12 +31,12 @@ public class GetAvroRecordByTopicUtils { switch (topic) { case "NTC-CONN-RECORD-LOG": return getConnRecordAvroRecord(recordAvro, dataJson, schemaStr); -// case "NTC-COLLECT-HTTP-LOG": -// return getCollHttpAvroRecord(recordAvro, dataJson); case "NTC-COLLECT-SSL-LOG": return getCollSslAvroRecord(recordAvro, dataJson, schemaStr); case "NTC-COLLECT-DNS-LOG": return getCollDnsAvroRecord(recordAvro, dataJson, schemaStr); + case "NTC-COLLECT-HTTP-HDR-LOG": + return getCollHttpHdrRecord(recordAvro, dataJson, schemaStr); case "NTC-COLLECT-HTTP-DOC-LOG": return getCollectHttpDocPostAvroRecord(recordAvro, dataJson, schemaStr); case "NTC-COLLECT-HTTP-AV-LOG": @@ -102,6 +103,12 @@ public class GetAvroRecordByTopicUtils { return recordAvro; } + private static GenericRecord getCollHttpHdrRecord(GenericRecord recordAvro, String dataJson, Schema schemaStr){ + NTC_COLLECT_HTTP_HDR_LOG ntcCollectHttpHdrLog = JSONObject.parseObject(dataJson, NTC_COLLECT_HTTP_HDR_LOG.class);//用于json字段压成字符串 + recordAvro = converter.convertToGenericDataRecord(JSONObject.toJSONString(ntcCollectHttpHdrLog).getBytes(), schemaStr); + return recordAvro; + } + private static GenericRecord getCollHttpAvroRecord(GenericRecord recordAvro, String dataJson) { NTC_COLLECT_HTTP_LOG ntcCollectHttpLog = JSONObject.parseObject(dataJson, NTC_COLLECT_HTTP_LOG.class); diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/avroUtils/avroSchema/GetAvroSchemaByTopicUtils.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/avroUtils/avroSchema/GetAvroSchemaByTopicUtils.java index 4aa2ef1..f1a5173 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/avroUtils/avroSchema/GetAvroSchemaByTopicUtils.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/avroUtils/avroSchema/GetAvroSchemaByTopicUtils.java @@ -15,6 +15,8 @@ public class GetAvroSchemaByTopicUtils { return RealtimeCountConfig.SCHEMA_NTC_COLLECT_SSL_LOG; case "NTC-COLLECT-DNS-LOG": return RealtimeCountConfig.SCHEMA_NTC_COLLECT_DNS_LOG; + case "NTC-COLLECT-HTTP-HDR-LOG": + return RealtimeCountConfig.SCHEMA_NTC_COLLECT_HTTP_HDR_LOG; case "NTC-COLLECT-MAIL-LOG": return RealtimeCountConfig.SCHEMA_NTC_COLLECT_MAIL_LOG; case "NTC-COLLECT-HTTP-DOC-LOG": diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/CommonLog_Msg_File.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/CommonLog_Msg_File.java deleted file mode 100644 index 1cac5d8..0000000 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/CommonLog_Msg_File.java +++ /dev/null @@ -1,254 +0,0 @@ -//package cn.ac.iie.cusflume.sink.bean; -// -//public class CommonLog_Msg_File { -// private int cfg_id; -// private int found_time; -// private int recv_time; -// private String trans_proto; -// private int addr_type; -// private String d_ip; -// private String s_ip; -// private int d_port; -// private int s_port; -// private int entrance_id; -// private int stream_dir; -// private String cap_ip; -// private String addr_list; -// private String server_locate; -// private String client_locate; -// private String s_asn; -// private String d_asn; -// private String user_region; -// private int service; -// private String scene_file; -// private String scene_file_id; -// -// public int getCfg_id() { -// return cfg_id; -// } -// -// public void setCfg_id(int cfg_id) { -// this.cfg_id = cfg_id; -// } -// -// public int getFound_time() { -// return found_time; -// } -// -// public void setFound_time(int found_time) { -// this.found_time = found_time; -// } -// -// public int getRecv_time() { -// return recv_time; -// } -// -// public void setRecv_time(int recv_time) { -// this.recv_time = recv_time; -// } -// -// public String getTrans_proto() { -// return trans_proto; -// } -// -// public void setTrans_proto(String trans_proto) { -// this.trans_proto = trans_proto; -// } -// -// public int getAddr_type() { -// return addr_type; -// } -// -// public void setAddr_type(int addr_type) { -// this.addr_type = addr_type; -// } -// -// public String getD_ip() { -// return d_ip; -// } -// -// public void setD_ip(String d_ip) { -// this.d_ip = d_ip; -// } -// -// public String getS_ip() { -// return s_ip; -// } -// -// public void setS_ip(String s_ip) { -// this.s_ip = s_ip; -// } -// -// public int getD_port() { -// return d_port; -// } -// -// public void setD_port(int d_port) { -// this.d_port = d_port; -// } -// -// public int getS_port() { -// return s_port; -// } -// -// public void setS_port(int s_port) { -// this.s_port = s_port; -// } -// -// public int getEntrance_id() { -// return entrance_id; -// } -// -// public void setEntrance_id(int entrance_id) { -// this.entrance_id = entrance_id; -// } -// -// public int getStream_dir() { -// return stream_dir; -// } -// -// public void setStream_dir(int stream_dir) { -// this.stream_dir = stream_dir; -// } -// -// public String getCap_ip() { -// return cap_ip; -// } -// -// public void setCap_ip(String cap_ip) { -// this.cap_ip = cap_ip; -// } -// -// public String getAddr_list() { -// return addr_list; -// } -// -// public void setAddr_list(String addr_list) { -// this.addr_list = addr_list; -// } -// -// public String getServer_locate() { -// return server_locate; -// } -// -// public void setServer_locate(String server_locate) { -// this.server_locate = server_locate; -// } -// -// public String getClient_locate() { -// return client_locate; -// } -// -// public void setClient_locate(String client_locate) { -// this.client_locate = client_locate; -// } -// -// public String getS_asn() { -// return s_asn; -// } -// -// public void setS_asn(String s_asn) { -// this.s_asn = s_asn; -// } -// -// public String getD_asn() { -// return d_asn; -// } -// -// public void setD_asn(String d_asn) { -// this.d_asn = d_asn; -// } -// -// public String getUser_region() { -// return user_region; -// } -// -// public void setUser_region(String user_region) { -// this.user_region = user_region; -// } -// -// public int getService() { -// return service; -// } -// -// public void setService(int service) { -// this.service = service; -// } -// -// public String getScene_file() { -// return scene_file; -// } -// -// public void setScene_file(String scene_file) { -// this.scene_file = scene_file; -// } -// -// public String getScene_file_id() { -// return scene_file_id; -// } -// -// public void setScene_file_id(String scene_file_id) { -// this.scene_file_id = scene_file_id; -// } -// -// @Override -// public String toString() { -// return cfg_id + "#" + -// found_time + "#" + -// recv_time + "#" + -// trans_proto + "#" + -// addr_type + "#" + -// d_ip + "#" + -// s_ip + "#" + -// d_port + "#" + -// s_port + "#" + -// entrance_id + "#" + -//// encap_type + "#" + -//// direction + "#" + -//// inner_smac + "#" + -//// inner_dmac + "#" + -// stream_dir + "#" + -// cap_ip + "#" + -// addr_list + "#" + -// server_locate + "#" + -// client_locate + "#" + -// s_asn + "#" + -// d_asn + "#" + -//// s_subscribe_id + "#" + -//// d_subscribe_id + "#" + -// user_region + "#" + -// service + "#" + -// scene_file + "#" + -// scene_file_id + "#"; -// } -// -// public String field() { -// return cfg_id + "#" + -// found_time + "#" + -// recv_time + "#" + -// trans_proto + "#" + -// addr_type + "#" + -// d_ip + "#" + -// s_ip + "#" + -// d_port + "#" + -// s_port + "#" + -// entrance_id + "#" + -//// encap_type + "#" + -//// direction + "#" + -//// inner_smac + "#" + -//// inner_dmac + "#" + -// stream_dir + "#" + -// cap_ip + "#" + -// addr_list + "#" + -// server_locate + "#" + -// client_locate + "#" + -// s_asn + "#" + -// d_asn + "#" + -//// s_subscribe_id + "#" + -//// d_subscribe_id + "#" + -// user_region + "#" + -// service + "#" + -// scene_file + "#" + -// scene_file_id + "#"; -// } -//} diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/CommonLog.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/CommonLog.java index 2280815..a00097f 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/CommonLog.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/CommonLog.java @@ -34,6 +34,8 @@ public class CommonLog { private String scene_file_id; private String s_mac; private String d_mac; + private int ip_ttl; // 20210401新增 + private int ip_id; // 20210401新增 public String getDevice_id() { return device_id; @@ -263,6 +265,14 @@ public class CommonLog { public void setD_mac(String d_mac) { this.d_mac = d_mac; } + public int getIp_ttl() { return ip_ttl; } + + public void setIp_ttl(int ip_ttl) { this.ip_ttl = ip_ttl; } + + public int getIp_id() { return ip_id; } + + public void setIp_id(int ip_id) { this.ip_id = ip_id; } + @Override public String toString() { return @@ -294,7 +304,9 @@ public class CommonLog { scene_file + "#" + scene_file_id + "#" + s_mac + "#" + - d_mac + "#" ; + d_mac + "#" + + ip_ttl + "#" + + ip_id + "#"; } public String field() { @@ -327,6 +339,9 @@ public class CommonLog { scene_file + "#" + scene_file_id + "#" + s_mac + "#" + - d_mac + "#" ; + d_mac + "#" + + ip_ttl + "#" + + ip_id + "#"; } -} + +} \ No newline at end of file diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/NTC_COLLECT_HTTP_HDR_LOG.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/NTC_COLLECT_HTTP_HDR_LOG.java new file mode 100644 index 0000000..3c7d986 --- /dev/null +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/NTC_COLLECT_HTTP_HDR_LOG.java @@ -0,0 +1,210 @@ +package cn.ac.iie.cusflume.sink.bean.dataBean; + +public class NTC_COLLECT_HTTP_HDR_LOG extends CommonLog{ + private String url; + private String referer; + private String user_agent; + private int http_seq; + private String req_line; + private String res_line; + private String cookie; + private String content_type; + private String content_len; + private String host; + private String transfer_encoding; + private String transfer_length; + private String cont_encoding; + private String cont_disposition; + private String charset; + private Long isn; + private int proxy_flag; + private String x_forward_for; + private String http_c2s_head; + private String http_s2c_head; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getReferer() { + return referer; + } + + public void setReferer(String referer) { + this.referer = referer; + } + + public String getUser_agent() { + return user_agent; + } + + public void setUser_agent(String user_agent) { + this.user_agent = user_agent; + } + + public int getHttp_seq() { + return http_seq; + } + + public void setHttp_seq(int http_seq) { + this.http_seq = http_seq; + } + + public String getReq_line() { + return req_line; + } + + public void setReq_line(String req_line) { + this.req_line = req_line; + } + + public String getRes_line() { + return res_line; + } + + public void setRes_line(String res_line) { + this.res_line = res_line; + } + + public String getCookie() { + return cookie; + } + + public void setCookie(String cookie) { + this.cookie = cookie; + } + + public String getContent_type() { + return content_type; + } + + public void setContent_type(String content_type) { + this.content_type = content_type; + } + + public String getContent_len() { + return content_len; + } + + public void setContent_len(String content_len) { + this.content_len = content_len; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public String getTransfer_encoding() { + return transfer_encoding; + } + + public void setTransfer_encoding(String transfer_encoding) { + this.transfer_encoding = transfer_encoding; + } + + public String getTransfer_length() { + return transfer_length; + } + + public void setTransfer_length(String transfer_length) { + this.transfer_length = transfer_length; + } + + public String getCont_encoding() { + return cont_encoding; + } + + public void setCont_encoding(String cont_encoding) { + this.cont_encoding = cont_encoding; + } + + public String getCont_disposition() { + return cont_disposition; + } + + public void setCont_disposition(String cont_disposition) { + this.cont_disposition = cont_disposition; + } + + public String getCharset() { + return charset; + } + + public void setCharset(String charset) { + this.charset = charset; + } + + public Long getIsn() { + return isn; + } + + public void setIsn(Long isn) { + this.isn = isn; + } + + public int getProxy_flag() { + return proxy_flag; + } + + public void setProxy_flag(int proxy_flag) { + this.proxy_flag = proxy_flag; + } + + public String getX_forward_for() { + return x_forward_for; + } + + public void setX_forward_for(String x_forward_for) { + this.x_forward_for = x_forward_for; + } + + public String getHttp_c2s_head() { + return http_c2s_head; + } + + public void setHttp_c2s_head(String http_c2s_head) { + this.http_c2s_head = http_c2s_head; + } + + public String getHttp_s2c_head() { + return http_s2c_head; + } + + public void setHttp_s2c_head(String http_s2c_head) { + this.http_s2c_head = http_s2c_head; + } + + @Override + public String toString() { + return "NTC_COLLECT_HTTP_HDR_LOG{" + + "url='" + url + '\'' + + ", referer='" + referer + '\'' + + ", user_agent='" + user_agent + '\'' + + ", http_seq=" + http_seq + + ", req_line='" + req_line + '\'' + + ", res_line='" + res_line + '\'' + + ", cookie='" + cookie + '\'' + + ", content_type='" + content_type + '\'' + + ", content_len='" + content_len + '\'' + + ", host='" + host + '\'' + + ", transfer_encoding='" + transfer_encoding + '\'' + + ", transfer_length='" + transfer_length + '\'' + + ", cont_encoding='" + cont_encoding + '\'' + + ", cont_disposition='" + cont_disposition + '\'' + + ", charset='" + charset + '\'' + + ", isn=" + isn + + ", proxy_flag=" + proxy_flag + + ", x_forward_for='" + x_forward_for + '\'' + + ", http_c2s_head='" + http_c2s_head + '\'' + + ", http_s2c_head='" + http_s2c_head + '\'' + + '}'; + } +} diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/NTC_COLLECT_MAIL_LOG.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/NTC_COLLECT_MAIL_LOG.java deleted file mode 100644 index 9e116a5..0000000 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/NTC_COLLECT_MAIL_LOG.java +++ /dev/null @@ -1,70 +0,0 @@ -//package cn.ac.iie.cusflume.sink.bean.dataBean; -// -//public class NTC_COLLECT_MAIL_LOG extends CommonLog { -// -// private String mail_proto; -// private String mail_from; -// private String mail_to; -// private String subject; -// private String eml_key; -// private String eml_file; -// -// public String getMail_proto() { -// return mail_proto; -// } -// -// public void setMail_proto(String mail_proto) { -// this.mail_proto = mail_proto; -// } -// -// public String getMail_from() { -// return mail_from; -// } -// -// public void setMail_from(String mail_from) { -// this.mail_from = mail_from; -// } -// -// public String getMail_to() { -// return mail_to; -// } -// -// public void setMail_to(String mail_to) { -// this.mail_to = mail_to; -// } -// -// public String getSubject() { -// return subject; -// } -// -// public void setSubject(String subject) { -// this.subject = subject; -// } -// -// public String getEml_key() { -// return eml_key; -// } -// -// public void setEml_key(String eml_key) { -// this.eml_key = eml_key; -// } -// -// public String getEml_file() { -// return eml_file; -// } -// -// public void setEml_file(String eml_file) { -// this.eml_file = eml_file; -// } -// -// @Override -// public String toString() { -// return field() + -// mail_proto + "\t" + -// mail_from + "\t" + -// mail_to + "\t" + -// subject + "\t" + -// eml_key + "\t" + -// eml_file + "\n"; -// } -//} diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/NTC_COLLECT_SSL_LOG.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/NTC_COLLECT_SSL_LOG.java index f7f28ed..f1e3006 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/NTC_COLLECT_SSL_LOG.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/dataBean/NTC_COLLECT_SSL_LOG.java @@ -35,6 +35,9 @@ public class NTC_COLLECT_SSL_LOG extends CommonLog { private String cert_finger; private String pub_key; + //202104新增 + private String cert_list_path; + public String getC2s_pkt_num() { return c2s_pkt_num; } @@ -146,43 +149,11 @@ public class NTC_COLLECT_SSL_LOG extends CommonLog { public String getPub_key() { return pub_key; } public void setPub_key(String pub_key) { this.pub_key = pub_key; } - // @Override -// public String toString() { -// return field() + -//// version + "\t" + -// c2s_pkt_num + "\t" + -// s2c_pkt_num + "\t" + -// c2s_byte_num + "\t" + -// s2c_byte_num + "\t" + -// sni + "\t" + -// create_time + "\t" + -// last_time + "\t" + -// server_ciphersuits + "\t" + -// client_ciphersuits + "\t" + -// -// server_cert + "\t" + -// client_cert + "\n"; -// } + public String getCert_list_path() { return cert_list_path; } + + public void setCert_list_path(String cert_list_path) { this.cert_list_path = cert_list_path; } - // @Override -// public String toString() { -// return "NTC_COLLECT_SSL_LOG{" + -// "c2s_pkt_num='" + c2s_pkt_num + '\'' + -// ", s2c_pkt_num='" + s2c_pkt_num + '\'' + -// ", c2s_byte_num='" + c2s_byte_num + '\'' + -// ", s2c_byte_num='" + s2c_byte_num + '\'' + -// ", sni='" + sni + '\'' + -// ", create_time=" + create_time + -// ", last_time=" + last_time + -// ", server_ciphersuits='" + server_ciphersuits + '\'' + -// ", client_ciphersuits='" + client_ciphersuits + '\'' + -// ", server_cert='" + server_cert + '\'' + -// ", client_cert='" + client_cert + '\'' + -// ", d_tag=" + d_tag + -// ", x_tag='" + x_tag + '\'' + -// '}'; -// } @Override public String toString() { return "NTC_COLLECT_SSL_LOG{" + @@ -201,6 +172,7 @@ public class NTC_COLLECT_SSL_LOG extends CommonLog { ", x_tag='" + x_tag + '\'' + ", cert_finger='" + cert_finger + '\'' + ", pub_key='" + pub_key + '\'' + + ", cert_list_path='" + cert_list_path + '\'' + '}'; } } diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/CommonLog_File.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/CommonLog_File.java index 7a390e8..b3c2fc8 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/CommonLog_File.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/CommonLog_File.java @@ -34,7 +34,8 @@ public class CommonLog_File { private String scene_file_id; private String s_mac; private String d_mac; - + private int ip_ttl; // 20210401新增 + private int ip_id; // 20210401新增 public String getDevice_id() { return device_id; @@ -224,6 +225,14 @@ public class CommonLog_File { public void setD_mac(String d_mac) { this.d_mac = d_mac; } + public int getIp_ttl() { return ip_ttl; } + + public void setIp_ttl(int ip_ttl) { this.ip_ttl = ip_ttl; } + + public int getIp_id() { return ip_id; } + + public void setIp_id(int ip_id) { this.ip_id = ip_id; } + @Override public String toString() { return @@ -256,7 +265,9 @@ public class CommonLog_File { scene_file + "#" + scene_file_id + "#" + s_mac + "#" + - d_mac + "#" ; + d_mac + "#" + + ip_ttl + "#" + + ip_id + "#"; } public String field() { @@ -290,6 +301,8 @@ public class CommonLog_File { scene_file + "#" + scene_file_id + "#" + s_mac + "#" + - d_mac + "#" ; + d_mac + "#" + + ip_ttl + "#" + + ip_id + "#"; } } diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_FTP_DOC_LOG.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_FTP_DOC_LOG.java index 89b310f..30ef8c9 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_FTP_DOC_LOG.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_FTP_DOC_LOG.java @@ -12,6 +12,7 @@ public class NTC_COLLECT_FTP_DOC_LOG extends CommonLog_File { private String ftp_content; // private String ftp_content_id;//旧-20200506废弃 private String content_path;//新-20200506新修改 + private String ftp_username; // v4.1新增 //20200211新增 private LOG_D_TAG_BEAN d_tag;//json版本 @@ -58,17 +59,9 @@ public class NTC_COLLECT_FTP_DOC_LOG extends CommonLog_File { this.x_tag = x_tag; } -// @Override -// public String toString() { -// return "NTC_COLLECT_FTP_DOC_LOG{" + -// "ftp_url='" + ftp_url + '\'' + -// ", ftp_content='" + ftp_content + '\'' + -// ", ftp_content_id='" + content_path + '\'' + -// ", d_tag=" + d_tag + -// ", x_tag='" + x_tag + '\'' + -// '}'; -// } + public String getFtp_username() { return ftp_username; } + public void setFtp_username(String ftp_username) { this.ftp_username = ftp_username; } @Override public String toString() { @@ -76,6 +69,7 @@ public class NTC_COLLECT_FTP_DOC_LOG extends CommonLog_File { "ftp_url='" + ftp_url + '\'' + ", ftp_content='" + ftp_content + '\'' + ", content_path='" + content_path + '\'' + + ", ftp_username='" + ftp_username + '\'' + ", d_tag=" + d_tag + ", x_tag='" + x_tag + '\'' + '}'; diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_HTTP_DOC_LOG.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_HTTP_DOC_LOG.java index fb2dd40..536f5e9 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_HTTP_DOC_LOG.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_HTTP_DOC_LOG.java @@ -19,6 +19,7 @@ public class NTC_COLLECT_HTTP_DOC_LOG extends CommonLog_File { private String req_body_file_path;//新增-20191216 private String res_body_file_path;//新增-20191216 + private String x_forward_for; // 20210401新增 //20191209新增 // private String d_tag;//json字符串版本 private LOG_D_TAG_BEAN d_tag;//json版本-20191216 @@ -26,7 +27,7 @@ public class NTC_COLLECT_HTTP_DOC_LOG extends CommonLog_File { //20191209新增 - private String set_cookie;//目前未使用-20191216 + private String set_cookie;//目前未使用-2019126 private String req_hdr_file;//原本;//目前未使用-20191216 private String req_hdr_file_id;//目前未使用-20191216 @@ -135,6 +136,10 @@ public class NTC_COLLECT_HTTP_DOC_LOG extends CommonLog_File { this.res_body_file_path = res_body_file_path; } + public String getX_forward_for() { return x_forward_for; } + + public void setX_forward_for(String x_forward_for) { this.x_forward_for = x_forward_for; } + public LOG_D_TAG_BEAN getD_tag() { return d_tag; } @@ -151,13 +156,9 @@ public class NTC_COLLECT_HTTP_DOC_LOG extends CommonLog_File { this.x_tag = x_tag; } - public String getSet_cookie() { - return set_cookie; - } + public String getSet_cookie() { return set_cookie; } - public void setSet_cookie(String set_cookie) { - this.set_cookie = set_cookie; - } + public void setSet_cookie(String set_cookie) { this.set_cookie = set_cookie; } public String getReq_hdr_file() { return req_hdr_file; @@ -240,6 +241,7 @@ public class NTC_COLLECT_HTTP_DOC_LOG extends CommonLog_File { ", res_body_file_path='" + res_body_file_path + '\'' + ", d_tag=" + d_tag + ", x_tag='" + x_tag + '\'' + + ", x_forward_for='" + x_forward_for + '\'' + '}'; } } diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_MAIL_LOG.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_MAIL_LOG.java index 56f969b..7d66af9 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_MAIL_LOG.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_MAIL_LOG.java @@ -40,7 +40,7 @@ public class NTC_COLLECT_MAIL_LOG extends CommonLog_File { private String resent_to; private String resent_date; private String message_id; - + private String x_originating_ip;// 20210401新增 public String getMail_proto() { return mail_proto; @@ -241,42 +241,11 @@ public class NTC_COLLECT_MAIL_LOG extends CommonLog_File { public void setMessage_id(String message_id) { this.message_id = message_id; } - // @Override -// public String toString() { -// return "NTC_COLLECT_MAIL_LOG{" + -// "mail_proto='" + mail_proto + '\'' + -// ", mail_from='" + mail_from + '\'' + -// ", mail_to='" + mail_to + '\'' + -// ", subject='" + subject + '\'' + -// ", eml_file='" + eml_file + '\'' + -// ", eml_file_id='" + eml_file_id + '\'' + -// ", attachments=" + Arrays.toString(attachments) + -// ", attachments_id='" + attachments_id + '\'' + -// ", subject_charset='" + subject_charset + '\'' + -// '}'; -// } + public String getX_originating_ip() { return x_originating_ip; } + + public void setX_originating_ip(String x_originating_ip) { this.x_originating_ip = x_originating_ip; } - // @Override -// public String toString() { -// return "NTC_COLLECT_MAIL_LOG{" + -// "mail_proto='" + mail_proto + '\'' + -// ", mail_from='" + mail_from + '\'' + -// ", mail_to='" + mail_to + '\'' + -// ", mail_cc='" + mail_cc + '\'' + -// ", mail_date='" + mail_date + '\'' + -// ", subject='" + subject + '\'' + -// ", eml_file_path='" + eml_file_path + '\'' + -// ", attachments_path='" + attachments_path + '\'' + -// ", subject_charset='" + subject_charset + '\'' + -// ", d_tag=" + d_tag + -// ", x_tag='" + x_tag + '\'' + -// ", eml_file='" + eml_file + '\'' + -// ", eml_file_id='" + eml_file_id + '\'' + -// ", attachments=" + Arrays.toString(attachments) + -// ", attachments_id='" + attachments_id + '\'' + -// '}'; -// } @Override public String toString() { return "NTC_COLLECT_MAIL_LOG{" + @@ -305,6 +274,7 @@ public class NTC_COLLECT_MAIL_LOG extends CommonLog_File { ", resent_to='" + resent_to + '\'' + ", resent_date='" + resent_date + '\'' + ", message_id='" + message_id + '\'' + + ", x_originating_ip='" + x_originating_ip + '\'' + '}'; } } diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_TELNET_LOG.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_TELNET_LOG.java index f6a898b..03ca3c1 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_TELNET_LOG.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/bean/fileBean/NTC_COLLECT_TELNET_LOG.java @@ -58,18 +58,6 @@ public class NTC_COLLECT_TELNET_LOG extends CommonLog_File { this.x_tag = x_tag; } -// @Override -// public String toString() { -// return "NTC_COLLECT_TELNET_LOG{" + -// "username='" + username + '\'' + -// ", password='" + password + '\'' + -// ", telnet_content_path='" + content_path + '\'' + -// ", d_tag=" + d_tag + -// ", x_tag='" + x_tag + '\'' + -// '}'; -// } - - @Override public String toString() { return "NTC_COLLECT_TELNET_LOG{" + diff --git a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/daoUtils/RealtimeCountConfig.java b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/daoUtils/RealtimeCountConfig.java index 257a7b7..b398dfd 100644 --- a/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/daoUtils/RealtimeCountConfig.java +++ b/yb_http_avro_sink_file/src/main/java/cn/ac/iie/cusflume/sink/daoUtils/RealtimeCountConfig.java @@ -87,6 +87,7 @@ public class RealtimeCountConfig implements Serializable { public static final String SCHEMA_NTC_CONN_RECORD_LOG = RealtimeCountConfigurations.getStringProperty(1, "schema.ntc-conn-record-log"); public static final String SCHEMA_NTC_COLLECT_SSL_LOG = RealtimeCountConfigurations.getStringProperty(1, "schema.ntc-collect-ssl-log"); public static final String SCHEMA_NTC_COLLECT_DNS_LOG = RealtimeCountConfigurations.getStringProperty(1, "schema.ntc-collect-dns-log"); + public static final String SCHEMA_NTC_COLLECT_HTTP_HDR_LOG = RealtimeCountConfigurations.getStringProperty(1,"schema.ntc-collect-http-hdr-log"); //文件消息 public static final String SCHEMA_NTC_COLLECT_MAIL_LOG = RealtimeCountConfigurations.getStringProperty(1, "schema.ntc-collect-mail-log"); public static final String SCHEMA_NTC_COLLECT_HTTP_DOC_LOG = RealtimeCountConfigurations.getStringProperty(1, "schema.ntc-collect-http-doc-log"); diff --git a/yb_http_avro_sink_file/src/main/resources/flume_config.properties b/yb_http_avro_sink_file/src/main/resources/flume_config.properties index 4548a52..5160203 100644 --- a/yb_http_avro_sink_file/src/main/resources/flume_config.properties +++ b/yb_http_avro_sink_file/src/main/resources/flume_config.properties @@ -8,7 +8,7 @@ http.async.socketTimeout=60000 http.async.connectTimeout=30000 #异步Http客户端-连接池最大连接数 -http.async.poolSize=4 +http.async.poolSize=4cdcd #异步Http客户端-每个主机的并发最多只有1500 http.async.maxPerRoute=4 @@ -19,15 +19,16 @@ http.async.connectionRequestTimeout=30000 #Schema配置信息 #纯消息-和前端定义相关联 -schema.ntc-conn-record-log={"type":"record","name":"connrecord","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"app_label","type":"string","doc":"label:应用识别标签","default":""},{"name":"c2s_pkt_num","type":"string","doc":"label:c2s包数","default":""},{"name":"s2c_pkt_num","type":"string","doc":"label:s2c包数","default":""},{"name":"c2s_byte_num","type":"string","doc":"label:c2s字节数","default":""},{"name":"s2c_byte_num","type":"string","doc":"label:s2c字节数","default":""},{"name":"s2c_data_head","type":"string","doc":"服务端到客户端负载前100字节","default":""},{"name":"c2s_data_head","type":"string","doc":"客户端到服务端负载前100字节","default":""}]} -schema.ntc-collect-ssl-log={"type":"record","name":"collssl","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"c2s_pkt_num","type":"string","doc":"label:c2s包数","default":""},{"name":"s2c_pkt_num","type":"string","doc":"label:s2c包数","default":""},{"name":"c2s_byte_num","type":"string","doc":"label:c2s字节数","default":""},{"name":"s2c_byte_num","type":"string","doc":"label:s2c字节数","default":""},{"name":"sni","type":"string","doc":"label:SSLservernameindication","default":""},{"name":"create_time","type":"long","doc":"label:会话创建时间unixtimestamp","default":0},{"name":"last_time","type":"long","doc":"label:会话最后修改时间unixtimestamp","default":0},{"name":"server_ciphersuits","type":"string","doc":"label:服务端ciphersuits","default":""},{"name":"client_ciphersuits","type":"string","doc":"label:客户端ciphersuits","default":""},{"name":"server_cert","type":"string","doc":"label:服务端证书信息","default":""},{"name":"client_cert","type":"string","doc":"label:客户端证书信息","default":""},{"name":"cert_finger","type":"string","doc":"label:证书指纹","default":""},{"name":"pub_key","type":"string","doc":"label:公钥","default":""}]} -schema.ntc-collect-dns-log={"type":"record","name":"colldns","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"dns_sub","type":"long","doc":"label:DNS种类","default":0},{"name":"hdr","type":"string","doc":"label:hdr信息","default":""},{"name":"flags","type":"string","doc":"label:标志","default":""},{"name":"question","type":"string","doc":"label:问题计数","default":""},{"name":"rr","type":"string","doc":"label:资源记录","default":""}]} +schema.ntc-conn-record-log={"type":"record","name":"connrecord","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"app_label","type":"string","doc":"label:应用识别标签","default":""},{"name":"c2s_pkt_num","type":"string","doc":"label:c2s包数","default":""},{"name":"s2c_pkt_num","type":"string","doc":"label:s2c包数","default":""},{"name":"c2s_byte_num","type":"string","doc":"label:c2s字节数","default":""},{"name":"s2c_byte_num","type":"string","doc":"label:s2c字节数","default":""},{"name":"s2c_data_head","type":"string","doc":"label:服务端到客户端负载前100字节","default":""},{"name":"c2s_data_head","type":"string","doc":"label:客户端到服务端负载前100字节","default":""}]} +schema.ntc-collect-ssl-log={"type":"record","name":"collssl","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"c2s_pkt_num","type":"string","doc":"label:c2s包数","default":""},{"name":"s2c_pkt_num","type":"string","doc":"label:s2c包数","default":""},{"name":"c2s_byte_num","type":"string","doc":"label:c2s字节数","default":""},{"name":"s2c_byte_num","type":"string","doc":"label:s2c字节数","default":""},{"name":"sni","type":"string","doc":"label:SSLservernameindication","default":""},{"name":"create_time","type":"long","doc":"label:会话创建时间unixtimestamp","default":0},{"name":"last_time","type":"long","doc":"label:会话最后修改时间unixtimestamp","default":0},{"name":"server_ciphersuits","type":"string","doc":"label:服务端ciphersuits","default":""},{"name":"client_ciphersuits","type":"string","doc":"label:客户端ciphersuits","default":""},{"name":"server_cert","type":"string","doc":"label:服务端证书信息","default":""},{"name":"client_cert","type":"string","doc":"label:客户端证书信息","default":""},{"name":"cert_finger","type":"string","doc":"label:证书指纹","default":""},{"name":"pub_key","type":"string","doc":"label:公钥","default":""},{"name":"cert_list_path","type":"string","doc":"label:证书文件存储地址","default":""}]} +schema.ntc-collect-dns-log={"type":"record","name":"colldns","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"dns_sub","type":"long","doc":"label:DNS种类","default":0},{"name":"hdr","type":"string","doc":"label:hdr信息","default":""},{"name":"flags","type":"string","doc":"label:标志","default":""},{"name":"question","type":"string","doc":"label:问题计数","default":""},{"name":"rr","type":"string","doc":"label:资源记录","default":""}]} +schema.ntc-collect-http-hdr-log={"type":"record","name":"collhttphdr","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"url","type":"string","doc":"label:HTTP url地址","default":""},{"name":"referer","type":"string","doc":"label:HTTP访问referer","default":""},{"name":"user_agent","type":"string","doc":"label:HTTP访问user agent","default":""},{"name":"http_seq","type":"int","doc":"label:HTTP请求会话序列号","default":0},{"name":"req_line","type":"string","doc":"label:请求行","default":""},{"name":"res_line","type":"string","doc":"label:应答行","default":""},{"name":"cookie","type":"string","doc":"label:cookiez值","default":""},{"name":"content_type","type":"string","doc":"label:content_type 值","default":""},{"name":"content_len","type":"string","doc":"label:content_len 值","default":""},{"name":"host","type":"string","doc":"label:host值","default":""},{"name":"transfer_encoding","type":"string","doc":"label:transfer_encoding","default":""},{"name":"transfer_length","type":"string","doc":"label:transfer_length","default":""},{"name":"cont_encoding","type":"string","doc":"label:cont_encoding","default":""},{"name":"cont_disposition","type":"string","doc":"label:cont_disposition","default":""},{"name":"charset","type":"string","doc":"label:编码方式","default":""},{"name":"isn","type":"long","doc":"label:","default":0},{"name":"proxy_flag","type":"int","doc":"label:","default":0},{"name":"x_forward_for","type":"string","doc":"label:","default":""},{"name":"http_c2s_head","type":"string","doc":"label:客户端到服务端负载前100字节","default":""},{"name":"http_s2c_head","type":"string","doc":"label:服务端到客户端负载前100字节","default":""}]} #文件消息-和前端定义相关联 -schema.ntc-collect-mail-log={"type":"record","name":"collmail","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"mail_proto","type":"string","doc":"label:邮件协议","default":""},{"name":"mail_from","type":"string","doc":"label:邮件发件人","default":""},{"name":"mail_to","type":"string","doc":"label:邮件收件人","default":""},{"name":"mail_cc","type":"string","doc":"label:邮件抄送人","default":""},{"name":"mail_date","type":"string","doc":"label:邮件日期","default":""},{"name":"subject","type":"string","doc":"label:邮件主题","default":""},{"name":"eml_file_path","type":"string","doc":"label:邮件EML文件转储地址","default":""},{"name":"attachments_path","type":"string","doc":"label:附件转储地址列表","default":""},{"name":"bcc","type":"string","doc":"label:暗抄送","default":""},{"name":"received","type":"string","doc":"label:系统添加部分","default":""},{"name":"return_path","type":"string","doc":"label:电子邮件地址","default":""},{"name":"reply_to","type":"string","doc":"label:回复","default":""},{"name":"mail_from_cmd","type":"string","doc":"label:mail_from_cc","default":""},{"name":"rcpt_to_cmd","type":"string","doc":"label:mail_to_cc","default":""},{"name":"resent_from","type":"string","doc":"label:转发邮件发件人","default":""},{"name":"resent_to","type":"string","doc":"label:转发邮件收件人","default":""},{"name":"resent_date","type":"string","doc":"label:转发时间","default":""},{"name":"message_id","type":"string","doc":"label:","default":""}]} -schema.ntc-collect-http-doc-log={"type":"record","name":"collhttpdoc","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"url","type":"string","doc":"label:HTTP_url地址","default":""},{"name":"referer","type":"string","doc":"label:HTTP访问referer","default":""},{"name":"user_agent","type":"string","doc":"label:HTTP访问user_agent","default":""},{"name":"http_seq","type":"int","doc":"label:HTTP请求会话序列号","default":0},{"name":"req_line","type":"string","doc":"label:请求行","default":""},{"name":"res_line","type":"string","doc":"label:应答行","default":""},{"name":"cookie","type":"string","doc":"label:cookie值","default":""},{"name":"content_type","type":"string","doc":"label:content_type值","default":""},{"name":"content_len","type":"long","doc":"label:content_len值","default":0},{"name":"doc_type","type":"string","doc":"label:文件类型","default":""},{"name":"req_body_file_path","type":"string","doc":"label:请求体转储文件地址","default":""},{"name":"res_body_file_path","type":"string","doc":"label:应答体转储文件地址","default":""}]} -schema.ntc-collect-http-av-log={"type":"record","name":"collhttpav","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"url","type":"string","doc":"label:HTTP_url地址","default":""},{"name":"referer","type":"string","doc":"label:HTTP访问referer","default":""},{"name":"user_agent","type":"string","doc":"label:HTTP访问user_agent","default":""},{"name":"http_seq","type":"int","doc":"label:HTTP请求会话序列号","default":0},{"name":"req_line","type":"string","doc":"label:请求行","default":""},{"name":"res_line","type":"string","doc":"label:应答行","default":""},{"name":"cookie","type":"string","doc":"label:cookie值","default":""},{"name":"content_type","type":"string","doc":"label:content_type值","default":""},{"name":"content_len","type":"long","doc":"label:content_len值","default":0},{"name":"doc_type","type":"string","doc":"label:文件类型","default":""},{"name":"req_body_file_path","type":"string","doc":"label:请求体转储文件地址","default":""},{"name":"res_body_file_path","type":"string","doc":"label:应答体转储文件地址","default":""}]} -schema.ntc-collect-ftp-doc-log={"type":"record","name":"collftpdoc","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ftp_url","type":"string","doc":"label:ftp_url地址","default":""},{"name":"content_path","type":"string","doc":"label:FTP文件转储文件地址","default":""}]} -schema.ntc-collect-telnet-log={"type":"record","name":"colltelnet","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"username","type":"string","doc":"label:用户名","default":""},{"name":"password","type":"string","doc":"label:密码","default":""},{"name":"content_path","type":"string","doc":"label:telnet内容转储文件地址","default":""}]} +schema.ntc-collect-mail-log={"type":"record","name":"collmail","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"mail_proto","type":"string","doc":"label:邮件协议","default":""},{"name":"mail_from","type":"string","doc":"label:邮件发件人","default":""},{"name":"mail_to","type":"string","doc":"label:邮件收件人","default":""},{"name":"mail_cc","type":"string","doc":"label:邮件抄送人","default":""},{"name":"mail_date","type":"string","doc":"label:邮件日期","default":""},{"name":"subject","type":"string","doc":"label:邮件主题","default":""},{"name":"eml_file_path","type":"string","doc":"label:邮件EML文件转储地址","default":""},{"name":"attachments_path","type":"string","doc":"label:附件转储地址列表","default":""},{"name":"bcc","type":"string","doc":"label:暗抄送","default":""},{"name":"received","type":"string","doc":"label:系统添加部分","default":""},{"name":"return_path","type":"string","doc":"label:电子邮件地址","default":""},{"name":"reply_to","type":"string","doc":"label:回复","default":""},{"name":"mail_from_cmd","type":"string","doc":"label:mail_from_cc","default":""},{"name":"rcpt_to_cmd","type":"string","doc":"label:mail_to_cc","default":""},{"name":"resent_from","type":"string","doc":"label:转发邮件发件人","default":""},{"name":"resent_to","type":"string","doc":"label:转发邮件收件人","default":""},{"name":"resent_date","type":"string","doc":"label:转发时间","default":""},{"name":"message_id","type":"string","doc":"label:","default":""},{"name":"x_originating_ip","type":"string","doc":"label:邮件经过的服务器IP","default":""}]} +schema.ntc-collect-http-doc-log={"type":"record","name":"collhttpav","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"url","type":"string","doc":"label:HTTP_url地址","default":""},{"name":"referer","type":"string","doc":"label:HTTP访问referer","default":""},{"name":"user_agent","type":"string","doc":"label:HTTP访问user_agent","default":""},{"name":"http_seq","type":"int","doc":"label:HTTP请求会话序列号","default":0},{"name":"req_line","type":"string","doc":"label:请求行","default":""},{"name":"res_line","type":"string","doc":"label:应答行","default":""},{"name":"cookie","type":"string","doc":"label:cookie值","default":""},{"name":"content_type","type":"string","doc":"label:content_type值","default":""},{"name":"content_len","type":"long","doc":"label:content_len值","default":0},{"name":"doc_type","type":"string","doc":"label:文件类型","default":""},{"name":"req_body_file_path","type":"string","doc":"label:请求体转储文件地址","default":""},{"name":"res_body_file_path","type":"string","doc":"label:应答体转储文件地址","default":""},{"name":"x_forward_for","type":"string","doc":"label:","default":""}]} +schema.ntc-collect-http-av-log={"type":"record","name":"collhttpav","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"url","type":"string","doc":"label:HTTP_url地址","default":""},{"name":"referer","type":"string","doc":"label:HTTP访问referer","default":""},{"name":"user_agent","type":"string","doc":"label:HTTP访问user_agent","default":""},{"name":"http_seq","type":"int","doc":"label:HTTP请求会话序列号","default":0},{"name":"req_line","type":"string","doc":"label:请求行","default":""},{"name":"res_line","type":"string","doc":"label:应答行","default":""},{"name":"cookie","type":"string","doc":"label:cookie值","default":""},{"name":"content_type","type":"string","doc":"label:content_type值","default":""},{"name":"content_len","type":"long","doc":"label:content_len值","default":0},{"name":"doc_type","type":"string","doc":"label:文件类型","default":""},{"name":"req_body_file_path","type":"string","doc":"label:请求体转储文件地址","default":""},{"name":"res_body_file_path","type":"string","doc":"label:应答体转储文件地址","default":""},{"name":"x_forward_for","type":"string","doc":"label:","default":""}]} +schema.ntc-collect-ftp-doc-log={"type":"record","name":"collftpdoc","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"ftp_url","type":"string","doc":"label:ftp_url地址","default":""},{"name":"content_path","type":"string","doc":"label:FTP文件转储文件地址","default":""},{"name":"ftp_username","type":"string","doc":"label:FTP登录用户名","default":""}]} +schema.ntc-collect-telnet-log={"type":"record","name":"colltelnet","fields":[{"name":"cfg_id","type":"int","doc":"label:配置ID","default":0},{"name":"found_time","type":"string","doc":"label:发现时间","default":0},{"name":"recv_time","type":"string","doc":"label:接收时间","default":0},{"name":"trans_proto","type":"string","doc":"label:传输层协议","default":""},{"name":"addr_type","type":"int","doc":"label:地址类型","default":0},{"name":"d_ip","type":"string","doc":"label:目的IP","default":""},{"name":"s_ip","type":"string","doc":"label:源IP","default":""},{"name":"d_port","type":"int","doc":"label:目的端口","default":0},{"name":"s_port","type":"int","doc":"label:源端口","default":0},{"name":"device_id","type":"string","doc":"label:设备ID","default":""},{"name":"stream_dir","type":"int","doc":"label:流类型","default":0},{"name":"cap_ip","type":"string","doc":"label:捕包机IP","default":""},{"name":"addr_list","type":"string","doc":"label:嵌套地址列表","default":""},{"name":"server_locate","type":"string","doc":"label:服务端地址定位信息","default":""},{"name":"client_locate","type":"string","doc":"label:客户端地址定位信息","default":""},{"name":"s_asn","type":"string","doc":"label:客户端ASN","default":""},{"name":"d_asn","type":"string","doc":"label:服务端ASN","default":""},{"name":"user_region","type":"string","doc":"label:用户自定义","default":""},{"name":"scene_file_id","type":"string","doc":"label:现场日志转储地址","default":""},{"name":"s_mac","type":"string","doc":"label:源MAC","default":""},{"name":"d_mac","type":"string","doc":"label:目的MAC","default":""},{"name":"ip_ttl","type":"int","doc":"label:IP报文的TTL","default":0},{"name":"ip_id","type":"int","doc":"label:IP报文的id","default":0},{"name":"username","type":"string","doc":"label:用户名","default":""},{"name":"password","type":"string","doc":"label:密码","default":""},{"name":"content_path","type":"string","doc":"label:telnet内容转储文件地址","default":""}]} #特殊-和前端定义相关联 schema.ntc-collect-file-log={"type":"record","name":"file_tag","fields":[{"type":"string","name":"tag_version","default":""},{"type":"int","name":"data_type","default":0},{"type":"int","name":"data_subtype","default":0},{"type":"int","name":"producer_id","default":0},{"type":"int","name":"data_source","default":0},{"type":{"type":"array","items":"int"},"name":"task_id","default":[]},{"type":"long","name":"file_id","default":0},{"type":"string","name":"file_path","default":""},{"type":"string","name":"file_name","default":""},{"type":"float","name":"file_size","default":0.0},{"type":"string","name":"md5","default":""},{"type":"string","name":"encrypted","default":""},{"type":"string","name":"timestamp","default":""},{"type":"long","name":"parent_id","default":0},{"type":"long","name":"flow_id","default":0}]} schema.log-tag={"type":"record","name":"log_tag","fields":[{"type":"string","name":"tag_version","default":""},{"type":"int","name":"data_type","default":0},{"type":"int","name":"data_subtype","default":0},{"type":"int","name":"producer_id","default":0},{"type":"int","name":"data_source","default":0},{"type":{"type":"array","items":"int"},"name":"task_id","default":[]},{"type":"long","name":"data_id","default":0},{"type":"long","name":"flow_id","default":0},{"type":{"type":"array","items":"long"},"name":"file_id_list","default":[]},{"type":{"type":"array","items":"string"},"name":"file_path_list","default":[]}]} diff --git a/yb_http_avro_sink_file/yb_http_avro_sink_file.iml b/yb_http_avro_sink_file/yb_http_avro_sink_file.iml new file mode 100644 index 0000000..65361cc --- /dev/null +++ b/yb_http_avro_sink_file/yb_http_avro_sink_file.iml @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file