From 50d0d7ead1ea6d9d6055767823c97dcb1102bc42 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 18 Sep 2018 12:01:56 +0800 Subject: [PATCH 1/9] =?UTF-8?q?http=E4=B8=9A=E5=8A=A1=E5=B1=82=E4=B8=8E?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E5=B1=82=E9=9B=86=E6=88=90=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin/business/pangu-http/pangu_http.cpp | 41 ++++++++++++++++------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/plugin/business/pangu-http/pangu_http.cpp b/plugin/business/pangu-http/pangu_http.cpp index 313d89f..b6e6d64 100644 --- a/plugin/business/pangu-http/pangu_http.cpp +++ b/plugin/business/pangu-http/pangu_http.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -114,7 +115,7 @@ static Maat_feather_t create_maat_feather(const char* profile, const char* secti } return target; } -void pangu_http_init(struct tfe_proxy * proxy) +int pangu_http_init(struct tfe_proxy * proxy) { const char* profile="./pangu/pangu_pxy.conf"; const char* logfile="./log/pangu_pxy.log"; @@ -158,11 +159,11 @@ void pangu_http_init(struct tfe_proxy * proxy) g_pangu_rt->tpl_451 = ctemplate::Template::GetTemplate(page_path,ctemplate::DO_NOT_STRIP); TFE_LOG_INFO(NULL, "Pangu HTTP init success."); - return; + return 0; error_out: TFE_LOG_ERROR(NULL, "Pangu HTTP init failed."); - return; + return -1; } static void _wrap_std_field_write(struct tfe_http_half * half, enum tfe_http_std_field field_id, const char * value) { @@ -344,7 +345,7 @@ static void html_free(char** page_buff) FREE(page_buff); return; } -static int is_http_request(uint64_t events) +static int is_http_request(enum tfe_http_event events) { if((events&EV_HTTP_REQ_HDR)|(events&EV_HTTP_REQ_BODY_BEGIN)|(events&EV_HTTP_REQ_BODY_END)|(events&EV_HTTP_REQ_BODY_CONT)) { @@ -365,7 +366,7 @@ enum replace_zone zone_name_to_id(const char* name) "http_resp_body", "http_resp_body"}; size_t i=0; - for(i=0;i< sizeof(std_name)/sizeof(const char*);i++) + for(i=0; isp),CHARSET_UTF8, body_frag, (int)frag_size + scan_ret=Maat_stream_scan_string(&(ctx->sp),CHARSET_UTF8, (const char*)body_frag, (int)frag_size ,result+hit_cnt, NULL, MAX_SCAN_RESULT-hit_cnt, &(ctx->mid)); if(scan_ret>0) { @@ -853,6 +854,7 @@ void pangu_on_http_begin(const struct tfe_stream * stream, { tfe_http_session_detach(session); } + return; } @@ -871,8 +873,8 @@ void pangu_on_http_end(const struct tfe_stream * stream, return; } -void pangu_on_http_data(const struct tfe_stream * stream, const struct tfe_http_session * session, - uint64_t events, const char* body_frag, size_t frag_size, unsigned int thread_id, void ** pme) +void pangu_on_http_data(const struct tfe_stream * stream, const struct tfe_http_session * session, + enum tfe_http_event events, const unsigned char * body_frag, size_t frag_size, unsigned int thread_id, void ** pme) { struct pangu_http_ctx* ctx=*(struct pangu_http_ctx**)pme; enum pangu_action hit_action=PG_ACTION_NONE; @@ -909,4 +911,17 @@ Re_Enter: return; } +struct tfe_plugin pangu_http_spec={ + .symbol=NULL, + .type = TFE_PLUGIN_TYPE_BUSINESS, + .on_init = pangu_http_init, + .on_deinit = NULL, + .on_open = NULL, + .on_data = NULL, + .on_close = NULL, + .on_session_begin=pangu_on_http_begin, + .on_session_data=pangu_on_http_data, + .on_session_end=pangu_on_http_end + }; +TFE_PLUGIN_REGISTER(pangu_http, pangu_http_spec) From 650623c562c7dabf96a2f584db8fa383934026b7 Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Tue, 18 Sep 2018 14:01:10 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=E9=9B=86=E6=88=90HTTP=E4=B8=9A=E5=8A=A1?= =?UTF-8?q?=E5=B1=82=E6=8F=92=E4=BB=B6=EF=BC=8C=E4=BF=AE=E6=AD=A3=E9=9B=86?= =?UTF-8?q?=E6=88=90=E5=90=8Erdkafka=E7=BC=BA=E5=B0=91=E7=AC=AC=E4=B8=89?= =?UTF-8?q?=E6=96=B9=E8=BF=9E=E6=8E=A5=E5=BA=93=E5=AF=BC=E8=87=B4=E7=9A=84?= =?UTF-8?q?=E7=BC=96=E8=AF=91=E9=97=AE=E9=A2=98=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- platform/CMakeLists.txt | 2 ++ plugin/business/pangu-http/CMakeLists.txt | 3 ++- vendor/CMakeLists.txt | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/platform/CMakeLists.txt b/platform/CMakeLists.txt index dd8bbc8..60d9e04 100644 --- a/platform/CMakeLists.txt +++ b/platform/CMakeLists.txt @@ -17,6 +17,8 @@ target_link_libraries(tfe pthread dl MESA_field_stat) target_link_libraries(tfe -Wl,--whole-archive http -Wl,--no-whole-archive) +target_link_libraries(tfe -Wl,--whole-archive pangu-http -Wl,--no-whole-archive) + #target_link_libraries(tfe -Wl,--whole-archive decrypt-mirroring -Wl,--no-whole-archive) install(TARGETS tfe RUNTIME DESTINATION ./) diff --git a/plugin/business/pangu-http/CMakeLists.txt b/plugin/business/pangu-http/CMakeLists.txt index fb43fd8..f064e3e 100644 --- a/plugin/business/pangu-http/CMakeLists.txt +++ b/plugin/business/pangu-http/CMakeLists.txt @@ -1,3 +1,4 @@ add_library(pangu-http pangu_logger.cpp pangu_http.cpp) target_link_libraries(pangu-http common http) -target_link_libraries(pangu-http librdkafka-static ctemplate-static cjson) \ No newline at end of file +target_link_libraries(pangu-http librdkafka-static ctemplate-static cjson) +target_link_libraries(pangu-http maatframe) diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index ddd6514..19dce43 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -177,6 +177,7 @@ file(MAKE_DIRECTORY ${INSTALL_DIR}/include) add_library(librdkafka-static STATIC IMPORTED GLOBAL) set_property(TARGET librdkafka-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/librdkafka.a) set_property(TARGET librdkafka-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) +set_property(TARGET librdkafka-static PROPERTY INTERFACE_LINK_LIBRARIES ssl crypto sasl2 m z dl pthread rt) ### ctemplate ExternalProject_Add(ctemplate PREFIX ctemplate @@ -185,6 +186,7 @@ ExternalProject_Add(ctemplate PREFIX ctemplate BUILD_COMMAND sh autogen.sh CONFIGURE_COMMAND ./configure --prefix= BUILD_IN_SOURCE 1) + ExternalProject_Get_Property(ctemplate INSTALL_DIR) file(MAKE_DIRECTORY ${INSTALL_DIR}/include) From ee4ef2d999ed79cd31b99b5f9a77c79a23e31227 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 18 Sep 2018 14:02:39 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=E8=B0=83=E6=95=B4pangu-http=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E6=96=87=E4=BB=B6=E7=9A=84=E7=9B=AE=E5=BD=95=E7=BB=93?= =?UTF-8?q?=E6=9E=84=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/pangu-http/conf/pangu_ctrl.json | 135 ++++++++++++++++++ .../business/pangu-http/conf/pangu_pxy.conf | 19 +++ .../business/pangu-http/conf/table_info.conf | 25 ++++ .../{ => conf}/template/HTTP403.html | 0 .../{ => conf}/template/HTTP404.html | 0 .../{ => conf}/template/HTTP451.html | 0 plugin/business/pangu-http/pangu_http.cpp | 60 +++++--- plugin/business/pangu-http/pangu_logger.h | 3 +- 8 files changed, 225 insertions(+), 17 deletions(-) create mode 100644 plugin/business/pangu-http/conf/pangu_ctrl.json create mode 100644 plugin/business/pangu-http/conf/pangu_pxy.conf create mode 100644 plugin/business/pangu-http/conf/table_info.conf rename plugin/business/pangu-http/{ => conf}/template/HTTP403.html (100%) rename plugin/business/pangu-http/{ => conf}/template/HTTP404.html (100%) rename plugin/business/pangu-http/{ => conf}/template/HTTP451.html (100%) diff --git a/plugin/business/pangu-http/conf/pangu_ctrl.json b/plugin/business/pangu-http/conf/pangu_ctrl.json new file mode 100644 index 0000000..fa3587a --- /dev/null +++ b/plugin/business/pangu-http/conf/pangu_ctrl.json @@ -0,0 +1,135 @@ +{ + "rules": [ + { + "compile_id": 125, + "service": 1, + "action": 48, + "do_blacklist": 1, + "do_log": 1, + "effective_range": 0, + "user_region": "code=302;url=http://news.baidu.com", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "PXY_CTRL_HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "kernel-3.10.0-327.el7.src.rpm", + "expr_type": "regex", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 128, + "service": 1, + "action": 16, + "do_blacklist": 1, + "do_log": 1, + "effective_range": 0, + "user_region": "code=302;url=http://news.baidu.com", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "PXY_CTRL_HTTP_RES_BODY", + "table_type": "string", + "table_content": { + "keywords": "hello=world", + "expr_type": "regex", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 127, + "service": 1, + "action": 80, + "do_blacklist": 1, + "do_log": 1, + "effective_range": 0, + "user_region": "zone=http_resp_body;regex=/house.qq.com/www.renren.com", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "PXY_CTRL_HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "www.qq.com", + "expr_type": "none", + "match_method": "left", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 130, + "service": 1, + "action": 48, + "do_blacklist": 1, + "do_log": 1, + "effective_range": 0, + "user_region": "code=404;content=", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "PXY_CTRL_HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "www.google.com", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 131, + "service": 1, + "action": 48, + "do_blacklist": 1, + "do_log": 1, + "effective_range": 0, + "user_region": "code=404;content=", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "PXY_CTRL_HTTP_RES_BODY", + "table_type": "string", + "table_content": { + "keywords": "动物世界", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + } + + ] +} diff --git a/plugin/business/pangu-http/conf/pangu_pxy.conf b/plugin/business/pangu-http/conf/pangu_pxy.conf new file mode 100644 index 0000000..0424123 --- /dev/null +++ b/plugin/business/pangu-http/conf/pangu_pxy.conf @@ -0,0 +1,19 @@ +[DEBUG] +LOG_LEVEL=0 +[LOG] +NIC_NAME=enp131s0f2 +ENTRANCE_ID=0 +KAFKA_BROKERLIST=192.168.10.73:9092 + +[MAAT] +# 0:json 1: redis 2: iris +MAAT_INPUT_MODE=0 +TABLE_INFO=./pangu_conf/table_info.conf +JSON_CFG_FILE=./pangu_conf/pangu_ctrl.json +STAT_FILE=./log/pangu_scan.status +FULL_CFG_DIR=./pangu_policy/full/index/ +INC_CFG_DIR=./pangu_policy/inc/index/ +MAAT_REDIS_SERVER=192.168.11.243 +MAAT_REDIS_PORT=6379 +MAAT_REDIS_DB_INDEX=4 +EFFECT_INTERVAL_S=1 diff --git a/plugin/business/pangu-http/conf/table_info.conf b/plugin/business/pangu-http/conf/table_info.conf new file mode 100644 index 0000000..043accc --- /dev/null +++ b/plugin/business/pangu-http/conf/table_info.conf @@ -0,0 +1,25 @@ +#each collumn seperate with '\t' +#id (0~65535) +#name string +#type one of ip,expr,expr_plus,digest,intval,compile or plugin +#src_charset one of GBK,BIG5,UNICODE,UTF8 +#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/' +#do_merege [yes/no] +#cross cache [number] +#quick mode [quickon/quickoff], default [quickoff] +#For ip/intval/digest/compile/group +#id name type +# +#For plugin table +#id name type valid_column +# +#For expr/expr_plus Table +#id name type src_charset dst_charset do_merge cross_cache quick_mode +0 PXY_CTRL_COMPILE compile escape -- +1 PXY_CTRL_GROUP group -- +2 PXY_CTRL_IP ip --- +3 PXY_CTRL_HTTP_URL expr UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff +4 PXY_CTRL_HTTP_REQ_HDR expr_plus UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff +5 PXY_CTRL_HTTP_REQ_BODY expr UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff +6 PXY_CTRL_HTTP_RES_HDR expr_plus UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff +7 PXY_CTRL_HTTP_RES_BODY expr UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff diff --git a/plugin/business/pangu-http/template/HTTP403.html b/plugin/business/pangu-http/conf/template/HTTP403.html similarity index 100% rename from plugin/business/pangu-http/template/HTTP403.html rename to plugin/business/pangu-http/conf/template/HTTP403.html diff --git a/plugin/business/pangu-http/template/HTTP404.html b/plugin/business/pangu-http/conf/template/HTTP404.html similarity index 100% rename from plugin/business/pangu-http/template/HTTP404.html rename to plugin/business/pangu-http/conf/template/HTTP404.html diff --git a/plugin/business/pangu-http/template/HTTP451.html b/plugin/business/pangu-http/conf/template/HTTP451.html similarity index 100% rename from plugin/business/pangu-http/template/HTTP451.html rename to plugin/business/pangu-http/conf/template/HTTP451.html diff --git a/plugin/business/pangu-http/pangu_http.cpp b/plugin/business/pangu-http/pangu_http.cpp index b6e6d64..7662844 100644 --- a/plugin/business/pangu-http/pangu_http.cpp +++ b/plugin/business/pangu-http/pangu_http.cpp @@ -61,22 +61,37 @@ struct pangu_rt int page_size; }; struct pangu_rt *g_pangu_rt; +#define MAAT_INPUT_JSON 0 +#define MAAT_INPUT_REDIS 1 +#define MAAT_INPUT_FILE 2 + static Maat_feather_t create_maat_feather(const char* profile, const char* section,int max_thread, void* logger) { Maat_feather_t target; - int maat_json_switch=0,maat_stat_on=0,maat_perf_on=0; + int input_mode=0,maat_stat_on=0,maat_perf_on=0; int ret=0,scan_detail=0,effect_interval=60; char table_info[TFE_STRING_MAX]={0},inc_cfg_dir[TFE_STRING_MAX]={0},ful_cfg_dir[TFE_STRING_MAX]={0}; + char redis_server[TFE_STRING_MAX]={0}; + int redis_port=0; + int redis_db_idx=0; char json_cfg_file[TFE_STRING_MAX]={0},maat_stat_file[TFE_STRING_MAX]={0}; const char* instance_name="pangu"; - MESA_load_profile_int_def(profile, section,"MAAT_JSON_SWITCH", &(maat_json_switch),0); + MESA_load_profile_int_def(profile, section,"MAAT_INPUT_MODE", &(input_mode),0); MESA_load_profile_int_def(profile, section,"STAT_SWITCH", &(maat_stat_on),1); MESA_load_profile_int_def(profile, section,"PERF_SWITCH", &(maat_perf_on),1); MESA_load_profile_string_def(profile,section,"TABLE_INFO",table_info, sizeof(table_info),""); + + MESA_load_profile_string_def(profile,section,"JSON_CFG_FILE",json_cfg_file, sizeof(json_cfg_file),""); + + MESA_load_profile_string_def(profile,section,"MAAT_REDIS_SERVER",redis_server, sizeof(redis_server),""); + MESA_load_profile_int_def(profile,section,"MAAT_REDIS_PORT", &(redis_port),6379); + MESA_load_profile_int_def(profile,section,"MAAT_REDIS_DB_INDEX", &(redis_db_idx),0); + + MESA_load_profile_string_def(profile,section,"INC_CFG_DIR",inc_cfg_dir, sizeof(inc_cfg_dir),""); MESA_load_profile_string_def(profile,section,"FULL_CFG_DIR",ful_cfg_dir, sizeof(ful_cfg_dir),""); - MESA_load_profile_string_def(profile,section,"JSON_CFG_FILE",json_cfg_file, sizeof(json_cfg_file),""); + MESA_load_profile_string_def(profile,section,"STAT_FILE",maat_stat_file, sizeof(maat_stat_file),""); MESA_load_profile_int_def(profile,section,"EFFECT_INTERVAL_S", &(effect_interval),60); @@ -85,14 +100,24 @@ static Maat_feather_t create_maat_feather(const char* profile, const char* secti target=Maat_feather(max_thread,table_info, logger); Maat_set_feather_opt(target,MAAT_OPT_INSTANCE_NAME,instance_name, strlen(instance_name)+1); - if(maat_json_switch==1) + switch(input_mode) { - Maat_set_feather_opt(target, MAAT_OPT_JSON_FILE_PATH, json_cfg_file, strlen(json_cfg_file)+1); - } - else - { - Maat_set_feather_opt(target, MAAT_OPT_FULL_CFG_DIR, ful_cfg_dir, strlen(ful_cfg_dir)+1); - Maat_set_feather_opt(target, MAAT_OPT_INC_CFG_DIR, inc_cfg_dir, strlen(inc_cfg_dir)+1); + case MAAT_INPUT_JSON: + Maat_set_feather_opt(target, MAAT_OPT_JSON_FILE_PATH, json_cfg_file, strlen(json_cfg_file)+1); + break; + case MAAT_INPUT_REDIS: + Maat_set_feather_opt(target, MAAT_OPT_REDIS_IP,redis_server, strlen(redis_server)+1); + Maat_set_feather_opt(target, MAAT_OPT_REDIS_PORT, &redis_port, sizeof(redis_port)); + Maat_set_feather_opt(target, MAAT_OPT_REDIS_INDEX, &redis_db_idx, sizeof(redis_db_idx)); + break; + case MAAT_INPUT_FILE: + Maat_set_feather_opt(target, MAAT_OPT_FULL_CFG_DIR, ful_cfg_dir, strlen(ful_cfg_dir)+1); + Maat_set_feather_opt(target, MAAT_OPT_INC_CFG_DIR, inc_cfg_dir, strlen(inc_cfg_dir)+1); + break; + default: + TFE_LOG_ERROR(logger, "Invalid MAAT Input Mode: %d.", input_mode); + goto error_out; + break; } if(maat_stat_on) { @@ -111,13 +136,16 @@ static Maat_feather_t create_maat_feather(const char* profile, const char* secti if(ret<0) { TFE_LOG_ERROR(logger, "%s MAAT init failed.", __FUNCTION__); - return NULL; + goto error_out; } return target; +error_out: + Maat_burn_feather(target); + return NULL; } int pangu_http_init(struct tfe_proxy * proxy) { - const char* profile="./pangu/pangu_pxy.conf"; + const char* profile="./pangu_conf/pangu_pxy.conf"; const char* logfile="./log/pangu_pxy.log"; g_pangu_rt=ALLOC(struct pangu_rt,1); MESA_load_profile_int_def(profile, "DEBUG", "LOG_LEVEL", &(g_pangu_rt->log_level),0); @@ -151,11 +179,11 @@ int pangu_http_init(struct tfe_proxy * proxy) } char page_path[256]; - MESA_load_profile_string_def(profile, "TEMPLATE", "PAGE_403", page_path,sizeof(page_path), "./template/HTTP403.html"); + MESA_load_profile_string_def(profile, "TEMPLATE", "PAGE_403", page_path,sizeof(page_path), "./pangu_conf/template/HTTP403.html"); g_pangu_rt->tpl_403 = ctemplate::Template::GetTemplate(page_path,ctemplate::DO_NOT_STRIP); - MESA_load_profile_string_def(profile, "TEMPLATE", "PAGE_404", page_path,sizeof(page_path), "./template/HTTP404.html"); + MESA_load_profile_string_def(profile, "TEMPLATE", "PAGE_404", page_path,sizeof(page_path), "./pangu_conf/template/HTTP404.html"); g_pangu_rt->tpl_404 = ctemplate::Template::GetTemplate(page_path,ctemplate::DO_NOT_STRIP); - MESA_load_profile_string_def(profile, "TEMPLATE", "PAGE_451", page_path,sizeof(page_path), "./template/HTTP451.html"); + MESA_load_profile_string_def(profile, "TEMPLATE", "PAGE_451", page_path,sizeof(page_path), "./pangu_conf/template/HTTP451.html"); g_pangu_rt->tpl_451 = ctemplate::Template::GetTemplate(page_path,ctemplate::DO_NOT_STRIP); TFE_LOG_INFO(NULL, "Pangu HTTP init success."); @@ -866,7 +894,7 @@ void pangu_on_http_end(const struct tfe_stream * stream, struct pangu_log log_msg={.stream=stream, .http=session, .result=ctx->enforce_rules, .result_num=ctx->n_enforce}; if(ctx->action!=PG_ACTION_NONE) { - pangu_log_send(g_pangu_rt->send_logger, &log_msg); + pangu_send_log(g_pangu_rt->send_logger, &log_msg); } pangu_http_ctx_free(ctx); *pme=NULL; diff --git a/plugin/business/pangu-http/pangu_logger.h b/plugin/business/pangu-http/pangu_logger.h index e9f6413..d9a3513 100644 --- a/plugin/business/pangu-http/pangu_logger.h +++ b/plugin/business/pangu-http/pangu_logger.h @@ -15,6 +15,7 @@ struct pangu_logger; struct pangu_logger* pangu_log_handle_create(const char* profile, const char* section, void* local_logger); //return 0 if SUCCESS, otherwise return -1 -int pangu_log_send(struct pangu_logger* logger, const pangu_log* log_msg); +int pangu_send_log(struct pangu_logger* handle, const struct pangu_log* log_msg); + From 8786a38fbbf93378d163fc29b5290808bf5c79c4 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 18 Sep 2018 14:11:44 +0800 Subject: [PATCH 4/9] =?UTF-8?q?pangu-http=E8=AE=BE=E7=BD=AE=E4=B8=B4?= =?UTF-8?q?=E6=97=B6=E7=9A=84tfe=E5=B7=A5=E4=BD=9C=E7=BA=BF=E7=A8=8B?= =?UTF-8?q?=E6=95=B0=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin/business/pangu-http/pangu_http.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin/business/pangu-http/pangu_http.cpp b/plugin/business/pangu-http/pangu_http.cpp index 7662844..60294d9 100644 --- a/plugin/business/pangu-http/pangu_http.cpp +++ b/plugin/business/pangu-http/pangu_http.cpp @@ -148,6 +148,7 @@ int pangu_http_init(struct tfe_proxy * proxy) const char* profile="./pangu_conf/pangu_pxy.conf"; const char* logfile="./log/pangu_pxy.log"; g_pangu_rt=ALLOC(struct pangu_rt,1); + g_pangu_rt->thread_num=16; MESA_load_profile_int_def(profile, "DEBUG", "LOG_LEVEL", &(g_pangu_rt->log_level),0); g_pangu_rt->local_logger=MESA_create_runtime_log_handle(logfile, g_pangu_rt->log_level); g_pangu_rt->send_logger=pangu_log_handle_create(profile, "LOG", g_pangu_rt->local_logger); From 386fb486f5e501028fd60ed203c3de1dba8c8485 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 18 Sep 2018 14:15:23 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dpangu-http=E5=88=9D?= =?UTF-8?q?=E5=A7=8B=E5=8C=96=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6=E8=AF=BB?= =?UTF-8?q?=E5=8F=96=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin/business/pangu-http/pangu_http.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugin/business/pangu-http/pangu_http.cpp b/plugin/business/pangu-http/pangu_http.cpp index 60294d9..b07f3a4 100644 --- a/plugin/business/pangu-http/pangu_http.cpp +++ b/plugin/business/pangu-http/pangu_http.cpp @@ -180,10 +180,13 @@ int pangu_http_init(struct tfe_proxy * proxy) } char page_path[256]; + memset(page_path, 0, sizeof(page_path)); MESA_load_profile_string_def(profile, "TEMPLATE", "PAGE_403", page_path,sizeof(page_path), "./pangu_conf/template/HTTP403.html"); - g_pangu_rt->tpl_403 = ctemplate::Template::GetTemplate(page_path,ctemplate::DO_NOT_STRIP); + g_pangu_rt->tpl_403 = ctemplate::Template::GetTemplate(page_path,ctemplate::DO_NOT_STRIP); + memset(page_path, 0, sizeof(page_path)); MESA_load_profile_string_def(profile, "TEMPLATE", "PAGE_404", page_path,sizeof(page_path), "./pangu_conf/template/HTTP404.html"); - g_pangu_rt->tpl_404 = ctemplate::Template::GetTemplate(page_path,ctemplate::DO_NOT_STRIP); + g_pangu_rt->tpl_404 = ctemplate::Template::GetTemplate(page_path,ctemplate::DO_NOT_STRIP); + memset(page_path, 0, sizeof(page_path)); MESA_load_profile_string_def(profile, "TEMPLATE", "PAGE_451", page_path,sizeof(page_path), "./pangu_conf/template/HTTP451.html"); g_pangu_rt->tpl_451 = ctemplate::Template::GetTemplate(page_path,ctemplate::DO_NOT_STRIP); From 0f67ba1790662de41ede823729732b5f1a7fb9d3 Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Tue, 18 Sep 2018 18:47:02 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=E4=BF=AE=E6=AD=A3HTTP=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E5=B1=82=E4=B8=A2=E5=A4=B1=E9=83=A8=E5=88=86=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=8C=E4=BF=AE=E6=AD=A3SSL=20Ses?= =?UTF-8?q?sion=20Reuse=E5=9B=9E=E8=B0=83=E5=87=BD=E6=95=B0=E6=AE=B5?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E7=9A=84=E9=97=AE=E9=A2=98=20*=20=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3HTTP=E8=A7=A3=E6=9E=90=E5=B1=82=E4=B8=A2=E5=A4=B1?= =?UTF-8?q?=E9=83=A8=E5=88=86=E8=AF=B7=E6=B1=82=E7=9A=84=E9=97=AE=E9=A2=98?= =?UTF-8?q?=EF=BC=8C=E5=8E=9F=E5=9B=9B=E5=B1=82=E8=BF=9E=E6=8E=A5=E5=9C=A8?= =?UTF-8?q?=E7=AC=AC=E4=B8=80=E6=AC=A1=E8=B0=83open=E5=90=8E=EF=BC=8C?= =?UTF-8?q?=E4=B8=8D=E4=BC=9A=E8=B0=83data=E5=9B=9E=E8=B0=83=E5=87=BD?= =?UTF-8?q?=E6=95=B0=EF=BC=8C=E7=8E=B0=E4=BF=AE=E6=AD=A3=EF=BC=9B=20*=20?= =?UTF-8?q?=E5=8E=9F=E5=AE=9E=E7=8E=B0=E5=8F=AA=E8=AE=BE=E7=BD=AE=E4=BA=86?= =?UTF-8?q?SSL=E7=9A=84user=20define=20ctx=EF=BC=8C=E6=B2=A1=E6=9C=89?= =?UTF-8?q?=E8=AE=BE=E7=BD=AESSL=5FCTX=E7=9A=84user=20define=20ctx?= =?UTF-8?q?=EF=BC=8C=E7=8E=B0=E4=BF=AE=E6=AD=A3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- platform/src/ssl_stream.cpp | 3 +- platform/src/tcp_stream.cpp | 77 ++++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/platform/src/ssl_stream.cpp b/platform/src/ssl_stream.cpp index 19b73c5..39a7f1a 100644 --- a/platform/src/ssl_stream.cpp +++ b/platform/src/ssl_stream.cpp @@ -1033,7 +1033,8 @@ static SSL * downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt) SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL); SSL_CTX_set_session_id_context(sslctx, (const unsigned char *) mgr->ssl_session_context, sizeof(mgr->ssl_session_context)); - + ret = SSL_CTX_set_ex_data(sslctx, SSL_EX_DATA_IDX_SSLMGR, mgr); + assert(ret == 1); if (mgr->dh) { SSL_CTX_set_tmp_dh(sslctx, mgr->dh); diff --git a/platform/src/tcp_stream.cpp b/platform/src/tcp_stream.cpp index dc6606d..0b64a2d 100644 --- a/platform/src/tcp_stream.cpp +++ b/platform/src/tcp_stream.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -337,10 +338,15 @@ static void __stream_bev_readcb(struct bufferevent * bev, void * arg) if (_stream->is_plugin_opened == 0) { - p_info_iter->on_open(&_stream->head, _stream->thread_ref->thread_id, dir, &(plug_ctx->pme)); + if (p_info_iter->on_open != NULL) + { + p_info_iter->on_open(&_stream->head, _stream->thread_ref->thread_id, dir, &(plug_ctx->pme)); + } + _stream->is_plugin_opened = 1; } - else + + if (p_info_iter->on_data != NULL) { action_tmp = p_info_iter->on_data(&_stream->head, _stream->thread_ref->thread_id, dir, contiguous_data, contigous_len, &(plug_ctx->pme)); @@ -634,6 +640,67 @@ void tfe_stream_destory(struct tfe_stream_private * stream) thread->load--; } +static struct tfe_stream_addr * __stream_addr_create_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downstream) +{ + struct tfe_stream_addr * __stream_addr = NULL; + + struct sockaddr_storage sk_src_storage{}; + struct sockaddr * sk_src_ptr = (struct sockaddr *) &sk_src_storage; + socklen_t sk_src_len = sizeof(sk_src_storage); + + struct sockaddr_storage sk_dst_storage{}; + struct sockaddr * sk_dst_ptr = (struct sockaddr *) &sk_dst_storage; + socklen_t sk_dst_len = sizeof(sk_dst_storage); + + int ret = getsockname(fd_downstream, sk_src_ptr, &sk_src_len); + if (unlikely(ret < 0)) + { + TFE_STREAM_LOG_ERROR(stream, "Failed at calling getsockaddr() for fd %d : %s", fd_downstream, strerror(errno)); + goto __errout; + } + + ret = getpeername(fd_downstream, sk_dst_ptr, &sk_dst_len); + if (unlikely(ret < 0)) + { + TFE_STREAM_LOG_ERROR(stream, "Failed at calling getpeername() for fd %d : %s", fd_downstream, strerror(errno)); + goto __errout; + } + + assert(sk_src_ptr->sa_family == sk_dst_ptr->sa_family); + if (sk_src_ptr->sa_family == AF_INET) + { + __stream_addr = (struct tfe_stream_addr *) malloc( + sizeof(struct tfe_stream_addr) + sizeof(struct tfe_stream_addr_tuple4_v4)); + + struct tfe_stream_addr_ipv4 * st_addr_v4 = __stream_addr->ipv4; + struct sockaddr_in * sk_v4_src_ptr = (struct sockaddr_in *)sk_src_ptr; + struct sockaddr_in * sk_v4_dst_ptr = (struct sockaddr_in *)sk_dst_ptr; + + __stream_addr->addrtype = TFE_ADDR_STREAM_TUPLE4_V4; + __stream_addr->addrlen = sizeof(struct tfe_stream_addr_tuple4_v4); + + st_addr_v4->saddr.s_addr = sk_v4_src_ptr->sin_addr.s_addr; + st_addr_v4->source = sk_v4_src_ptr->sin_port; + st_addr_v4->daddr.s_addr = sk_v4_dst_ptr->sin_addr.s_addr; + st_addr_v4->dest = sk_v4_dst_ptr->sin_port; + } + else if (sk_src_ptr->sa_family == AF_INET6) + { + assert(0); + } + else + { + TFE_STREAM_LOG_ERROR(stream, "Invalid sockaddr family for fd %d: sa_family is %d.", + fd_downstream, sk_src_ptr->sa_family); goto __errout; + } + + return __stream_addr; + +__errout: + if (__stream_addr != NULL) free(__stream_addr); + return NULL; +} + void tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downstream, evutil_socket_t fd_upstream) { struct tfe_stream_private * _stream = container_of(stream, struct tfe_stream_private, head); @@ -645,6 +712,12 @@ void tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downs evutil_make_socket_nonblocking(fd_downstream); evutil_make_socket_nonblocking(fd_upstream); + _stream->head.addr = __stream_addr_create_by_fds(stream, fd_downstream); + if(unlikely(_stream->head.addr == NULL)) + { + assert(0); + } + if (_stream->session_type == STREAM_PROTO_PLAIN) { _stream->conn_downstream = __conn_private_create_by_fd(_stream, fd_downstream); From 8dee0034837895a213d8aa48198ffe705081c184 Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Tue, 18 Sep 2018 18:48:21 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=A0=87=E5=87=86HTTP?= =?UTF-8?q?=E5=A4=B4=E9=83=A8=E5=AE=9A=E4=B9=89=E8=BD=AC=E5=AD=97=E7=AC=A6?= =?UTF-8?q?=E4=B8=B2=E7=9A=84=E5=B7=A5=E5=85=B7=E5=87=BD=E6=95=B0=EF=BC=8C?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3pango-http=E6=8F=92=E4=BB=B6=E5=9C=A8?= =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E6=97=B6=E6=B2=A1=E6=9C=89=E8=AE=BE?= =?UTF-8?q?=E7=BD=AEPME=E7=9A=84=E9=94=99=E8=AF=AF=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/include/tfe_http.h | 40 ++--------- common/src/tfe_http.cpp | 81 +++++++++++++++++++++++ plugin/business/pangu-http/pangu_http.cpp | 9 ++- 3 files changed, 92 insertions(+), 38 deletions(-) diff --git a/common/include/tfe_http.h b/common/include/tfe_http.h index f0ba194..14f8342 100644 --- a/common/include/tfe_http.h +++ b/common/include/tfe_http.h @@ -201,42 +201,10 @@ typedef void (http_session_data_cb)(const struct tfe_stream * stream, typedef void (http_session_end_cb)(const struct tfe_stream * stream, const struct tfe_http_session * session, unsigned int thread_id, void ** pme); -static inline struct http_field_name * http_field_name_duplicate(const struct http_field_name * orig) -{ - struct http_field_name * __duplicated = ALLOC(struct http_field_name, 1); - assert(__duplicated != NULL); - - if (orig->field_id == TFE_HTTP_UNKNOWN_FIELD) - { - __duplicated->field_id = TFE_HTTP_UNKNOWN_FIELD; - __duplicated->field_name = tfe_strdup(orig->field_name); - } - else - { - __duplicated->field_id = orig->field_id; - __duplicated->field_name = NULL; - } - - return __duplicated; -} - -static inline int http_field_name_compare(const struct http_field_name * lvalue, - const struct http_field_name * rvalue) -{ - if (lvalue->field_id != rvalue->field_id) - { - return (lvalue->field_id - rvalue->field_id); - } - - /* unknown field, compare field_name in string */ - if (lvalue->field_id == TFE_HTTP_UNKNOWN_FIELD) - { - return strcasecmp(lvalue->field_name, rvalue->field_name); - } - - /* field_id is equal, but not unknown, hit */ - return 0; -} +struct http_field_name * http_field_name_duplicate(const struct http_field_name * orig); +int http_field_name_compare(const struct http_field_name * lvalue, const struct http_field_name * rvalue); +const char * http_field_to_string(const struct http_field_name * field); +void http_field_destory(struct http_field_name *); static inline const char * tfe_http_field_read(const struct tfe_http_half * half, const struct http_field_name * name) diff --git a/common/src/tfe_http.cpp b/common/src/tfe_http.cpp index 42b0a89..5916121 100644 --- a/common/src/tfe_http.cpp +++ b/common/src/tfe_http.cpp @@ -14,6 +14,87 @@ struct tfe_http_half * tfe_http_response_create(int major_version, int resp_code return NULL; } +static const char * __str_std_header_field_map[] = +{ + [TFE_HTTP_UNKNOWN_FIELD] = NULL, + [TFE_HTTP_HOST] = "Host", + [TFE_HTTP_REFERER] = "Referer", + [TFE_HTTP_USER_AGENT] = "User-Agent", + [TFE_HTTP_COOKIE] = "Cookie", + [TFE_HTTP_PROXY_AUTHORIZATION] = "Proxy-Authorization", + [TFE_HTTP_AUTHORIZATION] = "Authorization", + [TFE_HTTP_LOCATION] = "Location", + [TFE_HTTP_SERVER] = "Server", + [TFE_HTTP_ETAG] = "Etag", + [TFE_HTTP_DATE] = "Date", + [TFE_HTTP_TRAILER] = "Trailer", + [TFE_HTTP_TRANSFER_ENCODING] = "Transfer-Encoding", + [TFE_HTTP_VIA] = "Via", + [TFE_HTTP_PRAGMA] = "Pragma", + [TFE_HTTP_CONNECTION] = "Connection", + [TFE_HTTP_CONT_ENCODING] = "Content-Encoding", + [TFE_HTTP_CONT_LANGUAGE] = "Content-Language", + [TFE_HTTP_CONT_LOCATION] = "Content-Location", + [TFE_HTTP_CONT_RANGE] = "Content-Range", + [TFE_HTTP_CONT_LENGTH] = "Content-Length", + [TFE_HTTP_CONT_TYPE] = "Content-Type", + [TFE_HTTP_CONT_DISPOSITION] = "Content-Disposition", + [TFE_HTTP_EXPIRES] = "Expires", + [TFE_HTTP_ACCEPT_ENCODING] = "Accept-Encoding" +}; + +struct http_field_name * http_field_name_duplicate(const struct http_field_name * orig) +{ + struct http_field_name * __duplicated = ALLOC(struct http_field_name, 1); + assert(__duplicated != NULL); + + if (orig->field_id == TFE_HTTP_UNKNOWN_FIELD) + { + __duplicated->field_id = TFE_HTTP_UNKNOWN_FIELD; + __duplicated->field_name = tfe_strdup(orig->field_name); + } + else + { + __duplicated->field_id = orig->field_id; + __duplicated->field_name = NULL; + } + + return __duplicated; +} + +int http_field_name_compare(const struct http_field_name * lvalue, const struct http_field_name * rvalue) +{ + if (lvalue->field_id != rvalue->field_id) + { + return (lvalue->field_id - rvalue->field_id); + } + + /* unknown field, compare field_name in string */ + if (lvalue->field_id == TFE_HTTP_UNKNOWN_FIELD) + { + return strcasecmp(lvalue->field_name, rvalue->field_name); + } + + /* field_id is equal, but not unknown, hit */ + return 0; +} + +struct http_field_name * http_field_construct_from_string(const char * str_field) +{ + return NULL; +} + +const char * http_field_to_string(const struct http_field_name * field) +{ + if (field->field_id != TFE_HTTP_UNKNOWN_FIELD) return __str_std_header_field_map[field->field_id]; + return field->field_name; +} + +void http_field_destory(struct http_field_name * field) +{ + free(field); +} + struct http_frame_session_ctx { struct http_frame_plugin_status * plugin_status; diff --git a/plugin/business/pangu-http/pangu_http.cpp b/plugin/business/pangu-http/pangu_http.cpp index b07f3a4..650520a 100644 --- a/plugin/business/pangu-http/pangu_http.cpp +++ b/plugin/business/pangu-http/pangu_http.cpp @@ -808,8 +808,11 @@ enum pangu_action http_scan(const struct tfe_http_session * session, enum tfe_ht { break; } - scan_ret=Maat_set_scan_status(g_pangu_rt->maat, &(ctx->mid), MAAT_SET_SCAN_DISTRICT, - field_name.field_name,strlen(field_name.field_name)); + + const char * str_field_name = http_field_to_string(&field_name); + scan_ret=Maat_set_scan_status(g_pangu_rt->maat, &(ctx->mid), MAAT_SET_SCAN_DISTRICT, + str_field_name,strlen(str_field_name)); + assert(scan_ret==0); scan_ret=Maat_full_scan_string(g_pangu_rt->maat, table_id, CHARSET_UTF8, field_val, strlen(field_val), @@ -886,6 +889,8 @@ void pangu_on_http_begin(const struct tfe_stream * stream, { tfe_http_session_detach(session); } + + *pme=ctx; return; } From 9ccc3d329e6f74932adeda0516ad564ef66c2965 Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Tue, 18 Sep 2018 18:50:25 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=E4=BF=AE=E6=AD=A3HTTP=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E5=B1=82=E5=A4=84=E7=90=86HTTP-Header=E7=9A=84=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E6=B5=81=E7=A8=8B=20*=20=E5=8E=9F=E6=B5=81=E7=A8=8B?= =?UTF-8?q?=E5=9C=A8=E8=A7=A3=E6=9E=90=E5=90=8C=E4=B8=80=E4=B8=AAField-Val?= =?UTF-8?q?ue=E5=AF=B9=E6=97=B6=EF=BC=8C=E6=97=A0=E6=B3=95=E6=AD=A3?= =?UTF-8?q?=E7=A1=AE=E5=A4=84=E7=90=86Header=E5=A4=9A=E6=AC=A1=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E6=8B=BC=E6=88=90=E5=AE=8C=E6=95=B4=E5=AD=97=E7=AC=A6?= =?UTF-8?q?=E4=B8=B2=E7=9A=84=E6=83=85=E5=86=B5=EF=BC=8C=E7=8E=B0=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=EF=BC=9B=20*=20=E5=8E=9F=E6=B5=81=E7=A8=8B=E5=9C=A8?= =?UTF-8?q?=E5=A4=84=E7=90=86Field-Value=E5=BA=95=E5=B1=82Buffer=E6=97=B6?= =?UTF-8?q?=E8=AE=A1=E7=AE=97=E9=95=BF=E5=BA=A6=E6=9C=89=E8=AF=AF=EF=BC=8C?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E6=B8=85=E7=A9=BAbuffer=E6=97=B6=E5=89=A9?= =?UTF-8?q?=E4=BD=99=E6=9C=80=E5=90=8E=E7=9A=84'\0'=EF=BC=8C=E7=8E=B0?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../http/include/internal/http_half.h | 7 ++++ plugin/protocol/http/src/http_entry.cpp | 12 ++---- plugin/protocol/http/src/http_half.cpp | 41 ++++++++++++------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/plugin/protocol/http/include/internal/http_half.h b/plugin/protocol/http/include/internal/http_half.h index cbe6f61..ac88810 100644 --- a/plugin/protocol/http/include/internal/http_half.h +++ b/plugin/protocol/http/include/internal/http_half.h @@ -51,8 +51,12 @@ struct http_half_private short minor; struct evbuffer * evbuf_uri; + struct evbuffer * evbuf_header_field; struct evbuffer * evbuf_header_value; + bool is_evbuf_header_field_set; + bool is_evbuf_header_value_set; + struct evbuffer * evbuf_body; enum hf_private_status body_status; @@ -64,6 +68,7 @@ struct http_half_private }; struct http_half_private * hf_private_create(tfe_http_direction ht_dir, short major, short minor); + void hf_private_destory(struct http_half_private * hf_private); /** Parse the raw tcp input for HTTP half structure @@ -83,5 +88,7 @@ int hf_private_parse(struct http_half_private * hf_private, const unsigned char void hf_private_set_callback(struct http_half_private * hf_private, hf_private_cb * cb, void * user, void (* fn_user_deleter)(void *)); +void hf_private_set_session(struct http_half_private * hf_private, struct http_session_private * hs_private); + struct http_session_private * hs_private_create(struct http_connection_private * hc_private, struct http_half_private * hf_private_req, struct http_half_private * hf_private_resp); diff --git a/plugin/protocol/http/src/http_entry.cpp b/plugin/protocol/http/src/http_entry.cpp index bbca1f3..c0ca9f4 100644 --- a/plugin/protocol/http/src/http_entry.cpp +++ b/plugin/protocol/http/src/http_entry.cpp @@ -67,6 +67,7 @@ enum tfe_stream_action __http_connection_entry_on_request(const struct tfe_strea /* HTTP Request and Session */ hf_private_request = hf_private_create(TFE_HTTP_REQUEST, 1, 0); hs_private = hs_private_create(hc_private, hf_private_request, NULL); + hf_private_set_session(hf_private_request, hs_private); /* Closure, catch stream, session and thread_id */ struct user_event_dispatch_closure * __closure = ALLOC(struct user_event_dispatch_closure, 1); @@ -139,13 +140,13 @@ enum tfe_stream_action __http_connection_entry_on_response(const struct tfe_stre int __http_connection_identify(const struct tfe_stream * stream, struct http_connection_private * ht_conn, const unsigned char * data, size_t len) { - struct http_half_private * hf_private = hf_private_create(TFE_HTTP_RESPONSE, 1, 0); + struct http_half_private * hf_private = hf_private_create(TFE_HTTP_REQUEST, 1, 0); int ret = hf_private_parse(hf_private, data, len); hf_private_destory(hf_private); return ret; } -#define HTTP_INDENTIFY_LENGTH 8 +#define HTTP_INDENTIFY_LENGTH 4 enum tfe_stream_action http_connection_entry_data(const struct tfe_stream * stream, unsigned int thread_id, enum tfe_conn_dir dir, const unsigned char * data, size_t len, void ** pme) @@ -159,12 +160,7 @@ enum tfe_stream_action http_connection_entry_data(const struct tfe_stream * stre goto __detach; /* Protocol Identification, we need 8 bytes at least to tell it is HTTP or not */ - if (len < HTTP_INDENTIFY_LENGTH) - { - static const unsigned int __defer_bytes = HTTP_INDENTIFY_LENGTH; - tfe_stream_action_set_opt(stream, ACTION_OPT_DEFER_BYTES, (void *) &__defer_bytes, sizeof(__defer_bytes)); - return ACTION_DEFER_DATA; - } + if (len < HTTP_INDENTIFY_LENGTH) goto __detach; /* Now, we want to identify this stream */ int ret = __http_connection_identify(stream, ht_conn, data, len); diff --git a/plugin/protocol/http/src/http_half.cpp b/plugin/protocol/http/src/http_half.cpp index 87a6eb3..836ff18 100644 --- a/plugin/protocol/http/src/http_half.cpp +++ b/plugin/protocol/http/src/http_half.cpp @@ -84,6 +84,8 @@ static void __http_half_header_kv_complete(struct http_half_private * hf_private /* Write a '\0' for evbuffers */ evbuffer_add(hf_private->evbuf_header_field, &__zero, sizeof(__zero)); evbuffer_add(hf_private->evbuf_header_value, &__zero, sizeof(__zero)); + sz_evbuf_field = evbuffer_get_length(hf_private->evbuf_header_field); + sz_evbuf_value = evbuffer_get_length(hf_private->evbuf_header_value); /* Convert evbuffer to const char * pair */ str_field = (const char *) evbuffer_pullup(hf_private->evbuf_header_field, sz_evbuf_field); @@ -107,6 +109,8 @@ static void __http_half_header_kv_complete(struct http_half_private * hf_private __clear_buffer: evbuffer_drain(hf_private->evbuf_header_field, sz_evbuf_field); evbuffer_drain(hf_private->evbuf_header_value, sz_evbuf_value); + hf_private->is_evbuf_header_value_set = false; + hf_private->is_evbuf_header_field_set = false; } void __hf_public_req_fill_from_private(struct http_half_private * hf_private, struct http_parser * parser) @@ -117,10 +121,10 @@ void __hf_public_req_fill_from_private(struct http_half_private * hf_private, st /* accept-encoding, host is located in header's K-V structure */ hf_req_spec->method = (enum tfe_http_std_method) parser->method; const static struct http_field_name __host_field_name = - { - .field_id = TFE_HTTP_HOST, - .field_name = NULL - }; + { + .field_id = TFE_HTTP_HOST, + .field_name = NULL + }; hf_req_spec->host = (char *) tfe_http_field_read(hf_public, &__host_field_name); @@ -131,7 +135,7 @@ void __hf_public_req_fill_from_private(struct http_half_private * hf_private, st /* TODO: URL * url is more complex. need to review RFC */ - hf_req_spec->url = NULL; + hf_req_spec->url = hf_req_spec->uri; } void __hf_public_resp_fill_from_private(struct http_half_private * hf_private, struct http_parser * parser) @@ -142,10 +146,10 @@ void __hf_public_resp_fill_from_private(struct http_half_private * hf_private, s /* Status Code */ hf_resp_spec->resp_code = parser->status_code; const static struct http_field_name __cont_encoding_field_name = - { - .field_id = TFE_HTTP_CONT_ENCODING, - .field_name = NULL - }; + { + .field_id = TFE_HTTP_CONT_ENCODING, + .field_name = NULL + }; /* Content Encoding */ hf_resp_spec->content_encoding = (char *) tfe_http_field_read(hf_public, &__cont_encoding_field_name); @@ -164,11 +168,13 @@ static int __parser_callback_on_message_begin(struct http_parser * parser) assert(hf_private->evbuf_uri == NULL && hf_private->evbuf_body == NULL); assert(hf_private->evbuf_header_field == NULL && hf_private->evbuf_header_value == NULL); - hf_private->evbuf_uri = evbuffer_new(); hf_private->evbuf_header_field = evbuffer_new(); hf_private->evbuf_header_value = evbuffer_new(); - hf_private->evbuf_body = evbuffer_new(); + hf_private->is_evbuf_header_field_set = false; + hf_private->is_evbuf_header_value_set = false; + hf_private->evbuf_uri = evbuffer_new(); + hf_private->evbuf_body = evbuffer_new(); hf_private->body_status = STATUS_INIT; hf_private->message_status = STATUS_READING; @@ -186,19 +192,19 @@ static int __parser_callback_on_uri_field(struct http_parser * parser, const cha static int __parser_callback_on_header_field(struct http_parser * parser, const char * at, size_t length) { struct http_half_private * hf_private = __PARSER_TO_HF_PRIVATE(parser); - - /* Last field-value tuple doesn't push into hf_private, flush these */ - if (evbuffer_get_length(hf_private->evbuf_header_field) != 0) + if (hf_private->is_evbuf_header_field_set && hf_private->is_evbuf_header_value_set) { __http_half_header_kv_complete(hf_private); } + hf_private->is_evbuf_header_field_set = true; return evbuffer_add(hf_private->evbuf_header_field, at, length); } static int __parser_callback_on_header_value(struct http_parser * parser, const char * at, size_t length) { struct http_half_private * hf_private = __PARSER_TO_HF_PRIVATE(parser); + hf_private->is_evbuf_header_value_set = true; return evbuffer_add(hf_private->evbuf_header_value, at, length); } @@ -429,6 +435,8 @@ struct http_half_private * hf_private_create(tfe_http_direction ht_dir, short ma hf_private->parse_settings = &__http_half_parse_setting; hf_private->parse_object->data = hf_private; + + TAILQ_INIT(&hf_private->header_list); return hf_private; } @@ -440,6 +448,11 @@ void hf_private_set_callback(struct http_half_private * hf_private, hf_private_c hf_private->event_cb_user_deleter = user_deleter; } +void hf_private_set_session(struct http_half_private * hf_private, struct http_session_private * hs_private) +{ + hf_private->session = hs_private; +} + int hf_private_parse(struct http_half_private * hf_private, const unsigned char * data, size_t len) { assert(hf_private->parse_cursor <= len); From 3d8498b648de1fb7f7646f982cf93288e67589cc Mon Sep 17 00:00:00 2001 From: Lu Qiuwen Date: Tue, 18 Sep 2018 20:45:21 +0800 Subject: [PATCH 9/9] =?UTF-8?q?=E5=A2=9E=E5=8A=A0HTTP=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E5=B1=82=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 1 + plugin/protocol/http/CMakeLists.txt | 10 +++ plugin/protocol/http/test/test_http_half.cpp | 83 ++++++++++++++++++++ vendor/CMakeLists.txt | 3 +- 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 plugin/protocol/http/test/test_http_half.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a3f6663..0dfc2f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ set(CMAKE_C_STANDARD 11) set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} -Wall) +enable_testing() add_subdirectory(vendor) add_subdirectory(common) add_subdirectory(plugin) diff --git a/plugin/protocol/http/CMakeLists.txt b/plugin/protocol/http/CMakeLists.txt index 80b268b..c16e36d 100644 --- a/plugin/protocol/http/CMakeLists.txt +++ b/plugin/protocol/http/CMakeLists.txt @@ -1,3 +1,4 @@ +### PLUGIN add_library(http src/http_entry.cpp src/http_half.cpp) target_include_directories(http PRIVATE include/internal) @@ -6,3 +7,12 @@ target_include_directories(http PUBLIC incluce/external) target_link_libraries(http common) target_link_libraries(http http-parser-static) target_link_libraries(http libevent-static) + + +### UNITTEST CASE +add_executable(test-http-half test/test_http_half.cpp) +target_include_directories(test-http-half PRIVATE include/internal) +target_link_libraries(test-http-half gtest common http) + +include(GoogleTest) +gtest_discover_tests(test-http-half) diff --git a/plugin/protocol/http/test/test_http_half.cpp b/plugin/protocol/http/test/test_http_half.cpp new file mode 100644 index 0000000..a0cb1e7 --- /dev/null +++ b/plugin/protocol/http/test/test_http_half.cpp @@ -0,0 +1,83 @@ + +#include +#include + +static const char * __identify_http_request = + "POST /gen_204 HTTP/1.1\r\n" + "Host: www.google.com\r\n" + "Connection: close\r\n" + "Content-Length: 0\r\n" + "Origin: https://www.google.com\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36\r\n" + "Content-Type: text/plain;charset=UTF-8\r\n" + "Accept: */*\r\n" + "X-Client-Data: CJG2yQEIorbJAQjEtskBCKmdygEI2J3KAQjZncoBCKijygEY+aXKAQ==\r\n" + "Referer: https://www.google.com/\r\n" + "Accept-Encoding: gzip, deflate\r\n" + "Accept-Language: zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7\r\n"; + +static const char * __identify_not_http_request = "MZMZ"; + +TEST(HttpHalfConstruct, ProtoIdentifyIsHTTP) +{ + auto * hf_request = hf_private_create(TFE_HTTP_REQUEST, 1, 1); + ASSERT_TRUE(hf_request != NULL); + + int ret = hf_private_parse(hf_request, (const unsigned char *) __identify_http_request, + strlen(__identify_http_request)); + + EXPECT_EQ(ret, 0); +} + +TEST(HttpHalfConstruct, ProtoIdentifyNotHTTP) +{ + auto * hf_request = hf_private_create(TFE_HTTP_REQUEST, 1, 1); + ASSERT_TRUE(hf_request != NULL); + + int ret = hf_private_parse(hf_request, (const unsigned char *) __identify_not_http_request, + strlen(__identify_not_http_request)); + + EXPECT_EQ(ret, -1); +} + +static const char * __get_http_request_01 = + "GET /gfwlist/gfwlist/master/gfwlist.txt HTTP/1.1\r\n" + "Host: raw.githubusercontent.com\r\n" + "Connection: close\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36\r\n" + "Accept: */*\r\n" + "Accept-Encoding: gzip, deflate\r\n" + "Accept-Language: zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7\r\n" + "If-None-Match: \"023aeae5eafc12082067c36031888adb3bafa797\"\r\n" + "\r\n"; + +void __get_http_request_01_verify_helper(struct http_half_private * hf_private) +{ + auto * hf_public = &hf_private->hf_public; + EXPECT_EQ(hf_public->major_version, 1); + EXPECT_EQ(hf_public->minor_version, 1); + + auto * hf_public_request = &hf_public->req_spec; + EXPECT_STREQ(hf_public_request->uri, "/gfwlist/gfwlist/master/gfwlist.txt"); + EXPECT_STREQ(hf_public_request->url, "/gfwlist/gfwlist/master/gfwlist.txt"); + EXPECT_STREQ(hf_public_request->host, "raw.githubusercontent.com"); + EXPECT_EQ(hf_public_request->method, TFE_HTTP_GET); +} + +TEST(HttpHalfConstruct, GetWithNoBody) +{ + auto * hf_request = hf_private_create(TFE_HTTP_REQUEST, 0, 0); + ASSERT_TRUE(hf_request != NULL); + + int ret = + hf_private_parse(hf_request, (const unsigned char *) __get_http_request_01, strlen(__get_http_request_01)); + ASSERT_EQ(ret, 0); + + __get_http_request_01_verify_helper(hf_request); +} + +int main(int argc, char ** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index 19dce43..edb79d9 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -112,9 +112,10 @@ file(MAKE_DIRECTORY ${INSTALL_DIR}/include) add_library(gtest STATIC IMPORTED GLOBAL) set_property(TARGET gtest PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libgtest.a) set_property(TARGET gtest PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) +set_property(TARGET gtest PROPERTY INTERFACE_LINK_LIBRARIES pthread) add_library(gmock STATIC IMPORTED GLOBAL) -set_property(TARGET gmock PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libgtest.a) +set_property(TARGET gmock PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libgmock.a) set_property(TARGET gmock PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include)