From 9614ea53c022b9fbf4468a22be5141fa63bb15b6 Mon Sep 17 00:00:00 2001 From: fengweihao Date: Thu, 11 Jun 2020 13:47:56 +0800 Subject: [PATCH] =?UTF-8?q?TSG-2020=20=E5=8A=9F=E8=83=BD=E7=AB=AF=E6=94=AF?= =?UTF-8?q?=E6=8C=81ip=E5=BD=92=E5=B1=9E=E5=9C=B0=E5=92=8CASN=E5=AF=B9?= =?UTF-8?q?=E8=B1=A1=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ci/travis.sh | 5 +- plugin/business/pangu-http/CMakeLists.txt | 2 +- plugin/business/pangu-http/src/pangu_http.cpp | 456 +++++++++++++++--- .../business/pangu-http/src/pangu_logger.cpp | 10 +- plugin/business/pangu-http/src/pangu_logger.h | 4 + resource/pangu/table_info.conf | 10 + vendor/CMakeLists.txt | 6 +- 7 files changed, 432 insertions(+), 61 deletions(-) diff --git a/ci/travis.sh b/ci/travis.sh index 7c5f799..77a9977 100644 --- a/ci/travis.sh +++ b/ci/travis.sh @@ -33,7 +33,10 @@ env | sort : "${COMPILER_IS_GNUCXX:=OFF}" # Install dependency from YUM -yum install -y mrzcpd framework numactl-devel zlib-devel librdkafka-devel systemd-devel +yum install -y mrzcpd numactl-devel zlib-devel librdkafka-devel systemd-devel +yum install -y libcjson-devel libmaatframe-devel libMESA_field_stat2-devel libMESA_handle_logger-devel +yum install -y libMESA_htable-devel libMESA_prof_load-devel librulescan-devel libwiredcfg-devel libWiredLB-devel sapp-devel + mkdir build || true cd build diff --git a/plugin/business/pangu-http/CMakeLists.txt b/plugin/business/pangu-http/CMakeLists.txt index c8ab3a2..7f56f24 100644 --- a/plugin/business/pangu-http/CMakeLists.txt +++ b/plugin/business/pangu-http/CMakeLists.txt @@ -1,6 +1,6 @@ add_library(pangu-http src/pangu_logger.cpp src/pangu_http.cpp src/pattern_replace.cpp src/pangu_web_cache.cpp) target_link_libraries(pangu-http PUBLIC common http tango-cache-client) -target_link_libraries(pangu-http PUBLIC librdkafka ctemplate-static cjson pcre2-static ratelimiter-static libdablooms pthread) +target_link_libraries(pangu-http PUBLIC rdkafka ctemplate-static cjson pcre2-static ratelimiter-static libdablooms pthread) target_link_libraries(pangu-http PUBLIC maatframe) add_executable(test_pattern_replace src/test_pattern_replace.cpp src/pattern_replace.cpp) diff --git a/plugin/business/pangu-http/src/pangu_http.cpp b/plugin/business/pangu-http/src/pangu_http.cpp index b88b1d0..4951ca8 100644 --- a/plugin/business/pangu-http/src/pangu_http.cpp +++ b/plugin/business/pangu-http/src/pangu_http.cpp @@ -64,6 +64,10 @@ enum scan_table PXY_CTRL_HTTP_RES_HDR, PXY_CTRL_HTTP_RES_BODY, PXY_CTRL_SUBSCRIBE_ID, + PXY_CTRL_IP_SRC_ASN, + PXY_CTRL_IP_DST_ASN, + PXY_CTRL_IP_SRC_LOCATION, + PXY_CTRL_IP_DST_LOCATION, PXY_CTRL_APP_ID, __SCAN_TABLE_MAX }; @@ -78,7 +82,9 @@ enum pangu_http_stat STAT_ACTION_PRE_REPLACE, STAT_ACTION_REPLACE, STAT_ACTION_HIJACK, + STAT_ACTION_HIJACK_SZ, STAT_ACTION_INSERT, + STAT_ACTION_INSERT_SZ, STAT_ACTION_WHITELSIT, STAT_SUSPENDING, __PG_STAT_MAX @@ -89,6 +95,10 @@ enum manipulate_profile_table POLICY_PROFLIE_TABLE_REJECT, POLICY_PROFILE_TABLE_INSERT, POLICY_PROFILE_TABLE_HIJACK, + POLICY_ASN_USER_DEFINED, + POLICY_ASN_BUILT_IN, + POLICY_LOCATION_USER_DEFINED, + POLICY_LOCATION_BUILT_IN, POLICY_PROFILE_TABLE_MAX }; @@ -105,6 +115,22 @@ struct manipulate_profile pthread_mutex_t lock; }; +struct ip_data_table +{ + int profile_id; + + int ref_cnt; + + char *asn; + char *organization; + + char *country_full; + char *province_full; + char *city_full; + + pthread_mutex_t lock; +}; + struct policy_action_param { int ref_cnt; @@ -288,7 +314,9 @@ static void pangu_http_stat_init(struct pangu_rt * pangu_runtime) spec[STAT_ACTION_PRE_REPLACE]="pre_replace"; spec[STAT_ACTION_REPLACE]="replace"; spec[STAT_ACTION_HIJACK]="hijack"; + spec[STAT_ACTION_HIJACK_SZ]="hijack_sz"; spec[STAT_ACTION_INSERT]="insert"; + spec[STAT_ACTION_INSERT_SZ]="insert_sz"; spec[STAT_ACTION_WHITELSIT]="whitelist"; spec[STAT_SUSPENDING]="suspending"; @@ -848,30 +876,175 @@ int maat_table_init(const char* table_name, return table_id; } +void ip_asn_table_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp) +{ + int addr_type; + int ret=0,profile_id=0,is_valid=0; + char start_ip[40], end_ip[40],asn[40]={0}; + char organization[TFE_PATH_MAX]; + + ret=sscanf(table_line, "%d\t%d\t%s\t%s\t%s\t%s\t%d", &profile_id, &addr_type, start_ip, end_ip, asn, organization, &is_valid); + if(ret!=7) + { + TFE_LOG_ERROR(g_pangu_rt->local_logger, "Policy table parse ip ASN failed, ret:%d, %s", ret, table_line); + return; + } + + struct ip_data_table* ip_asn=ALLOC(struct ip_data_table, 1); + memset(ip_asn, 0, sizeof(struct ip_data_table)); + ip_asn->profile_id=profile_id; + ip_asn->asn=tfe_strdup(asn); + ip_asn->organization=tfe_strdup(organization); + ip_asn->ref_cnt=1; + pthread_mutex_init(&(ip_asn->lock), NULL); + + TFE_LOG_INFO(g_pangu_rt->local_logger, "Policy table add success %d", profile_id); + + *ad = ip_asn; +} + +void ip_location_table_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp) +{ + int ret=0,profile_id=0,is_valid=0; + int geoname_id=0, addr_type=0; + double latitude, longitude, coords; + char language[40], start_ip[40], end_ip[40]; + char continent_abbr[TFE_PATH_MAX],continent_full[TFE_PATH_MAX]; + char country_abbr[TFE_PATH_MAX],province_abbr[TFE_PATH_MAX], time_zone[TFE_PATH_MAX]; + char country_full[TFE_PATH_MAX],province_full[TFE_PATH_MAX], city_full[TFE_PATH_MAX]; + + ret=sscanf(table_line, "%d\t%d\t%d\t%s\t%s\t%lf\t%lf\t%lf\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d", &profile_id,&geoname_id, + &addr_type, start_ip,end_ip,&latitude,&longitude,&coords,language, + continent_abbr,continent_full, country_abbr,country_full,province_abbr,province_full, + city_full,time_zone,&is_valid); + + if(ret!=18) + { + TFE_LOG_ERROR(g_pangu_rt->local_logger, "Policy table parse ip location failed, ret:%d, %s", ret, table_line); + return; + } + + struct ip_data_table* ip_asn=ALLOC(struct ip_data_table, 1); + memset(ip_asn, 0, sizeof(struct ip_data_table)); + ip_asn->profile_id=profile_id; + ip_asn->country_full=tfe_strdup(country_full); + ip_asn->province_full=tfe_strdup(province_full); + ip_asn->city_full=tfe_strdup(city_full); + ip_asn->ref_cnt=1; + pthread_mutex_init(&(ip_asn->lock), NULL); + + TFE_LOG_INFO(g_pangu_rt->local_logger, "Policy table add success %d", profile_id); + + *ad = ip_asn; +} + +void ip_table_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void* argp) +{ + struct ip_data_table* ip_asn=(struct ip_data_table*)(*from); + pthread_mutex_lock(&(ip_asn->lock)); + ip_asn->ref_cnt++; + pthread_mutex_unlock(&(ip_asn->lock)); + *to=ip_asn; +} + +void ip_table_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp) +{ + if(*ad==NULL) + { + return; + } + struct ip_data_table* ip_asn=(struct ip_data_table*)(*ad); + pthread_mutex_lock(&(ip_asn->lock)); + ip_asn->ref_cnt--; + if(ip_asn->ref_cnt>0) + { + pthread_mutex_unlock(&(ip_asn->lock)); + return; + } + pthread_mutex_unlock(&(ip_asn->lock)); + pthread_mutex_destroy(&(ip_asn->lock)); + + if(ip_asn->asn) FREE(&ip_asn->asn); + if(ip_asn->organization) FREE(&ip_asn->organization); + if(ip_asn->country_full) FREE(&ip_asn->country_full); + if(ip_asn->province_full) FREE(&ip_asn->province_full); + if(ip_asn->city_full) FREE(&ip_asn->city_full); + + FREE(&ip_asn); + *ad=NULL; + return; +} + +void ip_table_free(struct ip_data_table* ip_asn) +{ + ip_table_free_cb(0, (void **)&ip_asn, 0, NULL); +} + +const char* table_name_idx2str(int profile_idx) +{ + const char *table_name_map[] = {"TSG_PROFILE_RESPONSE_PAGES", + "PXY_PROFILE_INSERT_SCRIPTS", + "PXY_PROFILE_HIJACK_FILES", + "TSG_IP_ASN_USER_DEFINED", + "TSG_IP_ASN_BUILT_IN", + "TSG_IP_LOCATION_USER_DEFINED", + "TSG_IP_LOCATION_BUILT_IN"}; + + return table_name_map[profile_idx]; +} + int maat_table_ex_init(int profile_idx, - Maat_plugin_EX_new_func_t* new_func, Maat_plugin_EX_free_func_t* free_func, Maat_plugin_EX_dup_func_t* dup_func) { int table_id = 0; - const char *table_name_map[] = {"TSG_PROFILE_RESPONSE_PAGES", - "PXY_PROFILE_INSERT_SCRIPTS", - "PXY_PROFILE_HIJACK_FILES"}; + const char *table_name = table_name_idx2str(profile_idx); + Maat_plugin_EX_new_func_t *new_func[] = { + [POLICY_PROFLIE_TABLE_REJECT] = ma_profile_table_new_cb, + [POLICY_PROFILE_TABLE_INSERT] = ma_insert_profile_table_new_cb, + [POLICY_PROFILE_TABLE_HIJACK] = ma_hijack_profile_table_new_cb, + }; - table_id=g_pangu_rt->plolicy_table_id[profile_idx]=Maat_table_register(g_pangu_rt->maat, table_name_map[profile_idx]); + table_id=g_pangu_rt->plolicy_table_id[profile_idx]=Maat_table_register(g_pangu_rt->maat, table_name); if(table_id >= 0) { - table_id=Maat_plugin_EX_register(g_pangu_rt->maat, - table_id, - new_func, + table_id=Maat_plugin_EX_register(g_pangu_rt->maat, table_id, + new_func[profile_idx], free_func, dup_func, NULL, 0, NULL); return 0; } - TFE_LOG_INFO(NULL, "Pangu HTTP register table %s failed.", table_name_map[profile_idx]); + TFE_LOG_INFO(NULL, "Pangu HTTP register table %s failed.", table_name); + return -1; +} +int maat_ip_table_init(int profile_idx, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func) +{ + int table_id=0; + + Maat_plugin_EX_new_func_t *new_func[] = { + [POLICY_PROFLIE_TABLE_REJECT] = NULL, + [POLICY_PROFILE_TABLE_INSERT] = NULL, + [POLICY_PROFILE_TABLE_HIJACK] = NULL, + [POLICY_ASN_USER_DEFINED] = ip_asn_table_new_cb, + [POLICY_ASN_BUILT_IN] = ip_asn_table_new_cb, + [POLICY_LOCATION_USER_DEFINED] = ip_location_table_new_cb, + [POLICY_LOCATION_BUILT_IN] = ip_location_table_new_cb, + }; + + const char *table_name = table_name_idx2str(profile_idx); + table_id=g_pangu_rt->plolicy_table_id[profile_idx]=Maat_table_register(g_pangu_rt->maat, table_name); + if(table_id >= 0) + { + table_id=Maat_ip_plugin_EX_register(g_pangu_rt->maat, table_id, new_func[profile_idx], free_func, dup_func, + 0, NULL); + return 0; + } + TFE_LOG_INFO(NULL, "Pangu HTTP register table %s failed.", table_name); return -1; } @@ -894,6 +1067,10 @@ int pangu_policy_init(const char* profile_path, const char* static_section, cons table_name[PXY_CTRL_HTTP_RES_HDR] = "TSG_FIELD_HTTP_RES_HDR"; table_name[PXY_CTRL_HTTP_RES_BODY] = "TSG_FIELD_HTTP_RES_CONTENT"; table_name[PXY_CTRL_SUBSCRIBE_ID] = "TSG_OBJ_SUBSCRIBER_ID"; + table_name[PXY_CTRL_IP_SRC_ASN]="TSG_SECURITY_SOURCE_ASN"; + table_name[PXY_CTRL_IP_DST_ASN]="TSG_SECURITY_DESTINATION_ASN"; + table_name[PXY_CTRL_IP_SRC_LOCATION]="TSG_SECURITY_SOURCE_LOCATION"; + table_name[PXY_CTRL_IP_DST_LOCATION]="TSG_SECURITY_DESTINATION_LOCATION"; table_name[PXY_CTRL_APP_ID] = "TSG_OBJ_APP_ID"; for (int i = 0; i < __SCAN_TABLE_MAX; i++) { @@ -912,10 +1089,10 @@ int pangu_policy_init(const char* profile_path, const char* static_section, cons 0, NULL); ret = maat_table_init("PXY_PROFILE_TRUSTED_CA_CERT", - trusted_CA_update_start_cb, - trusted_CA_update_cert_cb, - trusted_CA_update_finish_cb, - g_pangu_rt); + trusted_CA_update_start_cb, + trusted_CA_update_cert_cb, + trusted_CA_update_finish_cb, + g_pangu_rt); if(ret<0) { TFE_LOG_INFO(NULL, "Pangu HTTP register table PXY_OBJ_TRUSTED_CA_CERT failed."); @@ -933,32 +1110,22 @@ int pangu_policy_init(const char* profile_path, const char* static_section, cons goto error_out; } - - ret = maat_table_ex_init(POLICY_PROFLIE_TABLE_REJECT, - ma_profile_table_new_cb, - ma_profile_table_free_cb, - ma_profile_table_dup_cb); - if(ret<0) + for(int i = 0; i <= POLICY_PROFILE_TABLE_HIJACK; i++) { - goto error_out; + ret = maat_table_ex_init(i, ma_profile_table_free_cb, ma_profile_table_dup_cb); + if(ret<0) + { + goto error_out; + } } - ret = maat_table_ex_init(POLICY_PROFILE_TABLE_INSERT, - ma_insert_profile_table_new_cb, - ma_profile_table_free_cb, - ma_profile_table_dup_cb); - if(ret<0) + for(int i = POLICY_ASN_USER_DEFINED; i < POLICY_PROFILE_TABLE_MAX; i++) { - goto error_out; - } - - ret = maat_table_ex_init(POLICY_PROFILE_TABLE_HIJACK, - ma_hijack_profile_table_new_cb, - ma_profile_table_free_cb, - ma_profile_table_dup_cb); - if(ret<0) - { - goto error_out; + ret = maat_ip_table_init(i, ip_table_free_cb, ip_table_dup_cb); + if(ret<0) + { + goto error_out; + } } g_pangu_rt->dyn_maat = create_maat_feather("dyn", profile_path, dynamic_section, g_pangu_rt->thread_num, g_pangu_rt->local_logger); @@ -1120,6 +1287,14 @@ struct insert_ctx int actually_replaced; }; +struct ip_data_ctx +{ + char *asn_client; + char *asn_server; + char *location_client; + char *location_server; +}; + struct pangu_http_ctx { int magic_num; @@ -1137,6 +1312,7 @@ struct pangu_http_ctx int manipulate_replaced; struct replace_ctx * rep_ctx; struct insert_ctx * ins_ctx; + struct ip_data_ctx ip_ctx; int (* resumed_cb)(const struct tfe_stream * stream, const struct tfe_http_session * session, enum tfe_http_event event, const unsigned char * data, @@ -1183,6 +1359,17 @@ void http_ins_ctx_free(struct insert_ctx* ins_ctx) return; } +void http_ip_ctx_free(struct ip_data_ctx *ip_ctx) +{ + if(ip_ctx->asn_client) + FREE(&ip_ctx->asn_client); + if(ip_ctx->asn_server) + FREE(&ip_ctx->asn_server); + if(ip_ctx->location_client) + FREE(&ip_ctx->location_client); + if(ip_ctx->location_server) + FREE(&ip_ctx->location_server); +} #define HTTP_CTX_MAGIC_NUM 20181021 static struct pangu_http_ctx * pangu_http_ctx_new(unsigned int thread_id) @@ -1197,16 +1384,18 @@ static struct pangu_http_ctx * pangu_http_ctx_new(unsigned int thread_id) static void pangu_http_ctx_free(struct pangu_http_ctx * ctx) { assert(ctx->magic_num==HTTP_CTX_MAGIC_NUM); - if (ctx->rep_ctx) + if(ctx->rep_ctx) { http_repl_ctx_free(ctx->rep_ctx); ctx->rep_ctx = NULL; } - if (ctx->ins_ctx) + if(ctx->ins_ctx) { http_ins_ctx_free(ctx->ins_ctx); ctx->ins_ctx = NULL; } + + http_ip_ctx_free(&ctx->ip_ctx); ctx->manipulate_replaced=0; FREE(&ctx->enforce_rules); policy_action_param_free(ctx->param); @@ -1842,6 +2031,7 @@ static void http_hijack(const struct tfe_http_session * session, enum tfe_http_e return; } ctx->inject_sz = hijack_size; + ATOMIC_ADD(&(g_pangu_rt->stat_val[STAT_ACTION_HIJACK_SZ]), hijack_size); char cont_len_str[16]; @@ -1947,6 +2137,7 @@ static void http_insert(const struct tfe_stream * stream, const struct tfe_http_ return; } ctx->inject_sz = ins_ctx->rule->inject_sz; + ATOMIC_ADD(&(g_pangu_rt->stat_val[STAT_ACTION_INSERT_SZ]), ctx->inject_sz); } else { @@ -2499,26 +2690,13 @@ void cache_write(const struct tfe_http_session * session, enum tfe_http_event ev ctx->cache_write_ctx=NULL; //printf("cache update success: %s\n", ctx->ref_session->req->req_spec.url); } - } -void pangu_on_http_begin(const struct tfe_stream * stream, - const struct tfe_http_session * session, unsigned int thread_id, void ** pme) +int http_ip_subscribe_id_scan(struct Maat_rule_t *result, const char* sip, const char* dip, int hit_cnt, unsigned int thread_id, struct pangu_http_ctx * ctx) { - struct pangu_http_ctx * ctx = *(struct pangu_http_ctx **) pme; - struct Maat_rule_t result[MAX_SCAN_RESULT]; - struct ipaddr sapp_addr; - int hit_cnt = 0, scan_ret=0; - UNUSED int tmp=0; - assert(ctx == NULL); - ATOMIC_INC(&(g_pangu_rt->stat_val[STAT_SESSION])); - ctx = pangu_http_ctx_new(thread_id); - char* addr_string=tfe_stream_addr_to_str(stream->addr); - const char* sip=NULL, *dip=NULL; + int scan_ret=0; char* source_subscribe_id=NULL, *dest_subscribe_id=NULL; - tmp=tfe_stream_addr_str_split(addr_string, &sip, NULL, &dip, NULL); - assert(tmp==0); source_subscribe_id=(char*)Maat_plugin_get_EX_data(g_pangu_rt->dyn_maat, g_pangu_rt->subscriber_id_table_id, sip); dest_subscribe_id=(char*)Maat_plugin_get_EX_data(g_pangu_rt->dyn_maat, g_pangu_rt->subscriber_id_table_id, dip); @@ -2544,6 +2722,174 @@ void pangu_on_http_begin(const struct tfe_stream * stream, hit_cnt+=scan_ret; } } + if(source_subscribe_id!=NULL) + free(source_subscribe_id); + if(dest_subscribe_id!=NULL) + free(dest_subscribe_id); + return hit_cnt; +} + +int http_ip_location_scan(struct Maat_rule_t *result, struct ip_address *sip, struct ip_address *dip, int hit_cnt, unsigned int thread_id, struct pangu_http_ctx * ctx ) +{ + int scan_ret=0; + char buff[TFE_STRING_MAX]={0}; + struct ip_data_table* ip_location_client=NULL, *ip_location_server=NULL; + + Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, g_pangu_rt->plolicy_table_id[POLICY_LOCATION_USER_DEFINED], sip, (void **)&ip_location_client, 1); + Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, g_pangu_rt->plolicy_table_id[POLICY_LOCATION_USER_DEFINED], dip, (void **)&ip_location_server, 1); + + if (ip_location_client == NULL) + { + Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, g_pangu_rt->plolicy_table_id[POLICY_LOCATION_BUILT_IN], sip, (void **)&ip_location_client, 1); + } + if (ip_location_server == NULL) + { + Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, g_pangu_rt->plolicy_table_id[POLICY_LOCATION_BUILT_IN], dip, (void **)&ip_location_server, 1); + } + + if(ip_location_server!=NULL) + { + snprintf(buff, sizeof(buff), "%s.%s.", ip_location_server->country_full, ip_location_server->city_full); + scan_ret = Maat_full_scan_string(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_IP_DST_LOCATION], + CHARSET_GBK, buff, strlen(buff), + result+hit_cnt, NULL, MAX_SCAN_RESULT-hit_cnt, + &(ctx->scan_mid), (int) thread_id); + if(scan_ret>0) + { + hit_cnt+=scan_ret; + } + ctx->ip_ctx.location_server=tfe_strdup(buff); + } + if(ip_location_client!=NULL) + { + snprintf(buff, sizeof(buff), "%s.%s.", ip_location_client->country_full, ip_location_client->city_full); + scan_ret = Maat_full_scan_string(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_IP_SRC_LOCATION], + CHARSET_GBK, buff, strlen(buff), + result+hit_cnt, NULL, MAX_SCAN_RESULT-hit_cnt, + &(ctx->scan_mid), (int) thread_id); + if(scan_ret>0) + { + hit_cnt+=scan_ret; + } + ctx->ip_ctx.location_client=tfe_strdup(buff); + } + + if(ip_location_server) + ip_table_free(ip_location_server); + if(ip_location_client) + ip_table_free(ip_location_client); + return hit_cnt; +} + +int http_ip_asn_scan(struct Maat_rule_t *result, struct ip_address* sip, struct ip_address* dip, int hit_cnt, unsigned int thread_id, struct pangu_http_ctx * ctx) +{ + int scan_ret=0; + struct ip_data_table* ip_asn_client=NULL, *ip_asn_server=NULL; + + Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, g_pangu_rt->plolicy_table_id[POLICY_ASN_USER_DEFINED], sip, (void **)&ip_asn_client, 1); + Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, g_pangu_rt->plolicy_table_id[POLICY_ASN_USER_DEFINED], dip, (void **)&ip_asn_server, 1); + + if (ip_asn_client == NULL) + { + Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, g_pangu_rt->plolicy_table_id[POLICY_ASN_BUILT_IN], sip, (void **)&ip_asn_client, 1); + } + if (ip_asn_server == NULL) + { + Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, g_pangu_rt->plolicy_table_id[POLICY_ASN_BUILT_IN], dip, (void **)&ip_asn_server, 1); + } + + if(ip_asn_server!=NULL) + { + scan_ret = Maat_full_scan_string(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_IP_DST_ASN], + CHARSET_UTF8, ip_asn_server->asn, strlen(ip_asn_server->asn), + result+hit_cnt, NULL, MAX_SCAN_RESULT-hit_cnt, + &(ctx->scan_mid), (int) thread_id); + + if(scan_ret>0) + { + hit_cnt+=scan_ret; + } + ctx->ip_ctx.asn_server=tfe_strdup(ip_asn_server->asn); + } + if(ip_asn_client!=NULL) + { + scan_ret = Maat_full_scan_string(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_IP_SRC_ASN], + CHARSET_UTF8, ip_asn_client->asn, strlen(ip_asn_client->asn), + result+hit_cnt, NULL, MAX_SCAN_RESULT-hit_cnt, + &(ctx->scan_mid), (int) thread_id); + if(scan_ret>0) + { + hit_cnt+=scan_ret; + } + ctx->ip_ctx.asn_client=tfe_strdup(ip_asn_client->asn); + } + if(ip_asn_server) + ip_table_free(ip_asn_server); + if(ip_asn_client) + ip_table_free(ip_asn_client); + return hit_cnt; +} + +int tfe_stream_addr_to_address(const struct tfe_stream_addr *addr, struct ip_address *dest_ip, struct ip_address *source_ip) +{ + if(addr==NULL) return -1; + if (addr->addrtype == TFE_ADDR_STREAM_TUPLE4_V4) + { + const struct tfe_stream_addr_tuple4_v4 * tuple4_v4 = addr->tuple4_v4; + source_ip->ip_type=4; + source_ip->ipv4=tuple4_v4->saddr.s_addr; + + dest_ip->ip_type=4; + dest_ip->ipv4=tuple4_v4->daddr.s_addr; + } + if (addr->addrtype == TFE_ADDR_STREAM_TUPLE4_V6) + { + const struct tfe_stream_addr_tuple4_v6 * tuple4_v6 = addr->tuple4_v6; + source_ip->ip_type=6; + memcpy((char *)(source_ip->ipv6), tuple4_v6->saddr.__in6_u.__u6_addr32, IPV6_ADDR_LEN); + + dest_ip->ip_type=6; + memcpy((char *)(dest_ip->ipv6),tuple4_v6->daddr.__in6_u.__u6_addr32, IPV6_ADDR_LEN); + } + return 0; +} + +void pangu_on_http_begin(const struct tfe_stream * stream, + const struct tfe_http_session * session, unsigned int thread_id, void ** pme) +{ + struct pangu_http_ctx * ctx = *(struct pangu_http_ctx **) pme; + struct Maat_rule_t result[MAX_SCAN_RESULT]; + struct ipaddr sapp_addr; + int hit_cnt = 0, scan_ret=0; + UNUSED int tmp=0; + assert(ctx == NULL); + ATOMIC_INC(&(g_pangu_rt->stat_val[STAT_SESSION])); + ctx = pangu_http_ctx_new(thread_id); + char* addr_string=tfe_stream_addr_to_str(stream->addr); + const char* sip=NULL, *dip=NULL; + + tmp=tfe_stream_addr_str_split(addr_string, &sip, NULL, &dip, NULL); + assert(tmp==0); + + scan_ret = http_ip_subscribe_id_scan(result, sip, dip, hit_cnt, thread_id, ctx); + if(scan_ret>0) + { + hit_cnt+=scan_ret; + } + + struct ip_address dest_ip, source_ip; + tfe_stream_addr_to_address(stream->addr, &dest_ip, &source_ip); + scan_ret = http_ip_location_scan(result, &source_ip, &dest_ip, hit_cnt, thread_id, ctx); + if(scan_ret>0) + { + hit_cnt+=scan_ret; + } + scan_ret = http_ip_asn_scan(result, &source_ip, &dest_ip, hit_cnt, thread_id, ctx); + if(scan_ret>0) + { + hit_cnt+=scan_ret; + } + const char *app_id = "http."; scan_ret = Maat_full_scan_string(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_APP_ID], CHARSET_UTF8, app_id, strlen(app_id), @@ -2571,8 +2917,6 @@ void pangu_on_http_begin(const struct tfe_stream * stream, *pme = ctx; free(addr_string); - free(source_subscribe_id); - free(dest_subscribe_id); return; } @@ -2649,7 +2993,9 @@ void pangu_on_http_end(const struct tfe_stream * stream, } } struct pangu_log log_msg = {.stream=stream, .http=session, .result=ctx->enforce_rules, .result_num=ctx->n_enforce, - .req_body=ctx->log_req_body, .resp_body=ctx->log_resp_body, .action=0, .inject_sz=ctx->inject_sz}; + .req_body=ctx->log_req_body, .resp_body=ctx->log_resp_body, .action=0, .inject_sz=ctx->inject_sz, + .asn_client=ctx->ip_ctx.asn_client, .asn_server=ctx->ip_ctx.asn_server, .location_client=ctx->ip_ctx.location_client, + .location_server=ctx->ip_ctx.location_server}; if(ctx->action == PG_ACTION_MANIPULATE) { diff --git a/plugin/business/pangu-http/src/pangu_logger.cpp b/plugin/business/pangu-http/src/pangu_logger.cpp index 8a17347..61ccebf 100644 --- a/plugin/business/pangu-http/src/pangu_logger.cpp +++ b/plugin/business/pangu-http/src/pangu_logger.cpp @@ -133,7 +133,7 @@ struct pangu_logger* pangu_log_handle_create(const char* profile, const char* s TFE_LOG_INFO(local_logger, "Pangu kafka brokerlist : %s", brokerlist); TFE_LOG_INFO(local_logger, "Pangu kafka topic : %s", topic_name); - + instance->kafka_logger = tfe_kafka_logger_create(instance->en_sendlog, nic_name, brokerlist, topic_name, local_logger); if (instance->kafka_logger == NULL) { @@ -361,6 +361,14 @@ int pangu_send_log(struct pangu_logger* handle, const struct pangu_log* log_msg) { cJSON_AddStringToObject(per_hit_obj, "common_sub_action", panggu_action_map[(unsigned char)(log_msg->result[i].action)]); } + if(log_msg->location_client) + { + cJSON_AddStringToObject(per_hit_obj, "common_client_location", log_msg->location_client); + } + if(log_msg->location_server) + { + cJSON_AddStringToObject(per_hit_obj, "common_server_location", log_msg->location_server); + } log_payload = cJSON_PrintUnformatted(per_hit_obj); diff --git a/plugin/business/pangu-http/src/pangu_logger.h b/plugin/business/pangu-http/src/pangu_logger.h index 91ca8cb..8de3cdc 100644 --- a/plugin/business/pangu-http/src/pangu_logger.h +++ b/plugin/business/pangu-http/src/pangu_logger.h @@ -13,6 +13,10 @@ struct pangu_log struct evbuffer* req_body, *resp_body; unsigned char action; size_t inject_sz; + char *asn_client; + char *asn_server; + char *location_client; + char *location_server; }; struct pangu_logger; struct pangu_logger* pangu_log_handle_create(const char* profile, const char* section, void* local_logger); diff --git a/resource/pangu/table_info.conf b/resource/pangu/table_info.conf index 371123e..1a57fa2 100644 --- a/resource/pangu/table_info.conf +++ b/resource/pangu/table_info.conf @@ -47,4 +47,14 @@ 26 TSG_SECURITY_COMPILE plugin {"key":1,"valid":8} 27 PXY_PROFILE_TRAFFIC_MIRROR plugin {"key":1,"valid":4} 28 TSG_PROFILE_DECRYPTION plugin {"key":1,"valid":4} +29 TSG_IP_ASN_BUILT_IN ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":7} +30 TSG_IP_ASN_USER_DEFINED ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":7} +31 TSG_IP_LOCATION_BUILT_IN ip_plugin {"row_id":1,"ip_type":3,"start_ip":4,"end_ip":5,"valid":18} +32 TSG_IP_LOCATION_USER_DEFINED ip_plugin {"row_id":1,"ip_type":3,"start_ip":4,"end_ip":5,"valid":18} +33 TSG_OBJ_AS_NUMBER expr UTF8 UTF8/GBK yes 0 +34 TSG_SECURITY_SOURCE_ASN virtual TSG_OBJ_AS_NUMBER -- +35 TSG_SECURITY_DESTINATION_ASN virtual TSG_OBJ_AS_NUMBER -- +36 TSG_OBJ_GEO_LOCATION expr UTF8 UTF8/GBK yes 0 +37 TSG_SECURITY_SOURCE_LOCATION virtual TSG_OBJ_GEO_LOCATION -- +38 TSG_SECURITY_DESTINATION_LOCATION virtual TSG_OBJ_GEO_LOCATION -- diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index 5e61b95..b897f49 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -169,9 +169,9 @@ add_library(MESA_field_stat SHARED IMPORTED GLOBAL) set_property(TARGET MESA_field_stat PROPERTY IMPORTED_LOCATION ${MESA_FRAMEWORK_LIB_DIR}/libMESA_field_stat2.so) set_property(TARGET MESA_field_stat PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MESA_FRAMEWORK_INCLUDE_DIR}) -add_library(librdkafka SHARED IMPORTED GLOBAL) -set_property(TARGET librdkafka PROPERTY IMPORTED_LOCATION ${MESA_FRAMEWORK_LIB_DIR}/librdkafka.so) -set_property(TARGET librdkafka PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MESA_FRAMEWORK_INCLUDE_DIR}) +#add_library(librdkafka SHARED IMPORTED GLOBAL) +#set_property(TARGET librdkafka PROPERTY IMPORTED_LOCATION ${MESA_FRAMEWORK_LIB_DIR}/librdkafka.so) +#set_property(TARGET librdkafka PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MESA_FRAMEWORK_INCLUDE_DIR}) add_library(mrzcpd SHARED IMPORTED GLOBAL) set_property(TARGET mrzcpd PROPERTY IMPORTED_LOCATION ${MRZCPD_LIB_DIR}/libmarsio.so)