From 0af47e8a4921cad71c999b27c956ca2c9f5aa55f Mon Sep 17 00:00:00 2001 From: liuxueli Date: Thu, 2 Jun 2022 10:48:00 +0800 Subject: [PATCH] =?UTF-8?q?TSG-9128:=20HOST=E5=92=8CURL=E5=B1=9E=E4=BA=8E?= =?UTF-8?q?=E5=90=8C=E4=B8=80=E4=BC=98=E5=85=88=E7=BA=A7=EF=BC=8C=E4=BF=9D?= =?UTF-8?q?=E8=AF=81=E5=AE=89=E5=85=A8=E6=A8=A1=E5=9D=97=E5=92=8C=E4=BB=A3?= =?UTF-8?q?=E7=90=86=E6=A8=A1=E5=9D=97=E7=9B=B8=E5=90=8C=E7=AD=96=E7=95=A5?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E4=B8=8B=EF=BC=8C=E4=BC=98=E5=85=88=E7=BA=A7?= =?UTF-8?q?=E6=95=88=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/tsg_log_field.conf | 1 + inc/tsg_label.h | 6 ++ inc/tsg_rule.h | 3 + src/tsg_entry.cpp | 122 +++++++++++++++++++++++------------- src/tsg_entry.h | 8 ++- src/tsg_rule.cpp | 25 +++++++- src/tsg_send_log_internal.h | 1 + 7 files changed, 121 insertions(+), 45 deletions(-) diff --git a/bin/tsg_log_field.conf b/bin/tsg_log_field.conf index 237e55e..bf3f6a0 100644 --- a/bin/tsg_log_field.conf +++ b/bin/tsg_log_field.conf @@ -108,3 +108,4 @@ LONG l2tp_network_server_port 96 STRING l2tp_version 97 STRING common_packet_capture_file 98 STRING common_app_behavior 99 +STRING http_url 100 diff --git a/inc/tsg_label.h b/inc/tsg_label.h index ea82160..11e4a35 100644 --- a/inc/tsg_label.h +++ b/inc/tsg_label.h @@ -92,6 +92,12 @@ struct policy_priority_label int domain_len; int result_type; //enum _PULL_RESULT_TYPE (tsg_rule.h) int result_num; + union + { + char *http_url; + char *quic_ua; + void *para; + }; char domain[MAX_DOMAIN_LEN]; Maat_rule_t result[MAX_RESULT_NUM]; }; diff --git a/inc/tsg_rule.h b/inc/tsg_rule.h index e3295b4..af88866 100644 --- a/inc/tsg_rule.h +++ b/inc/tsg_rule.h @@ -54,6 +54,9 @@ int tsg_rule_init(const char *conffile, void *logger); int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num); int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_result_type, Maat_rule_t*result, int result_num, struct identify_info *identify_info); +char *tsg_pull_quic_ua(struct streaminfo *a_stream); +char *tsg_pull_http_url(struct streaminfo *a_stream); + //return NULL if none exists, otherwise return one deny rule; struct Maat_rule_t *tsg_fetch_deny_rule(Maat_rule_t *result, int result_num); diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index c68900e..d62e7e6 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -581,6 +581,13 @@ static int master_send_log(const struct streaminfo *a_stream, struct Maat_rule_t TLD_append(TLD_handle, schema_field_name, (void *)g_tsg_proto_name2id[proto].name, TLD_TYPE_STRING); } + + if(proto==PROTO_HTTP) + { + TLD_append(TLD_handle, (char *)"http_version", (void *)"http1", TLD_TYPE_STRING); + TLD_append(TLD_handle, (char *)"http_sequence", (void *)1, TLD_TYPE_LONG); + } + if(context!=NULL && context->domain!=NULL) { switch(proto) @@ -588,6 +595,12 @@ static int master_send_log(const struct streaminfo *a_stream, struct Maat_rule_t case PROTO_HTTP: domain_field_name=log_field_id2name(g_tsg_log_instance, LOG_HTTP_HOST); TLD_append(TLD_handle, domain_field_name, (void *)context->domain, TLD_TYPE_STRING); + + if(context->http_url!=NULL) + { + domain_field_name=log_field_id2name(g_tsg_log_instance, LOG_HTTP_URL); + TLD_append(TLD_handle, domain_field_name, (void *)context->http_url, TLD_TYPE_STRING); + } break; case PROTO_SSL: domain_field_name=log_field_id2name(g_tsg_log_instance, LOG_SSL_SNI); @@ -735,7 +748,13 @@ static void free_tcpall_label(int thread_seq, void *project_req_value) static void free_policy_label(int thread_seq, void *project_req_value) { if(project_req_value!=NULL) - { + { + struct policy_priority_label *priority_label=(struct policy_priority_label *)project_req_value; + if(priority_label->para!=NULL) + { + dictator_free(thread_seq, priority_label->para); + priority_label->para=NULL; + } dictator_free(thread_seq, project_req_value); project_req_value=NULL; } @@ -750,6 +769,20 @@ void free_gather_app_result(int thread_seq, void *project_req_value) } } +static char *malloc_copy_string(char *origin, int length, int thread_seq) +{ + if(length<=0 && origin==NULL) + { + return NULL; + } + + char *dest=(char *)dictator_malloc(thread_seq, length+1); + memcpy(dest, origin, length); + dest[length]='\0'; + + return dest; +} + static void copy_monitor_result(const struct streaminfo *a_stream, struct master_context *context, struct Maat_rule_t *p_result, int result_num, int thread_seq) { int i=0; @@ -788,7 +821,7 @@ static void copy_monitor_result(const struct streaminfo *a_stream, struct master } -static void copy_result_to_project(const struct streaminfo *a_stream, struct master_context *context, struct Maat_rule_t *p_result, char *domain, tsg_protocol_t proto, PULL_RESULT_TYPE result_type, int thread_seq) +static void copy_result_to_project(const struct streaminfo *a_stream, struct master_context *context, struct Maat_rule_t *p_result, PULL_RESULT_TYPE result_type, int thread_seq) { int ret=0; struct policy_priority_label *priority_label=NULL; @@ -804,7 +837,7 @@ static void copy_result_to_project(const struct streaminfo *a_stream, struct mas RLOG_LV_DEBUG, "DUP_HIT_POLICY", "Hit policy, domain: %s policy_id: %d action: %d addr: %s", - (domain!=NULL ? domain : ""), + (context->domain!=NULL ? context->domain : ""), p_result->config_id, (unsigned char)p_result->action, PRINTADDR(a_stream, g_tsg_para.level) @@ -813,13 +846,21 @@ static void copy_result_to_project(const struct streaminfo *a_stream, struct mas memset(priority_label, 0, sizeof(struct policy_priority_label)); - priority_label->proto=proto; - if(domain!=NULL) + priority_label->proto=context->proto; + if(context->domain!=NULL) { - priority_label->domain_len=MIN(sizeof(priority_label->domain)-1 ,strlen(domain)); - memcpy(priority_label->domain, domain, priority_label->domain_len); + priority_label->domain_len=MIN(sizeof(priority_label->domain)-1 ,strlen(context->domain)); + memcpy(priority_label->domain, context->domain, priority_label->domain_len); } - + + if(context->para!=NULL) + { + int length=strlen(context->http_url); + priority_label->para=dictator_malloc(thread_seq,length+1); + memcpy(priority_label->para, context->para, length); + ((char *)priority_label->para)[length]='\0'; + } + priority_label->result_num=1; priority_label->result_type=result_type; memcpy(priority_label->result, p_result, sizeof(struct Maat_rule_t)); @@ -831,19 +872,22 @@ static void copy_result_to_project(const struct streaminfo *a_stream, struct mas MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_ADD", - "Add policy_priority_label failed, policy, domain: %s policy_id: %d action: %d addr: %s", - (domain!=NULL ? domain : ""), + "Add policy_priority_label failed, policy, domain: %s para(url/ua): %s policy_id: %d action: %d addr: %s", + (context->domain!=NULL ? context->domain : ""), + (context->para!=NULL ? context->http_url : ""), priority_label->result[0].config_id, (unsigned char)priority_label->result[0].action, PRINTADDR(a_stream, g_tsg_para.level) ); - } + return ; + } MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "COPY_RESULT", - "Hit policy, domain: %s policy_id: %d action: %d addr: %s", - (domain!=NULL ? domain : ""), + "Hit policy, domain: %s para(url/ua): %s policy_id: %d action: %d addr: %s", + (context->domain!=NULL ? context->domain : ""), + (context->para!=NULL ? context->http_url : ""), priority_label->result[0].config_id, (unsigned char)priority_label->result[0].action, PRINTADDR(a_stream, g_tsg_para.level) @@ -1319,17 +1363,18 @@ static int identify_application_protocol(const struct streaminfo *a_stream, stru case STREAM_TYPE_TCP: if(g_tsg_para.proto_flag&(1<ptcpdetail->pdata, (unsigned int)a_stream->ptcpdetail->datalen, a_stream->curdir, &host); - if(length>=0) + char *host=NULL, *url=NULL; + void *parser_result=http_field_parser((char *)a_stream->ptcpdetail->pdata, (unsigned int)a_stream->ptcpdetail->datalen, a_stream->curdir); + if(parser_result!=NULL) { context->proto=PROTO_HTTP; - if(length>0 && host!=NULL) - { - context->domain=(char *)dictator_malloc(a_stream->threadnum, length+1); - memset(context->domain, 0, length+1); - memcpy(context->domain, host, length); - } + + length=http_get_filed_result(parser_result, HTTP_HOST, &host); + context->domain=malloc_copy_string(host, length, a_stream->threadnum); + length=http_get_filed_result(parser_result, HTTP_MESSAGE_URL, &url); + context->http_url=malloc_copy_string(url, length, a_stream->threadnum); + + http_free_filed_result(parser_result); return 1; } } @@ -1345,10 +1390,7 @@ static int identify_application_protocol(const struct streaminfo *a_stream, stru context->proto=PROTO_SSL; if(chello->sni!=NULL) { - length=strlen(chello->sni); - context->domain=(char *)dictator_malloc(a_stream->threadnum, length+1); - memset(context->domain, 0, length+1); - memcpy(context->domain, chello->sni, length); + context->domain=malloc_copy_string(chello->sni, strlen(chello->sni), a_stream->threadnum); } context->is_esni=(int)chello->is_encrypt_sni; @@ -1452,19 +1494,8 @@ static int identify_application_protocol(const struct streaminfo *a_stream, stru if(context->quic_version > 0) { context->proto=PROTO_QUIC; - if(sni_len>0) - { - context->domain=(char *)dictator_malloc(a_stream->threadnum, sni_len+1); - memcpy(context->domain, sni_buff, sni_len); - context->domain[sni_len]='\0'; - } - - if(ua_len>0) - { - context->quic_ua=(char *)dictator_malloc(a_stream->threadnum, ua_len+1); - memcpy(context->quic_ua, ua_buff, ua_len); - context->quic_ua[ua_len]='\0'; - } + context->domain=malloc_copy_string(sni_buff, sni_len, a_stream->threadnum); + context->quic_ua=malloc_copy_string(ua_buff, ua_len, a_stream->threadnum); return 1; } } @@ -1605,7 +1636,7 @@ static unsigned char master_deal_scan_result(const struct streaminfo *a_stream, tmp_tcpall_context->hited_para=context->hited_para; } - copy_result_to_project(a_stream, context, p_result, context->domain, context->proto, PULL_FW_RESULT, a_stream->threadnum); + copy_result_to_project(a_stream, context, p_result, PULL_FW_RESULT, a_stream->threadnum); context->is_dropme=1; //only tcp state=APP_STATE_KILL_OTHER|APP_STATE_GIVEME; break; @@ -1624,7 +1655,7 @@ static unsigned char master_deal_scan_result(const struct streaminfo *a_stream, { context->hit_cnt=0; master_send_log(a_stream, p_result, 1, context, a_stream->threadnum); - copy_result_to_project(a_stream, context, p_result, context->domain, context->proto, PULL_FW_RESULT, a_stream->threadnum); + copy_result_to_project(a_stream, context, p_result, PULL_FW_RESULT, a_stream->threadnum); MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "DENY", @@ -1646,7 +1677,7 @@ static unsigned char master_deal_scan_result(const struct streaminfo *a_stream, break; case TSG_ACTION_BYPASS: copy_bypass_result(a_stream, context, p_result, a_stream->threadnum); - copy_result_to_project(a_stream, context, p_result, context->domain, context->proto, PULL_FW_RESULT, a_stream->threadnum); + copy_result_to_project(a_stream, context, p_result, PULL_FW_RESULT, a_stream->threadnum); FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_BYPASS], 0, FS_OP_ADD, 1); state=APP_STATE_GIVEME|APP_STATE_KILL_OTHER; @@ -1659,7 +1690,7 @@ static unsigned char master_deal_scan_result(const struct streaminfo *a_stream, break; } - copy_result_to_project(a_stream, context, p_result, context->domain, context->proto, PULL_KNI_RESULT, a_stream->threadnum); + copy_result_to_project(a_stream, context, p_result, PULL_KNI_RESULT, a_stream->threadnum); FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_INTERCEPT], 0, FS_OP_ADD, 1); state=APP_STATE_DROPME|APP_STATE_KILL_OTHER; @@ -1795,6 +1826,11 @@ static int deal_pending_state(const struct streaminfo *a_stream, struct master_c protocol_id=tsg_l7_protocol_name2id("ESNI"); hit_num+=tsg_scan_app_id_policy(g_tsg_maat_feather, a_stream, result+hit_num, MAX_RESULT_NUM-hit_num, &context->mid, (char *)"ESNI", protocol_id, a_stream->threadnum); } + + if(context->proto==PROTO_HTTP && context->http_url!=NULL) + { + hit_num+=tsg_scan_shared_policy(g_tsg_maat_feather, a_stream, context->http_url, result+hit_num, MAX_RESULT_NUM-hit_num, &context->mid, g_tsg_para.table_id[TABLE_HTTP_URL], a_stream->threadnum); + } } ret=tsg_scan_nesting_addr(g_tsg_maat_feather, a_stream, context->proto, &context->mid, result+hit_num, MAX_RESULT_NUM-hit_num); diff --git a/src/tsg_entry.h b/src/tsg_entry.h index cb55c34..9ac1276 100644 --- a/src/tsg_entry.h +++ b/src/tsg_entry.h @@ -89,6 +89,7 @@ enum MASTER_TABLE{ TABLE_RESPONSE_PAGES, TABLE_DNS_PROFILE_RECORD, TABLE_PROFILE_MIRROR, + TABLE_HTTP_URL, TABLE_MAX }; @@ -224,7 +225,12 @@ struct master_context int hited_app_id; unsigned int quic_version; char *domain; - char *quic_ua; + union + { + char *http_url; + char *quic_ua; + void *para; + }; scan_status_t mid; struct Maat_rule_t *result; struct hited_app_para hited_para; diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index 23acd97..d513d24 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -1704,7 +1704,8 @@ int tsg_rule_init(const char* conffile, void *logger) MESA_load_profile_string_def(conffile, "MAAT", "IP_ADDR_TABLE", g_tsg_para.table_name[TABLE_IP_ADDR], _MAX_TABLE_NAME_LEN, "TSG_SECURITY_ADDR"); MESA_load_profile_string_def(conffile, "MAAT", "SUBSCRIBER_ID_TABLE", g_tsg_para.table_name[TABLE_SUBSCRIBER_ID], _MAX_TABLE_NAME_LEN, "TSG_OBJ_SUBSCRIBER_ID"); MESA_load_profile_string_def(conffile, "MAAT", "APP_ID_TABLE", g_tsg_para.table_name[TABLE_APP_ID], _MAX_TABLE_NAME_LEN, "TSG_OBJ_APP_ID"); - MESA_load_profile_string_def(conffile, "MAAT", "HTTP_HOST_TABLE", g_tsg_para.table_name[TABLE_HTTP_HOST], _MAX_TABLE_NAME_LEN, "TSG_FIELD_HTTP_HOST"); + MESA_load_profile_string_def(conffile, "MAAT", "HTTP_HOST_TABLE", g_tsg_para.table_name[TABLE_HTTP_HOST], _MAX_TABLE_NAME_LEN, "TSG_FIELD_HTTP_HOST"); + MESA_load_profile_string_def(conffile, "MAAT", "HTTP_URL_TABLE", g_tsg_para.table_name[TABLE_HTTP_URL], _MAX_TABLE_NAME_LEN, "TSG_FIELD_HTTP_URL"); MESA_load_profile_string_def(conffile, "MAAT", "SSL_SNI_TABLE", g_tsg_para.table_name[TABLE_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_FIELD_SSL_SNI"); MESA_load_profile_string_def(conffile, "MAAT", "DECYPTION_EXCLUSION_SSL_SNI", g_tsg_para.table_name[TABLE_EXCLUSION_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_DECYPTION_EXCLUSION_SSL_SNI"); @@ -1991,6 +1992,28 @@ int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_re return 0; } +char *tsg_pull_quic_ua(struct streaminfo *a_stream) +{ + struct policy_priority_label *label=(struct policy_priority_label *)project_req_get_struct(a_stream, g_tsg_para.priority_project_id); + if(label!=NULL) + { + return label->quic_ua; + } + + return NULL; +} + +char *tsg_pull_http_url(struct streaminfo *a_stream) +{ + struct policy_priority_label *label=(struct policy_priority_label *)project_req_get_struct(a_stream, g_tsg_para.priority_project_id); + if(label!=NULL) + { + return label->http_url; + } + + return NULL; +} + int tsg_get_ip_asn(const struct streaminfo *a_stream, int table_id, MAAT_PLUGIN_EX_DATA* client_asn, MAAT_PLUGIN_EX_DATA* server_asn) { struct ip_address dest_ip={0}, source_ip={0}; diff --git a/src/tsg_send_log_internal.h b/src/tsg_send_log_internal.h index 2cf35fb..52b11b0 100644 --- a/src/tsg_send_log_internal.h +++ b/src/tsg_send_log_internal.h @@ -125,6 +125,7 @@ typedef enum _tsg_log_field_id LOG_COMMON_L2TP_VERSION, LOG_COMMON_PACKET_CAPTURE_FILE, LOG_COMMON_APPLICATION_BEHAVIOR, + LOG_HTTP_URL, LOG_COMMON_MAX }tsg_log_field_id_t;