diff --git a/bin/tsg_log_field.conf b/bin/tsg_log_field.conf index 7a40328..ae00fcd 100644 --- a/bin/tsg_log_field.conf +++ b/bin/tsg_log_field.conf @@ -42,3 +42,7 @@ STRING http_host 35 STRING ssl_sni 36 LONG common_establish_latency_ms 37 STRING common_sub_action 38 +STRING common_clinet_asn 39 +STRING common_server_asn 40 +STRING common_clinet_location 41 +STRING common_server_location 42 diff --git a/bin/tsg_static_tableinfo.conf b/bin/tsg_static_tableinfo.conf index ba7b0c9..4da5581 100644 --- a/bin/tsg_static_tableinfo.conf +++ b/bin/tsg_static_tableinfo.conf @@ -45,3 +45,9 @@ 33 TSG_SECURITY_ADDR composition {"source":"TSG_SECURITY_SOURCE_ADDR","destination":"TSG_SECURITY_DESTINATION_ADDR"} 34 TSG_DECYPTION_EXCLUSION_SSL_SNI virtual TSG_OBJ_FQDN 35 TSG_PROFILE_RESPONSE_PAGES plugin {"key":1,"foreign":"4","valid":5} -- +36 TSG_IP_ASN_BUILT_IN ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":7} +37 TSG_IP_ASN_USER_DEFINED ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":7} +38 TSG_IP_LOCATION_BUILT_IN ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":17} +39 TSG_IP_LOCATION_USER_DEFINED ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":17} +40 TSG_OBJ_IP_ASN expr UTF8 UTF8/GBK yes 0 +41 TSG_OBJ_IP_LOCATION expr UTF8 UTF8/GBK yes 0 diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index 6f13fef..986d225 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -41,7 +41,7 @@ static __attribute__((__used__)) const char * GIT_VERSION_UNKNOWN = NULL; #endif -char TSG_MASTER_VERSION_20200507=0; +char TSG_MASTER_VERSION_20200514=0; const char *tsg_conffile="tsgconf/main.conf"; g_tsg_para_t g_tsg_para; @@ -60,6 +60,50 @@ static void free_policy_label(int thread_seq, void *project_req_value) project_req_value=NULL; } +static void free_internal_label(int thread_seq, void *project_req_value) +{ + struct _internal_label *label=(struct _internal_label *)project_req_value; + + if(label->client_asn!=NULL) + { + ASN_free_data(label->client_asn->table_id, (MAAT_PLUGIN_EX_DATA *)&(label->client_asn), 0, g_tsg_para.logger); + label->client_asn=NULL; + } + + if(label->server_asn!=NULL) + { + ASN_free_data(label->server_asn->table_id, (MAAT_PLUGIN_EX_DATA *)&(label->server_asn), 0, g_tsg_para.logger); + label->server_asn=NULL; + } + + if(label->client_location!=NULL) + { + location_free_data(label->client_location->table_id, (MAAT_PLUGIN_EX_DATA *)&(label->client_location), 0, g_tsg_para.logger); + label->client_location=NULL; + } + + if(label->server_location!=NULL) + { + location_free_data(label->server_location->table_id, (MAAT_PLUGIN_EX_DATA *)&(label->server_location), 0, g_tsg_para.logger); + label->server_location=NULL; + } + + if(label->client_subscribe_id!=NULL) + { + free(label->client_subscribe_id); + label->client_subscribe_id=NULL; + } + + if(label->server_subscribe_id!=NULL) + { + free(label->server_subscribe_id); + label->server_subscribe_id=NULL; + } + + dictator_free(thread_seq, project_req_value); + project_req_value=NULL; +} + static void free_context(void **pme, int thread_seq) { struct _master_context *_context=(struct _master_context *)*pme; @@ -92,6 +136,7 @@ static int init_context(void **pme, tsg_protocol_t proto, struct Maat_rule_t *p_ return 0; } +#if 0 static int master_method_type(struct streaminfo *a_stream, struct Maat_rule_t *p_result) { cJSON *item=NULL; @@ -172,6 +217,7 @@ static int master_method_type(struct streaminfo *a_stream, struct Maat_rule_t *p return method_type; } + static int master_do_deny(struct streaminfo *a_stream, struct Maat_rule_t *p_result, int thread_seq) { int opt_value=0; @@ -218,6 +264,7 @@ static int master_do_deny(struct streaminfo *a_stream, struct Maat_rule_t *p_res return 0; } +#endif static char *schema_index2string(tsg_protocol_t proto) { @@ -346,12 +393,15 @@ static struct Maat_rule_t *tsg_policy_decision_criteria(struct streaminfo *a_str { if((unsigned char)result[i].action>(unsigned char)p_result->action) { - q_result=p_result; + if(p_result->action!=TSG_ACTION_INTERCEPT) + { + q_result=p_result; + } p_result=&result[i]; continue; } - if(q_result==NULL) + if(q_result==NULL && result[i].action!=TSG_ACTION_INTERCEPT) { q_result=&result[i]; } @@ -517,7 +567,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t Maat_rule_t all_result[MAX_RESULT_NUM]; policy_priority_label_t *priority_label=NULL; struct _master_context *_context=(struct _master_context *)*pme; - + struct _internal_label *internal_label=NULL; switch(a_tcp->opstate) { case OP_STATE_PENDING: @@ -527,9 +577,13 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t identify_application_protocol(a_tcp, &identify_info); if(identify_info.proto==PROTO_HTTP) { + internal_label=(struct _internal_label *)dictator_malloc(1, sizeof(struct _internal_label)); + memset(internal_label, 0, sizeof(struct _internal_label)); + clock_gettime(CLOCK_REALTIME, &tv); - establish_latency_ms=(tv.tv_sec-a_tcp->ptcpdetail->createtime)*1000+tv.tv_nsec/1000/1000; - ret=project_req_add_long(a_tcp, g_tsg_para.establish_latency_project_id, establish_latency_ms); + internal_label->establish_latency_ms=(tv.tv_sec-a_tcp->ptcpdetail->createtime)*1000+tv.tv_nsec/1000/1000; + + ret=project_req_add_struct(a_tcp, g_tsg_para.internal_project_id, (const void *)internal_label); if(ret<0) { MESA_handle_runtime_log(g_tsg_para.logger, @@ -561,7 +615,6 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t } ret=tsg_scan_shared_policy(g_tsg_maat_feather, &identify_info, all_result+hit_num, MAX_RESULT_NUM-hit_num, &mid, thread_seq); - if(ret>0) { FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_SHARE], 0, FS_OP_ADD, 1); @@ -585,7 +638,6 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t RLOG_LV_DEBUG, "SCAN_FQDN", "Not hit %s: %s stream_dir: %d addr: %s", - (ret==-1) ? "NULL" : ((identify_info.proto==PROTO_HTTP) ? "host" : "sni"), (ret==-1) ? "NULL" : identify_info.domain, a_tcp->dir, @@ -854,14 +906,14 @@ extern "C" int TSG_MASTER_INIT() return -1; } - MESA_load_profile_string_def(tsg_conffile, "SYSTEM", "ESTABLISH_LATENCY_LABEL", label_buff, sizeof(label_buff), "ESTABLISH_LATENCY"); - g_tsg_para.establish_latency_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_LONG, NULL); - if(g_tsg_para.establish_latency_project_id<0) + MESA_load_profile_string_def(tsg_conffile, "SYSTEM", "TSG_MASTER_INTERNAL_LABEL", label_buff, sizeof(label_buff), "TSG_MASTER_INTERNAL_LABEL"); + g_tsg_para.internal_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_STRUCT, free_internal_label); + if(g_tsg_para.internal_project_id<0) { MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_REGISTER", - "Register %s failed; please check :%s and add ", + "Register %s failed; please check :%s and add ", label_buff, "etc/project_list.conf" ); @@ -880,7 +932,7 @@ extern "C" int TSG_MASTER_INIT() MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_SENDLOG", "tsg_sendlog_init failed ..."); return -1; } - g_tsg_log_instance->establish_latency_project_id=g_tsg_para.establish_latency_project_id; + g_tsg_log_instance->internal_project_id=g_tsg_para.internal_project_id; MESA_load_profile_int_def(tsg_conffile, "FIELD_STAT", "CYCLE", &cycle, 30); MESA_load_profile_short_nodef(tsg_conffile, "FIELD_STAT","TELEGRAF_PORT", (short *)&(fs_server_port)); diff --git a/src/tsg_entry.h b/src/tsg_entry.h index c0e6ff8..5c839aa 100644 --- a/src/tsg_entry.h +++ b/src/tsg_entry.h @@ -5,6 +5,18 @@ #include #include "tsg_rule.h" +#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411) +#define atomic_inc(x) __sync_add_and_fetch((x),1) +#define atomic_dec(x) __sync_sub_and_fetch((x),1) +#define atomic_add(x,y) __sync_add_and_fetch((x),(y)) +#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y)) +typedef int atomic_t; +#define ATOMIC_INIT(i) { (i) } +#define atomic_read(x) __sync_add_and_fetch((x),0) +#define atomic_set(x,y) __sync_lock_test_and_set((x),y) +#else +#include +#endif #ifndef MIN #define MIN(a, b) (((a) < (b)) ? (a) : (b)) @@ -17,6 +29,12 @@ enum MASTER_TABLE{ TABLE_HTTP_HOST, TABLE_SSL_SNI, TABLE_EXCLUSION_SSL_SNI, + TABLE_IP_ASN, + TABLE_IP_LOCATION, + TABLE_ASN_USER_DEFINED, + TABLE_ASN_BUILT_IN, + TABLE_LOCATION_USER_DEFINED, + TABLE_LOCATION_BUILT_IN, TABLE_MAX }; @@ -39,6 +57,57 @@ struct _str2index char *type; }; +struct _asn_info +{ + int ref_cnt; + int addr_type; + char start_ip[40]; + char end_ip[40]; + char asn[40]; + char organization[256]; + int table_id; +}; + +struct _location_info +{ + int ref_cnt; + int addr_type; + char start_ip[40]; + char end_ip[40]; + double latitude; + double longitude; + double coords; + char language[40]; + char continent_abbr[256]; + char continent_full[256]; + char country_abbr[256]; + char country_full[256]; + char province_abbr[256]; + char province_full[256]; + char city_full[256]; + char time_zone[128]; + int parent_location_id; + int table_id; +}; + +struct _subscribe_id_info +{ + int ref_cnt; + char subscribe_id[256]; +}; + + +struct _internal_label +{ + long establish_latency_ms; + struct _asn_info *client_asn; + struct _asn_info *server_asn; + struct _location_info *client_location; + struct _location_info *server_location; + struct _subscribe_id_info *client_subscribe_id; + struct _subscribe_id_info *server_subscribe_id; +}; + typedef struct _policy_priority_label { @@ -67,7 +136,7 @@ typedef struct _tsg_para int table_id[TABLE_MAX]; int dyn_subscribe_ip_table_id; //TSG_DYN_SUBSCRIBER_IP int priority_project_id; - int establish_latency_project_id; + int internal_project_id; int fs2_field_id[TSG_FS2_MAX]; char table_name[TABLE_MAX][_MAX_TABLE_NAME_LEN]; void *logger; @@ -150,4 +219,8 @@ int tsg_statistic_init(const char *conffile, void *logger); int tsg_scan_shared_policy(Maat_feather_t maat_feather, struct _identify_info *identify_info, Maat_rule_t *result, int result_num, scan_status_t *mid, int thread_seq); +void location_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp); +void ASN_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp); + + #endif diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index ee66467..df1fed4 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -54,14 +54,221 @@ const struct _str2index g_tsg_proto_string[PROTO_MAX+1]={{PROTO_UNKONWN, 0, (cha {PROTO_MAX, 0, (char *)""} }; +void ASN_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void* argp) +{ + void *logger=argp; + struct _asn_info *asn=(struct _asn_info *)(*from); + + atomic_inc(&asn->ref_cnt); + *to=*from; + + MESA_handle_runtime_log(logger, + RLOG_LV_DEBUG, + "IP_ASN", + "Add ip ASN: [ start_ip: %s end_ip: %s asn: %d organization: %s ref_cnt: %d ] table_id: %d", + asn->start_ip, + asn->end_ip, + asn->asn, + asn->organization, + asn->ref_cnt, + table_id + ); + + return; +} + +void ASN_new_data(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp) +{ + void *logger=argp; + int ret=0,id=0,is_valid=0; + + struct _asn_info *asn=NULL; + asn=(struct _asn_info *)calloc(1, sizeof(struct _asn_info)); + + ret=sscanf(table_line, "%d\t%d\t%s\t%s\t%s\t%s\t%d", &id, &asn->addr_type, asn->start_ip, asn->end_ip, asn->asn, asn->organization, &is_valid); + if(ret!=7) + { + free(asn); + asn=NULL; + + MESA_handle_runtime_log(logger, + RLOG_LV_FATAL, + "IP_ASN", + "Parse ip ASN failed, ret: %d table_id: %d table_line: %s", + ret, + table_id, + table_line + ); + return; + } + + atomic_inc(&asn->ref_cnt); + *ad=(MAAT_PLUGIN_EX_DATA)asn; + + MESA_handle_runtime_log(logger, + RLOG_LV_DEBUG, + "IP_ASN", + "Add ip ASN: [ start_ip: %s end_ip: %s asn: %d organization: %s ] table_id: %d table_line: %s", + asn->start_ip, + asn->end_ip, + asn->asn, + asn->organization, + table_id, + table_line + ); + return; +} + +void ASN_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp) +{ + void *logger=argp; + struct _asn_info *asn=(struct _asn_info *)(*ad); + + atomic_dec(&asn->ref_cnt); + if(asn->ref_cnt<=0) + { + free(*ad); + *ad=NULL; + } + + MESA_handle_runtime_log(logger, + RLOG_LV_DEBUG, + "IP_ASN", + "Add ip ASN: [ start_ip: %s end_ip: %s asn: %d organization: %s ref_cnt: %d ] table_id: %d", + asn->start_ip, + asn->end_ip, + asn->asn, + asn->organization, + asn->ref_cnt, + table_id + ); + + return; +} + +void location_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void* argp) +{ + void *logger=argp; + struct _location_info *location=(struct _location_info *)(*from); + + atomic_inc(&location->ref_cnt); + *to=*from; + + MESA_handle_runtime_log(logger, + RLOG_LV_DEBUG, + "IP_LOCATION", + "Dup location: [ start_ip: %s end_ip: %s continent: %s country: %s city: %s ref_cnt: %d ] table_id: %d", + location->start_ip, + location->end_ip, + location->continent_full, + location->country_full, + location->city_full, + table_id); + return; +} + +void location_new_data(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp) +{ + void *logger=argp; + int ret=0,id=0,is_valid=0; + struct _location_info *location=NULL; + + location=(struct _location_info *)calloc(1, sizeof(struct _location_info)); + + ret=sscanf(table_line, + "%d\t%d\t%s\t%s\t%lf\t%lf\t%lf\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%u\t%d", + &id, + &location->addr_type, + location->start_ip, + location->end_ip, + &location->latitude, + &location->longitude, + &location->coords, + location->language, + location->continent_abbr, + location->continent_full, + location->country_abbr, + location->country_full, + location->province_abbr, + location->province_full, + location->city_full, + location->time_zone, + &location->parent_location_id, + &is_valid); + if(ret!=18) + { + free(location); + location=NULL; + + MESA_handle_runtime_log(logger, + RLOG_LV_FATAL, + "IP_LOCATION", + "Parse ip location failed, ret: %d table_id: %d table_line: %s", + ret, + table_id, + table_line + ); + return; + } + + atomic_inc(&location->ref_cnt); + location->table_id=table_id; + *ad=(MAAT_PLUGIN_EX_DATA)location; + + MESA_handle_runtime_log(logger, + RLOG_LV_DEBUG, + "IP_LOCATION", + "Add ip location: [ start_ip: %s end_ip: %s continent: %s country: %s city: %s ref_cnt: %d ] table_id: %d table_line: %s", + location->start_ip, + location->end_ip, + location->continent_full, + location->country_full, + location->city_full, + location->ref_cnt, + table_id, + table_line + ); + return; +} + +void location_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp) +{ + void *logger=argp; + struct _location_info *location=(struct _location_info *)(*ad); + + atomic_dec(&location->ref_cnt); + if(location->ref_cnt<=0) + { + free(*ad); + *ad=NULL; + } + + MESA_handle_runtime_log(logger, + RLOG_LV_DEBUG, + "IP_LOCATION", + "Delete location: [ start_ip: %s end_ip: %s continent: %s country: %s city: %s ref_cnt: %d ] table_id: %d", + location->start_ip, + location->end_ip, + location->continent_full, + location->country_full, + location->city_full, + location->ref_cnt, + table_id + ); + return; +} + + void subscribe_id_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void* argp) { void *logger=argp; + struct _subscribe_id_info *subscribe_id=(struct _subscribe_id_info *)(*from); + + atomic_inc(&subscribe_id->ref_cnt); + *to=*from; - *to=calloc(1, strlen((char *)*from)+1); - memcpy(*to, *from, strlen((char *)*from)); + MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Dup subscribe_id: %s ref_cnt: %d table_id: %d", subscribe_id->subscribe_id, subscribe_id->ref_cnt, table_id); - MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Dup subscribe_id: %s table_id: %d", (char *)*to, table_id); return; } @@ -69,34 +276,39 @@ void subscribe_id_new_data(int table_id, const char* key, const char* table_line { void *logger=argp; int ret=0,id=0,type=0,is_valid=0; - char subscribe_id[256]={0}; char ip_addr[MAX_IPV6_ADDR_LEN]={0}; + struct _subscribe_id_info *subscribe_id=NULL; + + subscribe_id=(struct _subscribe_id_info *)calloc(1, sizeof(struct _subscribe_id_info)); - ret=sscanf(table_line, "%d\t%d\t%s\t%s\t%d", &id, &type, ip_addr, subscribe_id, &is_valid); + ret=sscanf(table_line, "%d\t%d\t%s\t%s\t%d", &id, &type, ip_addr, subscribe_id->subscribe_id, &is_valid); if(ret!=5) { + free(subscribe_id); + subscribe_id=NULL; + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "SUBSCRIBE_ID", - "Parse subscribe_id failed, ret: %d table_id: %d key: %s table_line: %s", + "Parse subscribe_id failed, ret: %d table_id: %d table_line: %s", ret, table_id, - key, table_line ); return; } - *ad=calloc(1, strlen(subscribe_id)+1); - memcpy(*ad, subscribe_id, strlen(subscribe_id)); + + atomic_inc(&subscribe_id->ref_cnt); + *ad=(MAAT_PLUGIN_EX_DATA)subscribe_id; + MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", - "Add subscribe_id: %s table_id: %d key: %s table_line: %s", - *ad, + "Add subscribe_id: %s table_id: %d table_line: %s", + subscribe_id->subscribe_id, table_id, - key, table_line ); return; @@ -105,9 +317,16 @@ void subscribe_id_new_data(int table_id, const char* key, const char* table_line void subscribe_id_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp) { void *logger=argp; - MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Delete subscribe_id: %s table_id: %d", (char *)*ad, table_id); - free(*ad); - *ad=NULL; + struct _subscribe_id_info *subscribe_id=(struct _subscribe_id_info *)(*ad); + + atomic_dec(&subscribe_id->ref_cnt); + if(subscribe_id->ref_cnt<=0) + { + free(*ad); + *ad=NULL; + } + + MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Delete subscribe_id: %s ref_cnt: %d table_id: %d", subscribe_id->subscribe_id, subscribe_id->ref_cnt, table_id); return; } @@ -223,7 +442,15 @@ int tsg_rule_init(const char* conffile, void *logger) MESA_load_profile_string_def(conffile, "MAAT", "SSL_SNI_TABLE", g_tsg_para.table_name[TABLE_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_FIELD_SSL_SNI"); MESA_load_profile_string_def(conffile, "MAAT", "DECYPTION_EXCLUSION_SSL_SNI", g_tsg_para.table_name[TABLE_EXCLUSION_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_DECYPTION_EXCLUSION_SSL_SNI"); - //init dynamic maat feather + MESA_load_profile_string_def(conffile, "MAAT", "IP_ASN_TABLE", g_tsg_para.table_name[TABLE_IP_ASN], _MAX_TABLE_NAME_LEN, "TSG_OBJ_IP_ASN"); + MESA_load_profile_string_def(conffile, "MAAT", "IP_LOCATION_TABLE", g_tsg_para.table_name[TABLE_IP_LOCATION], _MAX_TABLE_NAME_LEN, "TSG_OBJ_IP_LOCATION"); + + MESA_load_profile_string_def(conffile, "MAAT", "ASN_BUILT_IN_TABLE", g_tsg_para.table_name[TABLE_ASN_BUILT_IN], _MAX_TABLE_NAME_LEN, "TSG_IP_ASN_BUILT_IN"); + MESA_load_profile_string_def(conffile, "MAAT", "ASN_USER_DEFINED_TABLE", g_tsg_para.table_name[TABLE_ASN_USER_DEFINED], _MAX_TABLE_NAME_LEN, "TSG_IP_ASN_USER_DEFINED"); + MESA_load_profile_string_def(conffile, "MAAT", "LOCATION_BUILT_IN_TABLE", g_tsg_para.table_name[TABLE_LOCATION_BUILT_IN], _MAX_TABLE_NAME_LEN, "TSG_IP_LOCATION_BUILT_IN"); + MESA_load_profile_string_def(conffile, "MAAT", "LOCATION_USER_DEFINED_TABLE", g_tsg_para.table_name[TABLE_LOCATION_USER_DEFINED], _MAX_TABLE_NAME_LEN, "TSG_IP_LOCATION_USER_DEFINED"); + + //init static maat feather g_tsg_maat_feather=init_maat_feather(maat_conffile, (char *)"TSG_STATIC", (char *)"STATIC", logger); if(g_tsg_maat_feather==NULL) { @@ -245,6 +472,49 @@ int tsg_rule_init(const char* conffile, void *logger) } } + for(i=TABLE_ASN_USER_DEFINED; i<=TABLE_ASN_BUILT_IN; i++) + { + ret=Maat_ip_plugin_EX_register(g_tsg_maat_feather, + g_tsg_para.table_id[i], + ASN_new_data, + ASN_free_data, + ASN_dup_data, + 0, + logger); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, + "RULE_INIT", + "Maat_ip_plugin_EX_register failed, table_name: %s table_id: %d", + g_tsg_para.table_name[i], + g_tsg_para.table_id[i] + ); + return -1; + } + } + + + for(i=TABLE_LOCATION_USER_DEFINED; i<=TABLE_LOCATION_BUILT_IN; i++) + { + ret=Maat_ip_plugin_EX_register(g_tsg_maat_feather, + g_tsg_para.table_id[i], + location_new_data, + location_free_data, + location_dup_data, + 0, + logger); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, + "RULE_INIT", + "Maat_ip_plugin_EX_register failed, table_name: %s table_id: %d", + g_tsg_para.table_name[i], + g_tsg_para.table_id[i] + ); + return -1; + } + } + //init dynamic maat feather g_tsg_dynamic_maat_feather=init_maat_feather(maat_conffile, (char *)"TSG_DYNAMIC", (char *)"DYNAMIC", logger); if(g_tsg_maat_feather==NULL) @@ -327,8 +597,83 @@ int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_re return 0; } +int tsg_get_ip_asn(const struct streaminfo *a_stream, int table_id, MAAT_PLUGIN_EX_DATA* client_asn, MAAT_PLUGIN_EX_DATA* server_asn) +{ + struct ip_address dest_ip={0}, source_ip={0}; + + switch(a_stream->addr.addrtype) + { + case ADDR_TYPE_IPV4: + source_ip.ip_type=4; + source_ip.ipv4=a_stream->addr.tuple4_v4->saddr; + + dest_ip.ip_type=4; + dest_ip.ipv4=a_stream->addr.tuple4_v4->daddr; + break; + case ADDR_TYPE_IPV6: + source_ip.ip_type=6; + memcpy((char *)(source_ip.ipv6), a_stream->addr.tuple4_v6->saddr, IPV6_ADDR_LEN); + + dest_ip.ip_type=6; + memcpy((char *)(source_ip.ipv6), a_stream->addr.tuple4_v6->daddr, IPV6_ADDR_LEN); + break; + default: + return 0; + break; + } -int tsg_get_subscribe_id(const struct streaminfo *a_stream, char **source_subscribe_id, char **dest_subscribe_id) + if(*client_asn==NULL) + { + Maat_ip_plugin_get_EX_data(g_tsg_maat_feather, table_id, &source_ip, client_asn, 1); + } + + if(*server_asn==NULL) + { + Maat_ip_plugin_get_EX_data(g_tsg_maat_feather, table_id, &dest_ip, server_asn, 1); + } + + return 0; +} + + +int tsg_get_ip_location(const struct streaminfo *a_stream, int table_id, MAAT_PLUGIN_EX_DATA *client_location, MAAT_PLUGIN_EX_DATA *server_location) +{ + struct ip_address dest_ip={0}, source_ip={0}; + + switch(a_stream->addr.addrtype) + { + case ADDR_TYPE_IPV4: + source_ip.ip_type=4; + source_ip.ipv4=a_stream->addr.tuple4_v4->saddr; + + dest_ip.ip_type=4; + dest_ip.ipv4=a_stream->addr.tuple4_v4->daddr; + break; + case ADDR_TYPE_IPV6: + source_ip.ip_type=6; + memcpy((char *)(source_ip.ipv6), a_stream->addr.tuple4_v6->saddr, IPV6_ADDR_LEN); + + dest_ip.ip_type=6; + memcpy((char *)(source_ip.ipv6), a_stream->addr.tuple4_v6->daddr, IPV6_ADDR_LEN); + break; + default: + return 0; + break; + } + + if(*client_location==NULL) + { + Maat_ip_plugin_get_EX_data(g_tsg_maat_feather, table_id, &source_ip, client_location, 1); + } + if(*server_location==NULL) + { + Maat_ip_plugin_get_EX_data(g_tsg_maat_feather, table_id, &dest_ip, server_location, 1); + } + + return 0; +} + +int tsg_get_subscribe_id(const struct streaminfo *a_stream, struct _subscribe_id_info **source_subscribe_id, struct _subscribe_id_info **dest_subscribe_id) { char source_ip[MAX_IPV6_ADDR_LEN]={0}; char dest_ip[MAX_IPV6_ADDR_LEN]={0}; @@ -351,29 +696,141 @@ int tsg_get_subscribe_id(const struct streaminfo *a_stream, char **source_subscr break; } - if(strlen(dest_ip)>0) + if(strlen(dest_ip)>0 && *dest_subscribe_id==NULL) { - *dest_subscribe_id = (char*)Maat_plugin_get_EX_data(g_tsg_dynamic_maat_feather, g_tsg_para.dyn_subscribe_ip_table_id, dest_ip); + *dest_subscribe_id = (struct _subscribe_id_info *)Maat_plugin_get_EX_data(g_tsg_dynamic_maat_feather, g_tsg_para.dyn_subscribe_ip_table_id, dest_ip); } - if(strlen(source_ip)>0) + if(strlen(source_ip)>0 && *source_subscribe_id==NULL) { - *source_subscribe_id = (char*)Maat_plugin_get_EX_data(g_tsg_dynamic_maat_feather, g_tsg_para.dyn_subscribe_ip_table_id, source_ip); + *source_subscribe_id = (struct _subscribe_id_info *)Maat_plugin_get_EX_data(g_tsg_dynamic_maat_feather, g_tsg_para.dyn_subscribe_ip_table_id, source_ip); } return 0; } +int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct _asn_info *asn, scan_status_t *mid, Maat_rule_t*result, int result_num) +{ + int ret=0; + + if(asn!=NULL) + { + ret=Maat_full_scan_string(maat_feather, + g_tsg_para.table_id[TABLE_IP_ASN], + CHARSET_GBK, + asn->asn, + strlen(asn->asn), + result, + NULL, + result_num, + mid, + a_stream->threadnum); + if(ret > 0) + { + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_DEBUG, + "SCAN_IP_ASN", + "Hit IP_ASN: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s", + asn->asn, + ret, + result[0].config_id, + result[0].service_id, + (unsigned char)result[0].action, + printaddr(&a_stream->addr, a_stream->threadnum) + ); + } + else + { + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_DEBUG, + "SCAN_IP_ASN", + "No hit IP_ASN: %s scan ret: %d addr: %s", + asn->asn, + ret, + printaddr(&a_stream->addr, a_stream->threadnum) + ); + } + } + else + { + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_DEBUG, + "SCAN_IP_ASN", + "IP_ASN is NULL scan ret: %d addr: %s", + ret, + printaddr(&a_stream->addr, a_stream->threadnum) + ); + } + + return ret; +} + + +int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct _location_info *location, scan_status_t *mid, Maat_rule_t*result, int result_num) +{ + int ret=0; + + if(location!=NULL) + { + ret=Maat_full_scan_string(maat_feather, + g_tsg_para.table_id[TABLE_IP_LOCATION], + CHARSET_GBK, + location->country_full, + strlen(location->country_full), + result, + NULL, + result_num, + mid, + a_stream->threadnum); + if(ret > 0) + { + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_DEBUG, + "SCAN_IP_LOCATION", + "Hit IP_LOCATION: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s", + location->country_full, + ret, + result[0].config_id, + result[0].service_id, + (unsigned char)result[0].action, + printaddr(&a_stream->addr, a_stream->threadnum) + ); + } + else + { + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_DEBUG, + "SCAN_IP_LOCATION", + "No hit IP_LOCATION: %s scan ret: %d addr: %s", + location->country_full, + ret, + printaddr(&a_stream->addr, a_stream->threadnum) + ); + } + } + else + { + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_DEBUG, + "SCAN_IP_LOCATION", + "IP_LOCATION is NULL scan ret: %d addr: %s", + ret, + printaddr(&a_stream->addr, a_stream->threadnum) + ); + } + + return ret; +} + int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num) { + int ret=0; struct ipaddr t_addr; struct ipaddr* p_addr=NULL; int hit_num=0,tans_proto=0; - char *source_subscribe_id=NULL; - char *dest_subscribe_id=NULL; int is_scan_addr=1, maat_ret=0,found_pos=0; const struct streaminfo *cur_stream = a_stream; - + struct _internal_label *internal_label=NULL; if(result==NULL || result_num<=0 || a_stream==NULL || maat_feather==NULL) { @@ -464,7 +921,6 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo * }while(cur_stream != NULL && hit_num < result_num); - if(hit_numPROTO_UNKONWN && protoclient_location), (void **)&(internal_label->server_location)); + tsg_get_ip_location(a_stream, g_tsg_para.table_id[TABLE_LOCATION_BUILT_IN], (void **)&(internal_label->client_location), (void **)&(internal_label->server_location)); - if(source_subscribe_id!=NULL) + hit_num+=tsg_scan_ip_location(maat_feather, a_stream, internal_label->client_location, mid, result+hit_num, result_num-hit_num); + hit_num+=tsg_scan_ip_location(maat_feather, a_stream, internal_label->server_location, mid, result+hit_num, result_num-hit_num); + } + + if(hit_numclient_asn), (void **)&(internal_label->server_asn)); + tsg_get_ip_asn(a_stream, g_tsg_para.table_id[TABLE_ASN_BUILT_IN], (void **)&(internal_label->client_asn), (void **)&(internal_label->server_asn)); + + hit_num+=tsg_scan_ip_asn(maat_feather, a_stream, internal_label->client_asn, mid, result+hit_num, result_num-hit_num); + hit_num+=tsg_scan_ip_asn(maat_feather, a_stream, internal_label->server_asn, mid, result+hit_num, result_num-hit_num); + } + + + if(hit_numclient_subscribe_id, &internal_label->server_subscribe_id); + + if(internal_label->client_subscribe_id!=NULL) { maat_ret=Maat_full_scan_string(maat_feather, g_tsg_para.table_id[TABLE_SUBSCRIBER_ID], CHARSET_GBK, - source_subscribe_id, - strlen(source_subscribe_id), + internal_label->client_subscribe_id->subscribe_id, + strlen(internal_label->client_subscribe_id->subscribe_id), result+hit_num, &found_pos, result_num-hit_num, @@ -529,7 +1010,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo * RLOG_LV_DEBUG, "SCAN_SUBSCRIBER", "Hit source subscribe id: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s", - source_subscribe_id, + internal_label->client_subscribe_id->subscribe_id, maat_ret, result[hit_num].config_id, result[hit_num].service_id, @@ -545,22 +1026,20 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo * RLOG_LV_DEBUG, "SCAN_SUBSCRIBER", "No hit source subscribe id: %s scan ret: %d addr: %s", - source_subscribe_id, + internal_label->client_subscribe_id->subscribe_id, maat_ret, printaddr(&a_stream->addr, a_stream->threadnum) ); } - - subscribe_id_free_data(g_tsg_para.dyn_subscribe_ip_table_id,(MAAT_PLUGIN_EX_DATA *)&source_subscribe_id, 0, g_tsg_para.logger); } - if(dest_subscribe_id!=NULL) + if(internal_label->server_subscribe_id!=NULL) { maat_ret=Maat_full_scan_string(maat_feather, g_tsg_para.table_id[TABLE_SUBSCRIBER_ID], CHARSET_GBK, - dest_subscribe_id, - strlen(dest_subscribe_id), + internal_label->server_subscribe_id->subscribe_id, + strlen(internal_label->server_subscribe_id->subscribe_id), result+hit_num, &found_pos, result_num-hit_num, @@ -572,7 +1051,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo * RLOG_LV_DEBUG, "SCAN_SUBSCRIBER", "Hit dest subscribe id: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s", - dest_subscribe_id, + internal_label->server_subscribe_id->subscribe_id, maat_ret, result[hit_num].config_id, result[hit_num].service_id, @@ -588,16 +1067,26 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo * RLOG_LV_DEBUG, "SCAN_SUBSCRIBER", "No hit dest subscribe id: %s scan ret: %d addr: %s", - dest_subscribe_id, + internal_label->server_subscribe_id->subscribe_id, maat_ret, printaddr(&a_stream->addr, a_stream->threadnum) ); } - - subscribe_id_free_data(g_tsg_para.dyn_subscribe_ip_table_id,(MAAT_PLUGIN_EX_DATA *)&dest_subscribe_id, 0, g_tsg_para.logger); } } + ret=project_req_add_struct((struct streaminfo *)a_stream, g_tsg_para.internal_project_id, (void *)internal_label); + if(ret<0) + { + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_FATAL, + "ADD_INTERNAL_LABEL", + "Add internal label failed, ret: %d addr: %s", + ret, + printaddr(&a_stream->addr, a_stream->threadnum) + ); + } + return hit_num; } diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 58f24e4..1949fcf 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -19,7 +19,7 @@ #include "tsg_send_log.h" #include "tsg_send_log_internal.h" -char TSG_SEND_LOG_VERSION_20200427=0; +char TSG_SEND_LOG_VERSION_20200514=0; struct tsg_log_instance_t *g_tsg_log_instance; @@ -164,6 +164,46 @@ static int set_common_sub_action(struct TLD_handle_t *handle, char *field_name, return 0; } +int set_common_field_from_label(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream) +{ + char buff[512]={0}; + struct _location_info *location=NULL; + struct _internal_label *internal_label=NULL; + + internal_label=(struct _internal_label *)project_req_get_struct(a_stream, _instance->internal_project_id); + if(internal_label!=NULL) + { + TLD_append(_handle, _instance->id2field[LOG_COMMON_ESTABLISH_LATENCY_MS].name, (void *)internal_label->establish_latency_ms, TLD_TYPE_LONG); + + if(internal_label->client_asn!=NULL) + { + snprintf(buff, sizeof(buff), "%s(%s)", internal_label->client_asn->asn, internal_label->client_asn->organization); + TLD_append(_handle, _instance->id2field[LOG_COMMON_CLINET_ASN].name, (void *)buff, TLD_TYPE_STRING); + } + + if(internal_label->server_asn!=NULL) + { + snprintf(buff, sizeof(buff), "%s(%s)", internal_label->server_asn->asn, internal_label->server_asn->organization); + TLD_append(_handle, _instance->id2field[LOG_COMMON_CLINET_ASN].name, (void *)buff, TLD_TYPE_STRING); + } + + if(internal_label->client_location!=NULL) + { + location=internal_label->client_location; + snprintf(buff, sizeof(buff), "%s%s%s%s%s", location->country_full, (strlen(location->province_full)>0) ? "/" : "", location->province_full, (strlen(location->city_full)>0) ? "/" : "", location->city_full); + TLD_append(_handle, _instance->id2field[LOG_COMMON_CLINET_LOCATION].name, (void *)buff, TLD_TYPE_STRING); + } + + if(internal_label->server_location!=NULL) + { + location=internal_label->server_location; + snprintf(buff, sizeof(buff), "%s%s%s%s%s", location->country_full, (strlen(location->province_full)>0) ? "/" : "", location->province_full, (strlen(location->city_full)>0) ? "/" : "", location->city_full); + TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_LOCATION].name, (void *)buff, TLD_TYPE_STRING); + } + } + + return 0; +} int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, struct streaminfo *a_stream) { @@ -468,10 +508,9 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl { int i=0,status=0; char *payload=NULL; - long establish_latency_ms=0; struct TLD_handle_t *_handle=handle; struct tsg_log_instance_t *_instance=instance; - + if(_instance==NULL || _handle==NULL || log_msg==NULL) { TLD_cancel(handle); @@ -488,6 +527,8 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl TLD_append_streaminfo(instance, handle, log_msg->a_stream); TLD_append(_handle, _instance->id2field[LOG_COMMON_SLED_IP].name, (void *)(_instance->local_ip_str), TLD_TYPE_STRING); + + set_common_field_from_label(_instance, _handle, log_msg->a_stream); for(i=0;iresult_num; i++) { @@ -511,15 +552,6 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl break; } - if(log_msg->result[i].action==TSG_ACTION_MONITOR && _instance->establish_latency_project_id>0) - { - establish_latency_ms=project_req_get_long(log_msg->a_stream, _instance->establish_latency_project_id); - if(establish_latency_ms>=0) - { - TLD_append(_handle, _instance->id2field[LOG_COMMON_ESTABLISH_LATENCY_MS].name, (void *)establish_latency_ms, TLD_TYPE_LONG); - } - } - TLD_append(_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name, (void *)(long)(log_msg->result[i].config_id), TLD_TYPE_LONG); TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVICE].name, (void *)(long)(log_msg->result[i].service_id), TLD_TYPE_LONG); TLD_append(_handle, _instance->id2field[LOG_COMMON_ACTION].name, (void *)(long)((unsigned char)log_msg->result[i].action), TLD_TYPE_LONG); diff --git a/src/tsg_send_log_internal.h b/src/tsg_send_log_internal.h index f271530..4aa55e3 100644 --- a/src/tsg_send_log_internal.h +++ b/src/tsg_send_log_internal.h @@ -61,6 +61,10 @@ typedef enum _tsg_log_field_id LOG_SSL_SNI, LOG_COMMON_ESTABLISH_LATENCY_MS, LOG_COMMON_SUB_ACTION, + LOG_COMMON_CLINET_ASN, + LOG_COMMON_SERVER_ASN, + LOG_COMMON_CLINET_LOCATION, + LOG_COMMON_SERVER_LOCATION, LOG_COMMON_MAX }tsg_log_field_id_t; @@ -81,7 +85,7 @@ struct tsg_log_instance_t { int mode; int max_service; - int establish_latency_project_id; + int internal_project_id; void *logger; char common_field_file[MAX_STRING_LEN*4]; char broker_list[MAX_STRING_LEN*4];