diff --git a/bin/main.conf b/bin/main.conf index 4dbfd90..f95ac68 100644 --- a/bin/main.conf +++ b/bin/main.conf @@ -1,6 +1,6 @@ [MAAT] PROFILE=./tsgconf/maat.conf -IP_ADDR_TABLE=TSG_OBJ_IP_ADDR +IP_ADDR_TABLE=TSG_SECURITY_ADDR SUBSCRIBER_ID_TABLE=TSG_OBJ_SUBSCRIBER_ID CB_SUBSCRIBER_IP_TABLE=TSG_DYN_SUBSCRIBER_IP diff --git a/bin/tsg_log_field.conf b/bin/tsg_log_field.conf index 310485f..7a40328 100644 --- a/bin/tsg_log_field.conf +++ b/bin/tsg_log_field.conf @@ -41,3 +41,4 @@ STRING common_schema_type 34 STRING http_host 35 STRING ssl_sni 36 LONG common_establish_latency_ms 37 +STRING common_sub_action 38 diff --git a/bin/tsg_static_tableinfo.conf b/bin/tsg_static_tableinfo.conf index d285290..ba7b0c9 100644 --- a/bin/tsg_static_tableinfo.conf +++ b/bin/tsg_static_tableinfo.conf @@ -43,3 +43,5 @@ 31 TSG_SECURITY_SOURCE_ADDR virtual TSG_OBJ_IP_ADDR -- 32 TSG_SECURITY_DESTINATION_ADDR virtual TSG_OBJ_IP_ADDR -- 33 TSG_SECURITY_ADDR composition {"source":"TSG_SECURITY_SOURCE_ADDR","destination":"TSG_SECURITY_DESTINATION_ADDR"} +34 TSG_DECYPTION_EXCLUSION_SSL_SNI virtual TSG_OBJ_FQDN +35 TSG_PROFILE_RESPONSE_PAGES plugin {"key":1,"foreign":"4","valid":5} -- diff --git a/inc/tsg_rule.h b/inc/tsg_rule.h index 5accc0a..b34b423 100644 --- a/inc/tsg_rule.h +++ b/inc/tsg_rule.h @@ -18,6 +18,7 @@ enum TSG_ETHOD_TYPE TSG_METHOD_TYPE_REDIRECTION, TSG_METHOD_TYPE_BLOCK, TSG_METHOD_TYPE_RESET, + TSG_METHOD_TYPE_ALERT, TSG_METHOD_TYPE_MAX }; diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index 41b3192..32ae0f7 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -41,7 +41,7 @@ static __attribute__((__used__)) const char * GIT_VERSION_UNKNOWN = NULL; #endif -char TSG_MASTER_VERSION_20200330=0; +char TSG_MASTER_VERSION_20200430=0; const char *tsg_conffile="tsgconf/main.conf"; g_tsg_para_t g_tsg_para; @@ -291,24 +291,70 @@ static int master_send_log(struct streaminfo *a_stream, struct Maat_rule_t *p_re return 1; } -static struct Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int result_num) +static struct Maat_rule_t *tsg_policy_decision_criteria(struct streaminfo *a_stream, Maat_rule_t *result, int result_num, struct _identify_info *identify_info, int thread_seq) { - int i=0; + int i=0,ret=0; + Maat_rule_t tmp_result; Maat_rule_t *p_result=NULL; + Maat_rule_t *q_result=NULL; + scan_status_t mid=NULL; + if(result==NULL || result_num<=0) { return NULL; } - +#if 0 + ret=Maat_rule_sort_by_evaluation_order(g_tsg_maat_feather, result, result_num); + if(ret!=result_num) + { + return NULL; + } + p_result=&result[0]; + if(p_result->action==TSG_ACTION_INTERCEPT && domain!=NULL && domain_len>0) + { + ret=Maat_full_scan_string(g_tsg_maat_feather, + g_tsg_para.table_id[TABLE_EXCLUSION_SSL_SNI], + CHARSET_UTF8, + domain, + domain_len, + &tmp_result, + NULL, + 1, + &mid, + thread_seq); + if(ret>0) + { + p_result=NULL; + for(i=1; iaction==TSG_ACTION_INTERCEPT) + { + continue; + } + p_result=&result[i]; + } + } + } + + return p_result; +#endif + p_result=&result[0]; + for(i=1; i(unsigned char)p_result->action) { + q_result=p_result; p_result=&result[i]; continue; } + + if(q_result==NULL) + { + q_result=&result[i]; + } if(result[i].action==p_result->action) { @@ -319,6 +365,49 @@ static struct Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int } } + if(p_result->action==TSG_ACTION_INTERCEPT && identify_info!=NULL && identify_info->domain_len>0) + { + ret=Maat_full_scan_string(g_tsg_maat_feather, + g_tsg_para.table_id[TABLE_EXCLUSION_SSL_SNI], + CHARSET_UTF8, + identify_info->domain, + identify_info->domain_len, + &tmp_result, + NULL, + 1, + &mid, + thread_seq); + + if(ret>0) + { + p_result=q_result; + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_DEBUG, + "EXCLUSION_SSL_SNI", + "Hit %s policy_id: %d service: %d action: %d addr: %s", + identify_info->domain, + tmp_result.config_id, + tmp_result.service_id, + (unsigned char)tmp_result.action, + printaddr(&a_stream->addr, thread_seq) + ); + + master_send_log(a_stream, &tmp_result, 1, identify_info, thread_seq); + } + else + { + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_DEBUG, + "EXCLUSION_SSL_SNI", + "Not hit %s stream_dir: %d addr: %s scan ret: %d", + identify_info->domain, + a_stream->dir, + printaddr(&a_stream->addr, thread_seq), + ret + ); + } + } + return p_result; } @@ -467,7 +556,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t if(ret>0) { hit_num+=ret; - q_result=tsg_policy_decision_criteria(all_result, hit_num); + q_result=tsg_policy_decision_criteria(a_tcp, all_result, hit_num, NULL, thread_seq); FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_ADDR], 0, FS_OP_ADD, 1); } @@ -510,7 +599,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t mid=NULL; } - p_result=tsg_policy_decision_criteria(all_result, hit_num); + p_result=tsg_policy_decision_criteria(a_tcp, all_result, hit_num, &identify_info, thread_seq); if(p_result!=NULL) { @@ -543,6 +632,16 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t memcpy(_context->domain, identify_info.domain, identify_info.domain_len); _context->domain_len=identify_info.domain_len; } + + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_DEBUG, + "MONITOR", + "Hit monitor policy, policy_id: %d service: %d action: %d addr: %s", + p_result[0].config_id, + p_result[0].service_id, + (unsigned char)p_result[0].action, + printaddr(&a_tcp->addr, thread_seq) + ); state=APP_STATE_GIVEME; } break; @@ -554,6 +653,17 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t memcpy(_context->domain, identify_info.domain, identify_info.domain_len); _context->domain_len=identify_info.domain_len; } + + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_DEBUG, + "ALLOW", + "Hit allow policy, policy_id: %d service: %d action: %d addr: %s", + p_result[0].config_id, + p_result[0].service_id, + (unsigned char)p_result[0].action, + printaddr(&a_tcp->addr, thread_seq) + ); + state=APP_STATE_GIVEME|APP_STATE_KILL_OTHER; FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_BYPASS], 0, FS_OP_ADD, 1); break; @@ -651,7 +761,7 @@ extern "C" char TSG_MASTER_UDP_ENTRY(struct streaminfo *a_udp, void **pme, int t Maat_clean_status(&mid); mid=NULL; } - p_result=tsg_policy_decision_criteria(result, ret); + p_result=tsg_policy_decision_criteria(a_udp, result, ret, NULL, thread_seq); if(p_result!=NULL) { switch((unsigned char)p_result->action) @@ -732,7 +842,13 @@ extern "C" int TSG_MASTER_INIT() g_tsg_para.priority_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_STRUCT, free_policy_label); if(g_tsg_para.priority_project_id<0) { - MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_REGISTER", "Register %s failed ...", label_buff); + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_FATAL, + "PROJECT_REGISTER", + "Register %s failed; please check :%s and add ", + label_buff, + "etc/project_list.conf" + ); return -1; } @@ -740,7 +856,13 @@ extern "C" int TSG_MASTER_INIT() g_tsg_para.establish_latency_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_LONG, NULL); if(g_tsg_para.establish_latency_project_id<0) { - MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_REGISTER", "Register %s failed ...", label_buff); + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_FATAL, + "PROJECT_REGISTER", + "Register %s failed; please check :%s and add ", + label_buff, + "etc/project_list.conf" + ); } ret=tsg_rule_init(tsg_conffile, g_tsg_para.logger); diff --git a/src/tsg_entry.h b/src/tsg_entry.h index 72f4b12..c0e6ff8 100644 --- a/src/tsg_entry.h +++ b/src/tsg_entry.h @@ -16,6 +16,7 @@ enum MASTER_TABLE{ TABLE_APP_ID, TABLE_HTTP_HOST, TABLE_SSL_SNI, + TABLE_EXCLUSION_SSL_SNI, TABLE_MAX }; diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index e4d96a6..ee66467 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -34,7 +34,8 @@ const struct _str2index method2index[TSG_METHOD_TYPE_MAX]={ {TSG_METHOD_TYPE_UNK {TSG_METHOD_TYPE_DROP, 4, (char *)"drop"}, {TSG_METHOD_TYPE_REDIRECTION, 8, (char *)"redirect"}, {TSG_METHOD_TYPE_BLOCK, 5, (char *)"block"}, - {TSG_METHOD_TYPE_RESET, 3, (char *)"rst"} + {TSG_METHOD_TYPE_RESET, 3, (char *)"rst"}, + {TSG_METHOD_TYPE_ALERT, 5, (char *)"alert"} }; const struct _str2index g_tsg_proto_string[PROTO_MAX+1]={{PROTO_UNKONWN, 0, (char *)""}, @@ -161,7 +162,7 @@ static Maat_feather_t init_maat_feather(const char* conffile, char* instance_nam Maat_set_feather_opt(_maat_feather, MAAT_OPT_PERF_ON, NULL, 0); Maat_set_feather_opt(_maat_feather, MAAT_OPT_REDIS_INDEX, &redis_index, sizeof(redis_index)); Maat_set_feather_opt(_maat_feather, MAAT_OPT_SCAN_DETAIL, &scan_detail, sizeof(scan_detail)); - //Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_DEFERRED_LOAD, NULL,0); + Maat_set_feather_opt(_maat_feather, MAAT_OPT_FOREIGN_CONT_DIR, "./alerts_files", strlen("./alerts_files")+1); } else { @@ -215,11 +216,12 @@ int tsg_rule_init(const char* conffile, void *logger) char cb_subscriber_ip_table[32]={0}; MESA_load_profile_string_def(conffile, "MAAT", "PROFILE", maat_conffile, sizeof(maat_conffile), "./tsgconf/maat_profile.conf"); - MESA_load_profile_string_def(conffile, "MAAT", "IP_ADDR_TABLE", g_tsg_para.table_name[TABLE_IP_ADDR], _MAX_TABLE_NAME_LEN, "TSG_OBJ_IP_ADDR"); + MESA_load_profile_string_def(conffile, "MAAT", "IP_ADDR_TABLE", g_tsg_para.table_name[TABLE_IP_ADDR], _MAX_TABLE_NAME_LEN, "TSG_SECURITY_ADDR"); MESA_load_profile_string_def(conffile, "MAAT", "SUBSCRIBER_ID_TABLE", g_tsg_para.table_name[TABLE_SUBSCRIBER_ID], _MAX_TABLE_NAME_LEN, "TSG_OBJ_SUBSCRIBER_ID"); MESA_load_profile_string_def(conffile, "MAAT", "APP_ID_TABLE", g_tsg_para.table_name[TABLE_APP_ID], _MAX_TABLE_NAME_LEN, "TSG_OBJ_APP_ID"); MESA_load_profile_string_def(conffile, "MAAT", "HTTP_HOST_TABLE", g_tsg_para.table_name[TABLE_HTTP_HOST], _MAX_TABLE_NAME_LEN, "TSG_FIELD_HTTP_HOST"); MESA_load_profile_string_def(conffile, "MAAT", "SSL_SNI_TABLE", g_tsg_para.table_name[TABLE_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_FIELD_SSL_SNI"); + MESA_load_profile_string_def(conffile, "MAAT", "DECYPTION_EXCLUSION_SSL_SNI", g_tsg_para.table_name[TABLE_EXCLUSION_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_DECYPTION_EXCLUSION_SSL_SNI"); //init dynamic maat feather g_tsg_maat_feather=init_maat_feather(maat_conffile, (char *)"TSG_STATIC", (char *)"STATIC", logger); @@ -234,7 +236,11 @@ int tsg_rule_init(const char* conffile, void *logger) g_tsg_para.table_id[i]=Maat_table_register(g_tsg_maat_feather, g_tsg_para.table_name[i]); if(g_tsg_para.table_id[i]<0) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "Maat_table_register %s failed, Please check tsgconf/tsg_static_tableinfo.conf", g_tsg_para.table_name[i]); + MESA_handle_runtime_log(logger, + RLOG_LV_FATAL, + "Maat_table_register %s failed, Please check tsgconf/tsg_static_tableinfo.conf", + g_tsg_para.table_name[i] + ); return -1; } } @@ -252,7 +258,12 @@ int tsg_rule_init(const char* conffile, void *logger) g_tsg_para.dyn_subscribe_ip_table_id=Maat_table_register(g_tsg_dynamic_maat_feather, cb_subscriber_ip_table); if(g_tsg_para.dyn_subscribe_ip_table_id<0) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "RULE_INIT", "Maat_table_register %s failed", cb_subscriber_ip_table); + MESA_handle_runtime_log(logger, + RLOG_LV_FATAL, + "RULE_INIT", + "Maat_table_register %s failed, Please check tsgconf/tsg_static_tableinfo.conf", + cb_subscriber_ip_table + ); return -1; } ret=Maat_plugin_EX_register(g_tsg_dynamic_maat_feather, diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 8bb1a84..58f24e4 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -19,7 +19,7 @@ #include "tsg_send_log.h" #include "tsg_send_log_internal.h" -char TSG_SEND_LOG_VERSION_20200330=0; +char TSG_SEND_LOG_VERSION_20200427=0; struct tsg_log_instance_t *g_tsg_log_instance; @@ -129,6 +129,42 @@ char *log_field_id2name(struct tsg_log_instance_t *instance, tsg_log_field_id_t return NULL; } +static int set_common_sub_action(struct TLD_handle_t *handle, char *field_name, struct Maat_rule_t *p_result) +{ + cJSON *item=NULL; + cJSON *object=NULL; + char *tmp_buff=NULL; + + if(p_result->serv_def_len<128) + { + object=cJSON_Parse(p_result->service_defined); + } + else + { + tmp_buff=(char *)calloc(1, p_result->serv_def_len+1); + Maat_read_rule(g_tsg_maat_feather, p_result, MAAT_RULE_SERV_DEFINE, tmp_buff, p_result->serv_def_len); + object=cJSON_Parse(tmp_buff); + + free(tmp_buff); + tmp_buff=NULL; + } + + if(object!=NULL) + { + item=cJSON_GetObjectItem(object, "method"); + if(item!=NULL && item->valuestring!=NULL) + { + TLD_append(handle, field_name, (void *)item->valuestring, TLD_TYPE_STRING); + } + + cJSON_Delete(object); + object=NULL; + } + + return 0; +} + + int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, struct streaminfo *a_stream) { int ret=0,addr_type=0; @@ -366,7 +402,12 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile) ret=MESA_get_dev_ipv4(nic_name, (int *)&local_ip_nr); if(ret<0) { - MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "GET_LOCAL_IP", "MESA_get_dev_ipv4 is error, nic_name: %s", nic_name); + MESA_handle_runtime_log(_instance->logger, + RLOG_LV_FATAL, + "GET_LOCAL_IP", + "MESA_get_dev_ipv4 is error, nic_name: %s, please check tsgconf/main.conf", + nic_name + ); return NULL; } inet_ntop(AF_INET,&(local_ip_nr),_instance->local_ip_str,sizeof(_instance->local_ip_str)); @@ -385,7 +426,12 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile) if(rd_kafka_brokers_add(kafka_handle, _instance->broker_list) == 0) { - MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "KAFKA_INIT", "rd_kafka_brokers_add is error, broker_list: %s", _instance->broker_list); + MESA_handle_runtime_log(_instance->logger, + RLOG_LV_FATAL, + "KAFKA_INIT", + "rd_kafka_brokers_add is error, broker_list: %s, please check tsgconf/main.conf", + _instance->broker_list + ); return NULL; } @@ -395,7 +441,7 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile) { _instance->topic_rkt=(rd_kafka_topic_t **)calloc(1, (_instance->max_service)*sizeof(rd_kafka_topic_t*)); - for(i=0; i<_instance->max_service+1; i++) + for(i=0; i<_instance->max_service; i++) { if(_instance->service2topic[i].type==TLD_TYPE_MAX) { @@ -406,7 +452,12 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile) } else { - MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "KAFKA_INIT", "load_log_common_field is error, please check %s", _instance->common_field_file); + MESA_handle_runtime_log(_instance->logger, + RLOG_LV_FATAL, + "KAFKA_INIT", + "load_log_common_field is error, please check %s", + _instance->common_field_file + ); } return _instance; @@ -472,7 +523,12 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl TLD_append(_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name, (void *)(long)(log_msg->result[i].config_id), TLD_TYPE_LONG); TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVICE].name, (void *)(long)(log_msg->result[i].service_id), TLD_TYPE_LONG); TLD_append(_handle, _instance->id2field[LOG_COMMON_ACTION].name, (void *)(long)((unsigned char)log_msg->result[i].action), TLD_TYPE_LONG); - + + if(log_msg->result[i].action==TSG_ACTION_DENY) + { + set_common_sub_action(_handle, _instance->id2field[LOG_COMMON_SUB_ACTION].name, &(log_msg->result[i])); + } + payload = cJSON_PrintUnformatted(_handle->object); status = rd_kafka_produce(_instance->topic_rkt[log_msg->result[i].service_id], RD_KAFKA_PARTITION_UA, RD_KAFKA_MSG_F_COPY, payload, strlen(payload), NULL, 0, NULL); @@ -495,6 +551,7 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl TLD_delete(_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name); TLD_delete(_handle, _instance->id2field[LOG_COMMON_SERVICE].name); TLD_delete(_handle, _instance->id2field[LOG_COMMON_ACTION].name); + TLD_delete(_handle, _instance->id2field[LOG_COMMON_SUB_ACTION].name); TLD_delete(_handle, _instance->id2field[LOG_COMMON_ESTABLISH_LATENCY_MS].name); FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_LOG], 0, FS_OP_ADD, 1); diff --git a/src/tsg_send_log_internal.h b/src/tsg_send_log_internal.h index bf94d6c..6f031a8 100644 --- a/src/tsg_send_log_internal.h +++ b/src/tsg_send_log_internal.h @@ -60,6 +60,7 @@ typedef enum _tsg_log_field_id LOG_HTTP_HOST, LOG_SSL_SNI, LOG_COMMON_ESTABLISH_LATENCY_MS, + LOG_COMMON_SUB_ACTION, LOG_COMMON_MAX }tsg_log_field_id_t;