diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 0debece..95088c0 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -1135,7 +1135,7 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id { return NULL; } - table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL); table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key); @@ -1227,7 +1227,7 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc return 0; } - table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_IP_PLUGIN, NULL); + table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_IP_PLUGIN, NULL); table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); if(table_rt->table_type!=TABLE_TYPE_IP_PLUGIN) { @@ -1273,7 +1273,7 @@ int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const cha { return 0; } - table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_FQDN_PLUGIN, NULL); + table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_FQDN_PLUGIN, NULL); table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN) { @@ -1328,7 +1328,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id return 0; } - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id); + p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1466,7 +1466,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id _mid->scan_cnt++; int virtual_table_id=0; - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_INTERVAL, &virtual_table_id); + p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_INTERVAL, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1568,7 +1568,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id _mid->scan_cnt++; int virtual_table_id=0; - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id); + p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1632,7 +1632,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id return compile_ret; } -static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, unsigned short int proto, enum MAAT_TABLE_CHILD_TYPE child_type, int table_id) +static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, unsigned short int proto, enum MAAT_TABLE_COMPONENT_TYPE child_type, int table_id) { memset(scan_data, 0, sizeof(scan_data_t)); scan_data->sub_type=make_sub_type(table_id, CHARSET_NONE, 0); @@ -1643,15 +1643,15 @@ static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, u scan_data->ipv4_data.proto=proto; switch(child_type) { - case CHILD_TABLE_TYPE_SOURCE_IP: + case COMPONENT_TABLE_TYPE_SOURCE_IP: scan_data->ipv4_data.saddr=ntohl(addr->v4->saddr); scan_data->ipv4_data.sport=ntohs(addr->v4->source); break; - case CHILD_TABLE_TYPE_DESTINATION_IP: + case COMPONENT_TABLE_TYPE_DESTINATION_IP: scan_data->ipv4_data.saddr=ntohl(addr->v4->daddr); scan_data->ipv4_data.sport=ntohs(addr->v4->dest); break; - case CHILD_TABLE_TYPE_SESSION: + case COMPONENT_TABLE_TYPE_SESSION: scan_data->ipv4_data.saddr=ntohl(addr->v4->saddr); scan_data->ipv4_data.sport=ntohs(addr->v4->source); scan_data->ipv4_data.daddr=ntohl(addr->v4->daddr); @@ -1667,17 +1667,17 @@ static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, u scan_data->ipv6_data.proto=proto; switch(child_type) { - case CHILD_TABLE_TYPE_SOURCE_IP: + case COMPONENT_TABLE_TYPE_SOURCE_IP: memcpy(scan_data->ipv6_data.saddr, addr->v6->saddr, sizeof(scan_data->ipv6_data.saddr)); ipv6_ntoh(scan_data->ipv6_data.saddr); scan_data->ipv6_data.sport=ntohs(addr->v6->source); break; - case CHILD_TABLE_TYPE_DESTINATION_IP: + case COMPONENT_TABLE_TYPE_DESTINATION_IP: memcpy(scan_data->ipv6_data.saddr, addr->v6->daddr, sizeof(scan_data->ipv6_data.saddr)); ipv6_ntoh(scan_data->ipv6_data.saddr); scan_data->ipv6_data.sport=ntohs(addr->v6->dest); break; - case CHILD_TABLE_TYPE_SESSION: + case COMPONENT_TABLE_TYPE_SESSION: memcpy(scan_data->ipv6_data.saddr, addr->v6->saddr, sizeof(scan_data->ipv6_data.saddr)); ipv6_ntoh(scan_data->ipv6_data.saddr); scan_data->ipv6_data.sport=ntohs(addr->v6->source); @@ -1697,7 +1697,7 @@ static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, u } return 0; } -static int IP_composition_scan(const struct ipaddr* addr, unsigned short int proto, Maat_table_schema* parent_table, enum MAAT_TABLE_CHILD_TYPE child_type, +static int IP_composition_scan(const struct ipaddr* addr, unsigned short int proto, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE child_type, scan_result_t *region_result, unsigned int result_num, int* virtual_table_id, rule_scanner_t ip_scanner, struct Maat_table_manager* table_mgr, struct Maat_table_runtime_manager* table_rt_mgr, @@ -1705,21 +1705,21 @@ static int IP_composition_scan(const struct ipaddr* addr, unsigned short int pro { int child_table_id=0; - if(child_type==CHILD_TABLE_TYPE_NONE) + if(child_type==COMPONENT_TABLE_TYPE_NONE) { - child_table_id=parent_table->table_id; - child_type=CHILD_TABLE_TYPE_SESSION; + child_table_id=parent_table_id; + child_type=COMPONENT_TABLE_TYPE_SESSION; } else { - child_table_id=Maat_table_get_child_id(parent_table, child_type); + child_table_id=Maat_table_get_child_id(table_mgr, parent_table_id, child_type); } if(child_table_id<0) { return 0; } - Maat_table_schema* real_table=Maat_table_get_scan_by_id(table_mgr, child_table_id, TABLE_TYPE_IP, virtual_table_id); + Maat_table_schema* real_table=Maat_table_get_scan_by_id(table_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id); if(real_table==NULL) { return 0; @@ -1760,7 +1760,6 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id int region_ret=0, compile_ret=0; struct _OUTER_scan_status_t* _mid=NULL; scan_result_t *region_result=NULL; - Maat_table_schema* p_table=NULL; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct Maat_scanner* my_scanner=NULL; struct timespec start,end; @@ -1771,8 +1770,9 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id _mid=grab_mid(mid, _feather, thread_num, 0); _mid->scan_cnt++; int virtual_table_id=0; - p_table=Maat_table_get_by_id_raw(_feather->table_mgr, table_id); - if(p_table==NULL) + enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_INVALID; + table_type=Maat_table_get_type_by_id(_feather->table_mgr, table_id); + if(table_type==TABLE_TYPE_INVALID) { _feather->scan_err_cnt++; return -1; @@ -1791,12 +1791,12 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); INC_SCANNER_REF(my_scanner, thread_num); - if(p_table->table_type==TABLE_TYPE_COMPOSITION) + if(table_type==TABLE_TYPE_COMPOSITION) { - enum MAAT_TABLE_CHILD_TYPE childs[3]={CHILD_TABLE_TYPE_SOURCE_IP, CHILD_TABLE_TYPE_DESTINATION_IP, CHILD_TABLE_TYPE_SESSION}; + enum MAAT_TABLE_COMPONENT_TYPE childs[3]={COMPONENT_TABLE_TYPE_SOURCE_IP, COMPONENT_TABLE_TYPE_DESTINATION_IP, COMPONENT_TABLE_TYPE_SESSION}; for(int i=0; i<3; i++) { - region_ret=IP_composition_scan(addr, proto, p_table, childs[i], + region_ret=IP_composition_scan(addr, proto, table_id, childs[i], region_result+region_hit_cnt, MAX_SCANNER_HIT_NUM-region_hit_cnt, &virtual_table_id, my_scanner->region, _feather->table_mgr, _feather->scanner->table_rt_mgr, thread_num); if(region_ret<0) @@ -1814,7 +1814,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id } else { - region_ret=IP_composition_scan(addr, proto, p_table, CHILD_TABLE_TYPE_NONE, + region_ret=IP_composition_scan(addr, proto, table_id, COMPONENT_TABLE_TYPE_NONE, region_result+region_hit_cnt, MAX_SCANNER_HIT_NUM-region_hit_cnt, &virtual_table_id, my_scanner->region, _feather->table_mgr, _feather->scanner->table_rt_mgr, thread_num); if(region_ret<0) @@ -1834,7 +1834,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id _mid=grab_mid(mid, _feather, thread_num, 1); scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_hit_cnt, _mid->is_last_region, virtual_table_id, _mid->scan_cnt); - if(p_table->table_type==TABLE_TYPE_COMPOSITION) + if(table_type==TABLE_TYPE_COMPOSITION) { region_hit_wraper.virtual_table_ids=region_rslt_virtual_table_id; } @@ -1884,7 +1884,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, struct Maat_table_schema *p_table=NULL; int virtual_table_id=0; assert(thread_num<_feather->scan_thread_num); - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id); + p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -2172,7 +2172,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, sfh_instance_t * tmp_fuzzy_handle=NULL; struct Maat_table_schema *p_table=NULL; int virtual_table_id=0; - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_DIGEST, &virtual_table_id); + p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 21655e6..02ce3ca 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -1625,6 +1625,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m int update_type=CM_UPDATE_TYPE_INC; long long new_version=0; enum MAAT_TABLE_TYPE table_type; + enum MAAT_SCAN_TYPE scan_type; struct Maat_table_schema* table_schema=NULL; void* logger=feather->logger; @@ -1714,7 +1715,9 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m table_type=Maat_table_get_type_by_id(feather->table_mgr, table_id); if(rule_list[i].op==MAAT_OP_DEL) { - table_schema=Maat_table_get_scan_by_id(feather->table_mgr, table_id, table_type, NULL); + + scan_type=Maat_table_get_scan_type(table_type); + table_schema=Maat_table_get_scan_by_id(feather->table_mgr, table_id, scan_type, NULL); valid_column=Maat_table_xx_plugin_table_get_valid_flag_column(table_schema); ret=invalidate_line(rule_list[i].table_line, table_type, valid_column); if(ret<0) diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index e099b03..6b01597 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -34,7 +34,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_3_0_20200919=1; +int MAAT_FRAME_VERSION_3_0_20200927=1; int is_valid_table_name(const char* str) { diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table.cpp index 098bdbf..552eb94 100644 --- a/src/entry/Maat_table.cpp +++ b/src/entry/Maat_table.cpp @@ -18,7 +18,45 @@ struct Maat_table_manager MESA_htable_handle map_tablename2id; int active_plugin_table_num; int is_last_plugin_table_updating; + void* logger; }; +enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type) +{ + enum MAAT_SCAN_TYPE ret=SCAN_TYPE_INVALID; + switch(table_type) + { + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + case TABLE_TYPE_SIMILARITY: + case TABLE_TYPE_DIGEST: + ret=SCAN_TYPE_STRING; + break; + case TABLE_TYPE_INTERVAL: + case TABLE_TYPE_INTERVAL_PLUS: + ret=SCAN_TYPE_INTERVAL; + break; + case TABLE_TYPE_IP: + case TABLE_TYPE_IP_PLUS: + case TABLE_TYPE_COMPOSITION: + ret=SCAN_TYPE_IP; + break; + case TABLE_TYPE_PLUGIN: + ret=SCAN_TYPE_PLUGIN; + break; + case TABLE_TYPE_IP_PLUGIN: + ret=SCAN_TYPE_IP; + break; + case TABLE_TYPE_FQDN_PLUGIN: + ret=SCAN_TYPE_FQDN_PLUGIN; + break; + case TABLE_TYPE_COMPILE: + ret=SCAN_TYPE_NONE; + break; + default: + break; + } + return ret; +} int read_expr_table_info(const char* line, struct Maat_table_schema* table, MESA_htable_handle string2int_map) { @@ -74,25 +112,7 @@ int read_expr_table_info(const char* line, struct Maat_table_schema* table, MESA } return 0; } -int read_virtual_table_schema(const char* line, struct Maat_table_schema* table, MESA_htable_handle string2int_map) -{ - int ret=0; - char table_type[16]; - ret=sscanf(line, "%d\t%s\t%s\t%s", &(table->table_id), - table->table_name[0], - table_type, - table->virtual_table.real_table_name); - if(ret!=4) - { - return -1; - } - ret=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type)); - if(ret<0) - { - return -1; - } - return 0; -} + Maat_table_schema* table_info_new(void) { struct Maat_table_schema*p=ALLOC(struct Maat_table_schema, 1); @@ -121,6 +141,81 @@ int _read_integer_arrary(char* string, int *array, int size) #define COLUMN_IP_PLUGIN_SCHEMA_JSON 4 #define COLUMN_FQDN_PLUGIN_SHCEMA_JSON 4 #define COLUMN_COMPOSITION_SCHEMA_JSON 4 +#define COLUMN_VIRUTAL_SCHEMA_JSON 4 +int read_virtual_table_schema(struct Maat_table_manager* table_mgr, const char* line, struct Maat_table_schema* table, MESA_htable_handle reserved_word_map) +{ + int ret=0, tmp_table_id=0; + enum MAAT_TABLE_TYPE physical_table_type=TABLE_TYPE_INVALID; + enum MAAT_SCAN_TYPE physical_table_scan_type=SCAN_TYPE_INVALID; + cJSON* json=NULL, *tmp=NULL; + char *json_str; + + + size_t offset=0, len=0; + char* copy_line=NULL; + copy_line=_maat_strdup(line); + ret=get_column_pos(copy_line, COLUMN_VIRUTAL_SCHEMA_JSON, &offset, &len); + if(ret<0) + { + goto error_out; + } + if(offset+lentype!=cJSON_Array) + { + goto error_out; + } + cJSON_ArrayForEach(tmp, json) + { + if(tmp->type!=cJSON_String) + { + goto error_out; + } + ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &tmp_table_id); + if(ret<0) + { + goto error_out; + } + physical_table_type=table_mgr->p_table_info[tmp_table_id]->table_type; + physical_table_scan_type=Maat_table_get_scan_type(physical_table_type); + if(physical_table_scan_typevirtual_table.physical_table_id[physical_table_scan_type]=tmp_table_id; + } + } + else //For compatible non-json physical description + { + ret=map_str2int(table_mgr->map_tablename2id, json_str, &tmp_table_id); + if(ret<0) + { + goto error_out; + } + physical_table_type=table_mgr->p_table_info[tmp_table_id]->table_type; + physical_table_scan_type=Maat_table_get_scan_type(physical_table_type); + table->virtual_table.physical_table_id[physical_table_scan_type]=tmp_table_id; + } + cJSON_Delete(json); + free(copy_line); + return 0; + +error_out: + if(json) cJSON_Delete(json); + free(copy_line); + return -1; + +} int read_plugin_table_schema(const char* line, struct Maat_table_schema* p) { @@ -137,7 +232,7 @@ int read_plugin_table_schema(const char* line, struct Maat_table_schema* p) } if(offset+lentype==cJSON_String) { - strncpy(composition_schema->source_table.real_table_name, tmp->valuestring, sizeof(composition_schema->source_table.real_table_name)); + ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SOURCE_IP])); + if(ret<0) + { + MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module, + "Child table %s of table %s (id=%d) are not defined.", + tmp->valuestring, + p->table_name[0], + p->table_id); + goto error_out; + } } tmp=cJSON_GetObjectItem(json, "destination"); if(tmp!=NULL && tmp->type==cJSON_String) { - strncpy(composition_schema->destination_table.real_table_name, tmp->valuestring, sizeof(composition_schema->destination_table.real_table_name)); + ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_DESTINATION_IP])); + if(ret<0) + { + MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module, + "Child table %s of table %s (id=%d) are not defined.", + tmp->valuestring, + p->table_name[0], + p->table_id); + goto error_out; + } + } tmp=cJSON_GetObjectItem(json, "session"); if(tmp!=NULL && tmp->type==cJSON_String) { - strncpy(composition_schema->session_table.real_table_name, tmp->valuestring, sizeof(composition_schema->session_table.real_table_name)); + ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SESSION])); + if(ret<0) + { + MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module, + "Child table %s of table %s (id=%d) are not defined.", + tmp->valuestring, + p->table_name[0], + p->table_id); + goto error_out; + } } cJSON_Delete(json); - free(copy_line); return 0; + error_out: free(copy_line); return -1; } -static int Maat_table_build_map(struct Maat_table_manager* table_mgr, void* logger) -{ - struct Maat_table_schema** p_table_info=table_mgr->p_table_info; - size_t n_table=MAX_TABLE_NUM; - - MESA_htable_handle map_tablename2id=map_create(); - size_t i=0; - int j=0, ret=0; - for(i=0;itable_type) - { - case TABLE_TYPE_VIRTUAL: - ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id)); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Undefined real table %s, virtual table %s of table id %d.", - p_table_info[i]->virtual_table.real_table_name, - p_table_info[i]->table_name[0], - p_table_info[i]->table_id); - goto failed; - } - break; - case TABLE_TYPE_COMPOSITION: - if(strlen(p_table_info[i]->composition.source_table.real_table_name)>0) - { - ret=map_str2int(map_tablename2id, p_table_info[i]->composition.source_table.real_table_name, - &(p_table_info[i]->composition.source_table.real_table_id)); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Child table %s of table %s (id=%d) are not defined.", - p_table_info[i]->composition.source_table.real_table_name, - p_table_info[i]->table_name[0], - p_table_info[i]->table_id); - goto failed; - } - } - - if(strlen(p_table_info[i]->composition.destination_table.real_table_name)>0) - { - ret=map_str2int(map_tablename2id, p_table_info[i]->composition.destination_table.real_table_name, - &(p_table_info[i]->composition.destination_table.real_table_id)); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Child table %s of table %s (id=%d) are not defined.", - p_table_info[i]->composition.destination_table.real_table_name, - p_table_info[i]->table_name[0], - p_table_info[i]->table_id); - goto failed; - } - } - if(strlen(p_table_info[i]->composition.session_table.real_table_name)>0) - { - ret=map_str2int(map_tablename2id, p_table_info[i]->composition.session_table.real_table_name, - &(p_table_info[i]->composition.session_table.real_table_id)); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Child table %s of table %s (id=%d) are not defined.", - p_table_info[i]->composition.session_table.real_table_name, - p_table_info[i]->table_name[0], - p_table_info[i]->table_id); - goto failed; - } - } - default: - break; - } - - - for(j=0; jconj_cnt; j++) - { - ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Duplicate table %s of table id %d", - p_table_info[i]->table_name[j], - p_table_info[i]->table_id); - continue; - } - } - - } - table_mgr->map_tablename2id=map_tablename2id; - return 0; -failed: - map_destroy(map_tablename2id); - return -1; -} - void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr) { size_t i=0; @@ -532,7 +556,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path char line[MAX_TABLE_LINE_SIZE]; int i=0, ret=0; char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; - MESA_htable_handle string2int_map=NULL;; + MESA_htable_handle reserved_word_map=NULL;; struct Maat_table_schema*p=NULL; struct Maat_table_schema*conj_table=NULL; fp=fopen(table_info_path,"r"); @@ -546,38 +570,40 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path table_mgr=ALLOC(struct Maat_table_manager, 1); struct Maat_table_schema** p_table_info=table_mgr->p_table_info; size_t n_table=MAX_TABLE_NUM; + table_mgr->logger=logger; + table_mgr->map_tablename2id=map_create(); - string2int_map=map_create(); - map_register(string2int_map,"expr", TABLE_TYPE_EXPR); - map_register(string2int_map,"ip", TABLE_TYPE_IP); - map_register(string2int_map,"ip_plus", TABLE_TYPE_IP_PLUS); - map_register(string2int_map,"compile", TABLE_TYPE_COMPILE); - map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN); - map_register(string2int_map,"ip_plugin", TABLE_TYPE_IP_PLUGIN); - map_register(string2int_map,"fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN); - map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL); - map_register(string2int_map,"interval", TABLE_TYPE_INTERVAL); - map_register(string2int_map,"intval_plus", TABLE_TYPE_INTERVAL_PLUS); - map_register(string2int_map,"interval_plus", TABLE_TYPE_INTERVAL_PLUS); - map_register(string2int_map,"digest", TABLE_TYPE_DIGEST); - map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS); - map_register(string2int_map,"group", TABLE_TYPE_GROUP); - map_register(string2int_map,"group2group", TABLE_TYPE_GROUP2GROUP); - map_register(string2int_map,"group2compile", TABLE_TYPE_GROUP2COMPILE); - map_register(string2int_map,"similar", TABLE_TYPE_SIMILARITY); - map_register(string2int_map,"virtual", TABLE_TYPE_VIRTUAL); - map_register(string2int_map,"composition", TABLE_TYPE_COMPOSITION); - map_register(string2int_map,"quickoff", 0); - map_register(string2int_map,"quickon", 1); - map_register(string2int_map,"escape", USER_REGION_ENCODE_ESCAPE); -// map_register(string2int_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet + reserved_word_map=map_create(); + map_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); + map_register(reserved_word_map, "ip", TABLE_TYPE_IP); + map_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); + map_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE); + map_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); + map_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); + map_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN); + map_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL); + map_register(reserved_word_map, "interval", TABLE_TYPE_INTERVAL); + map_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS); + map_register(reserved_word_map, "interval_plus", TABLE_TYPE_INTERVAL_PLUS); + map_register(reserved_word_map, "digest", TABLE_TYPE_DIGEST); + map_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); + map_register(reserved_word_map, "group", TABLE_TYPE_GROUP); + map_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP); + map_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE); + map_register(reserved_word_map, "similar", TABLE_TYPE_SIMILARITY); + map_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); + map_register(reserved_word_map, "composition", TABLE_TYPE_COMPOSITION); + map_register(reserved_word_map, "quickoff", 0); + map_register(reserved_word_map, "quickon", 1); + map_register(reserved_word_map, "escape", USER_REGION_ENCODE_ESCAPE); +// map_register(reserved_word_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet const char** charset_name_list=charset_get_all_name(); for(i=0;i0) { - map_register(string2int_map, charset_name_list[i], i); + map_register(reserved_word_map, charset_name_list[i], i); } else { @@ -585,8 +611,8 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path } } - map_register(string2int_map,"yes", 1); - map_register(string2int_map,"no", 0); + map_register(reserved_word_map,"yes", 1); + map_register(reserved_word_map,"no", 0); i=0; @@ -610,7 +636,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path "Maat read table info %s line %d error: not enough column.",table_info_path,i); continue; } - ret=map_str2int(string2int_map,str_tolower(table_type_str),(int*)&(p->table_type)); + ret=map_str2int(reserved_word_map,str_tolower(table_type_str),(int*)&(p->table_type)); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, @@ -621,7 +647,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - ret=read_expr_table_info(line, p, string2int_map); + ret=read_expr_table_info(line, p, reserved_word_map); if(ret<0) { fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i); @@ -661,7 +687,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path } break; case TABLE_TYPE_COMPOSITION: - ret=read_composition_table_schema(line, p, string2int_map); + ret=read_composition_table_schema(table_mgr, line, p, reserved_word_map); if(ret<0) { fprintf(stderr,"Maat read table info %s line %d error:illegal composition table schema.\n", table_info_path,i); @@ -671,7 +697,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path } break; case TABLE_TYPE_VIRTUAL: - ret=read_virtual_table_schema(line, p, string2int_map); + ret=read_virtual_table_schema(table_mgr, line, p, reserved_word_map); if(ret<0) { fprintf(stderr,"Maat read table info %s line %d error:illegal virtual table schema.\n", table_info_path,i); @@ -684,7 +710,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path ret=sscanf(not_care,"%[a-z0-9]",tmp_str); if(ret>0) { - ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding)); + ret=map_str2int(reserved_word_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding)); } if(ret!=1) { @@ -703,6 +729,15 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path goto invalid_table; } + ret=map_register(table_mgr->map_tablename2id, p->table_name[0], p->table_id); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Duplicate table %s of table id %d", + p->table_name[0], + p->table_id); + goto invalid_table; + } if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table; { conj_table=p_table_info[p->table_id]; @@ -722,21 +757,16 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path //use goto to free the conjunctioned table_info goto invalid_table; } - p_table_info[p->table_id]=p; table_mgr->table_cnt++; continue; + invalid_table: table_info_free(p); p=NULL; } fclose(fp); - ret=Maat_table_build_map(table_mgr, logger); - if(ret<0) - { - return NULL; - } - map_destroy(string2int_map); + map_destroy(reserved_word_map); return table_mgr; } size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr) @@ -822,13 +852,13 @@ struct Maat_table_schema * Maat_table_get_by_id_raw(struct Maat_table_manager* t return table_mgr->p_table_info[table_id]; } -struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id) +struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id) { - + enum MAAT_SCAN_TYPE tab_scan_type; struct Maat_table_schema **p_table_info=table_mgr->p_table_info; size_t n_table=MAX_TABLE_NUM; - struct Maat_table_schema *p_table=NULL, *p_real_table=NULL; + struct Maat_table_schema *p_table=NULL, *p_physical_table=NULL; if((unsigned int) table_id>n_table) { return NULL; @@ -844,23 +874,20 @@ struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* } if(p_table->table_type==TABLE_TYPE_VIRTUAL) { - p_real_table=p_table_info[p_table->virtual_table.real_table_id]; + p_physical_table=p_table_info[p_table->virtual_table.physical_table_id[scan_type]]; *virutal_table_id=table_id; } else { - p_real_table=p_table; + p_physical_table=p_table; if(virutal_table_id) *virutal_table_id=0; } - if(p_real_table->table_type!=expect_type) + tab_scan_type=Maat_table_get_scan_type(p_physical_table->table_type); + if(tab_scan_type!=scan_type) { - if((expect_type==TABLE_TYPE_EXPR && p_real_table->table_type!=TABLE_TYPE_EXPR_PLUS)|| - (expect_type==TABLE_TYPE_IP && p_real_table->table_type!=TABLE_TYPE_IP_PLUS)) - { - return NULL; - } + return NULL; } - return p_real_table; + return p_physical_table; } int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name) { @@ -883,7 +910,7 @@ int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, void* u_para) { int idx=0; - struct Maat_table_schema *p_table=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + struct Maat_table_schema *p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL); struct plugin_table_schema *plugin_desc=&(p_table->plugin); if(p_table==NULL) { @@ -913,7 +940,7 @@ struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_tabl { return NULL; } - p_table=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL); + p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL); if(!p_table) { return NULL; @@ -938,7 +965,7 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c { return -1; } - p_table=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL); + p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL); if(!p_table) { return -1; @@ -1175,28 +1202,15 @@ void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char* assert(i<=p_table->conj_cnt); } -int Maat_table_get_child_id(struct Maat_table_schema* p_table, enum MAAT_TABLE_CHILD_TYPE type) +int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type) { int ret=-1; + struct Maat_table_schema* p_table=Maat_table_get_by_id_raw(table_mgr, parent_table_id); if(p_table->table_type!=TABLE_TYPE_COMPOSITION) { return -1; } - switch (type) - { - case CHILD_TABLE_TYPE_SOURCE_IP: - ret=p_table->composition.source_table.real_table_id; - break; - case CHILD_TABLE_TYPE_DESTINATION_IP: - ret=p_table->composition.destination_table.real_table_id; - break; - case CHILD_TABLE_TYPE_SESSION: - ret=p_table->composition.session_table.real_table_id; - break; - default: - assert(0); - break; - } + ret=p_table->composition.component_table_id[type]; return ret; } int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table) @@ -1219,3 +1233,4 @@ int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p } return valid_flag_column; } + diff --git a/src/inc_internal/Maat_table.h b/src/inc_internal/Maat_table.h index e6662a0..c34a1de 100644 --- a/src/inc_internal/Maat_table.h +++ b/src/inc_internal/Maat_table.h @@ -17,6 +17,28 @@ enum USER_REGION_ENCODE USER_REGION_ENCODE_ESCAPE, USER_REGION_ENCODE_BASE64 }; +enum MAAT_SCAN_TYPE +{ + SCAN_TYPE_INVALID=-1, + SCAN_TYPE_NONE=0, + SCAN_TYPE_PLUGIN, + SCAN_TYPE_IP_PLUGIN, + SCAN_TYPE_FQDN_PLUGIN, + SCAN_TYPE_IP, + SCAN_TYPE_INTERVAL, + SCAN_TYPE_STRING, + __SCAN_TYPE_MAX +}; + +enum MAAT_TABLE_COMPONENT_TYPE +{ + COMPONENT_TABLE_TYPE_NONE=-1, + COMPONENT_TABLE_TYPE_SOURCE_IP=0, + COMPONENT_TABLE_TYPE_DESTINATION_IP, + COMPONENT_TABLE_TYPE_SESSION, + __COMPONENT_TABLE_TYPE_MAX +}; + enum MAAT_TABLE_TYPE { @@ -28,6 +50,8 @@ enum MAAT_TABLE_TYPE TABLE_TYPE_DIGEST, TABLE_TYPE_EXPR_PLUS, TABLE_TYPE_SIMILARITY, + TABLE_TYPE_INTERVAL_PLUS, + //Above are physical table for scan TABLE_TYPE_VIRTUAL, TABLE_TYPE_COMPOSITION, TABLE_TYPE_GROUP2GROUP, @@ -36,7 +60,6 @@ enum MAAT_TABLE_TYPE TABLE_TYPE_COMPILE, TABLE_TYPE_PLUGIN, TABLE_TYPE_IP_PLUGIN, - TABLE_TYPE_INTERVAL_PLUS, TABLE_TYPE_FQDN_PLUGIN }; @@ -69,12 +92,11 @@ struct expr_table_schema }; struct virtual_table_schema { - int real_table_id; - char real_table_name[MAX_TABLE_NAME_LEN]; + int physical_table_id[__SCAN_TYPE_MAX]; }; struct composition_table_schema { - struct virtual_table_schema source_table, destination_table, session_table; + int component_table_id[__COMPONENT_TABLE_TYPE_MAX]; }; struct plugin_table_callback_schema { @@ -155,8 +177,9 @@ void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr); size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr); size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr); -struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id); +struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id); struct Maat_table_schema * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id); +enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type); int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name); int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, @@ -206,13 +229,7 @@ void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mg int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr); struct Maat_table_schema* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name); void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char* table_name); -enum MAAT_TABLE_CHILD_TYPE -{ - CHILD_TABLE_TYPE_NONE=-1, - CHILD_TABLE_TYPE_SOURCE_IP=0, - CHILD_TABLE_TYPE_DESTINATION_IP, - CHILD_TABLE_TYPE_SESSION -}; -int Maat_table_get_child_id(struct Maat_table_schema* p_table, enum MAAT_TABLE_CHILD_TYPE type); + +int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type); int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table); diff --git a/test/maat_json.json b/test/maat_json.json index 11e1520..8f835c3 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -60,7 +60,7 @@ "regions": [ { "table_name": "GeoLocation", - "table_type": "string", + "table_type": "expr", "table_content": { "keywords": "Greece.Sparta", "expr_type": "none", @@ -69,6 +69,35 @@ } } ] + }, + { + "group_name": "FQDN_OBJ1", + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "sports.example.com", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "FQDN_CAT1", + "regions": [ + { + "table_name": "INTERGER_PLUS", + "table_type": "intval_plus", + "table_content": { + "district": "fqdn_cat_id", + "low_boundary": 1724, + "up_boundary": 1724 + } + } + ] } ], "rules": [ @@ -1220,7 +1249,7 @@ "do_blacklist": 0, "do_log": 0, "effective_rage": 0, - "user_region": "VirtualWithPhysical", + "user_region": "VirtualWithOnePhysical", "is_valid": "yes", "groups": [ { @@ -1850,7 +1879,31 @@ ] } ] - } + }, + { + "compile_id": 180, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "Hierarchy_VirtualWithTwoPhysical", + "is_valid": "yes", + "groups": [ + { + "group_name":"FQDN_OBJ1", + "virtual_table":"VIRTUAL_SSL_SNI", + "not_flag" : 0, + "clause_index":0 + }, + { + "group_name":"FQDN_CAT1", + "virtual_table":"VIRTUAL_SSL_SNI", + "not_flag" : 0, + "clause_index":0 + } + ] + } ], "plugin_table": [ { diff --git a/test/table_info.conf b/test/table_info.conf index c92aa17..adfaf31 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -56,4 +56,5 @@ 33 GeoLocation expr UTF8 UTF8 yes 0 34 SOURCE_IP_GEO virtual GeoLocation -- 35 INTERGER_PLUS intval_plus -- -36 TEST_FQDN_PLUGIN_WITH_EXDATA fqdn_plugin {"row_id":1,"is_suffix_match":2,"fqdn":3,"valid":5} -- \ No newline at end of file +36 TEST_FQDN_PLUGIN_WITH_EXDATA fqdn_plugin {"row_id":1,"is_suffix_match":2,"fqdn":3,"valid":5} -- +37 VIRTUAL_SSL_SNI virtual ["KEYWORDS_TABLE","INTERGER_PLUS"] -- \ No newline at end of file diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 9cfdc5b..2e6876e 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -1835,9 +1835,9 @@ TEST(ScanResult, LongerServiceDefine) free(buff); return; } -TEST(Hierarchy, VirtualWithPhysical) +TEST(Hierarchy, VirtualOfOnePhysical) { -#define Hierarchy_VirtualWithPhysical +#define Hierarchy_VirtualOfOnePhysical int ret=0, table_id=0; const char* http_content="Batman\\:Take me Home.Superman/:Fine,stay with me."; const char* http_url="https://blog.csdn.net/littlefang/article/details/8213058"; @@ -1876,6 +1876,42 @@ TEST(Hierarchy, VirtualWithPhysical) return; } +TEST(Hierarchy, VirtualOfTwoPhysical) +{ +#define Hierarchy_VirtualWithTwoPhysical + int ret=0, table_id=0; + const char* sni="sports.example.com"; + int cat_id=1724; + + + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + + table_id=Maat_table_register(g_feather, "VIRTUAL_SSL_SNI"); + ASSERT_GT(table_id, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_UTF8, sni, strlen(sni), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 180); + Maat_clean_status(&mid); + + memset(result, 0, sizeof(result)); + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "fqdn_cat_id", strlen("fqdn_cat_id")); + ASSERT_EQ(ret, 0); + + ret=Maat_scan_intval(g_feather, table_id, cat_id, result,4, &mid, 0); + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 180); + + Maat_clean_status(&mid); + + return; +} + TEST(Hierarchy, VirtualWithVirtual) { #define Hierarchy_VirtualWithVirtual @@ -1916,6 +1952,7 @@ TEST(Hierarchy, VirtualWithVirtual) return; } + TEST(Hierarchy, OneGroupInTwoVirtual) { #define Hierarchy_OneGroupInTwoVirtual