diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index e332f45..d072527 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -158,12 +158,49 @@ static int compare_compile_inner(const void *a, const void *b) return (rb->compile_id-ra->compile_id); } } - -int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num) +struct scan_region_hit_wraper { + void* elem_array; + size_t elem_size; + size_t n_elem; + size_t group_offset; + int virtual_table_id; + int is_last_region; +}; +void scan_region_hit_wraper_build_with_rulescan(struct scan_region_hit_wraper* region_hit, scan_result_t* rulescan_rslt, size_t n_rslt, int is_last_region, int virual_table_id) +{ + memset(®ion_hit, 0, sizeof(region_hit)); + region_hit->elem_array=rulescan_rslt; + region_hit->n_elem=n_rslt; + region_hit->elem_size=sizeof(scan_result_t); + region_hit->group_offset=offsetof(scan_result_t, tag); + region_hit->is_last_region=is_last_region; + region_hit->virtual_table_id=virual_table_id; + return; +} +void scan_region_hit_wraper_build_with_GIE(struct scan_region_hit_wraper* region_hit, GIE_result_t* GIE_rslt, size_t n_rslt, int is_last_region, int virual_table_id) +{ + memset(®ion_hit, 0, sizeof(region_hit)); + region_hit->elem_array=GIE_rslt; + region_hit->n_elem=n_rslt; + region_hit->elem_size=sizeof(GIE_result_t); + region_hit->group_offset=offsetof(GIE_result_t, tag); + region_hit->is_last_region=is_last_region; + region_hit->virtual_table_id=virual_table_id; + return; +} + +int region_compile(_Maat_feather_t*feather, struct _INNER_scan_status_t *_mid, const struct scan_region_hit_wraper* region_hit_wraper, struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num) +{ + int is_last_region=region_hit_wraper->is_last_region; + void* region_hit=region_hit_wraper->elem_array; + size_t region_type_size=region_hit_wraper->elem_size; + size_t group_offset=region_hit_wraper->group_offset; + size_t region_hit_num=region_hit_wraper->n_elem; int scan_ret=0, result_cnt=0; - int ret=0, i=0, j=0; + int ret=0; + size_t i=0, j=0; size_t r_in_c_cnt=0; unsigned char has_not_flag=0; struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler; @@ -206,7 +243,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation**), compare_compile_inner); } - for(i=0;ihit_cnt, thread_num, 1); } + scan_region_hit_wraper_build_with_rulescan(®ion_hit, region_result, hit_region_cnt, _mid->is_last_region, 0); _mid=grab_mid(mid,_feather,thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), - hit_region_cnt, + ®ion_hit, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); @@ -1486,6 +1523,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id intval_scan_data.int_data=intval; Maat_table_desc* p_table=NULL; struct timespec start,end; + struct scan_region_hit_wraper region_hit; if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&start); @@ -1525,10 +1563,9 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); } _mid=grab_mid(mid, _feather, thread_num, 1); + scan_region_hit_wraper_build_with_rulescan(®ion_hit, region_result, region_ret, _mid->is_last_region, 0); compile_ret=region_compile(_feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), - region_ret, + ®ion_hit, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); @@ -1647,11 +1684,10 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); } _mid=grab_mid(mid, _feather, thread_num, 1); - + struct scan_region_hit_wraper region_hit; + scan_region_hit_wraper_build_with_rulescan(®ion_hit, region_result, region_ret, _mid->is_last_region, 0); compile_ret=region_compile(_feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), - region_ret, + ®ion_hit, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); @@ -1867,10 +1903,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1); } _mid=grab_mid(mid, sp->feather,sp->thread_num, 1); + struct scan_region_hit_wraper region_hit; + scan_region_hit_wraper_build_with_rulescan(®ion_hit, region_result, region_ret, _mid->is_last_region, 0); + compile_ret=region_compile(sp->feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), - hit_region_cnt, + ®ion_hit, result,compile_result,rule_num, sp->thread_num); assert(_mid->is_last_region<2); @@ -2040,7 +2077,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); int do_query=0; - GIE_result_t query_result[MAX_SCANNER_HIT_NUM]; + GIE_result_t region_result[MAX_SCANNER_HIT_NUM]; int hit_region_cnt=0,compile_ret=0; _compile_result_t compile_result[rule_num];//dynamic array if(data==NULL||data_len<=0) @@ -2084,7 +2121,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int if(GIE_handle!=NULL) { - hit_region_cnt=GIE_query(GIE_handle, digest_buff,(int)strlen(digest_buff), query_result, MAX_SCANNER_HIT_NUM); + hit_region_cnt=GIE_query(GIE_handle, digest_buff,(int)strlen(digest_buff), region_result, MAX_SCANNER_HIT_NUM); } free(digest_buff); @@ -2102,10 +2139,10 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1); } _mid=grab_mid(mid,sp->feather, sp->thread_num,1); + struct scan_region_hit_wraper region_hit; + scan_region_hit_wraper_build_with_GIE(®ion_hit, region_result, hit_region_cnt, _mid->is_last_region, 0); compile_ret=region_compile(sp->feather,_mid->inner, - _mid->is_last_region, - query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag), - hit_region_cnt, + ®ion_hit, result,compile_result,rule_num, sp->thread_num); assert(_mid->is_last_region<2); @@ -2268,10 +2305,10 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); + struct scan_region_hit_wraper region_hit; + scan_region_hit_wraper_build_with_GIE(®ion_hit, region_result, hit_region_cnt, _mid->is_last_region, 0); compile_ret=region_compile(_feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag), - hit_region_cnt, + ®ion_hit, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 0dd5571..687cd69 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -941,7 +941,7 @@ void make_group_set(struct Maat_compile_group_relation* relation, struct bool_ex { continue; } - a_set->items[j].item_id=group->group_id; + a_set->items[j].item_id=(unsigned long long)relation->virtual_table_id[j]<<32|group->group_id; a_set->items[j].not_flag=relation->not_flag[j]; if(a_set->items[j].not_flag) { @@ -1894,7 +1894,7 @@ unsigned int del_region_from_group(struct Maat_group_inner* group,int region_id, return i; } -int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maat_group_inner* a_rule_group, int not_flag) +int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maat_group_inner* a_rule_group, int virual_table_id, int not_flag) { int i=0,ret=-1; int write_pos=-1; @@ -1945,6 +1945,7 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa { relation->not_flag[write_pos]=0; } + relation->virtual_table_id[write_pos]=virual_table_id; relation->group_cnt++; a_rule_group->ref_by_parent_cnt++; ret=1; @@ -2500,7 +2501,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro { compile_rule=create_compile_group_relation(db_group_rule->parent_id, scanner); } - ret=add_group_to_compile(compile_rule, group_rule, db_group_rule->not_flag); + ret=add_group_to_compile(compile_rule, group_rule, db_group_rule->virtual_table_id, db_group_rule->not_flag); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, @@ -2648,18 +2649,20 @@ int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_ scanner->to_update_compile_cnt++; return 1; } -void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger) +void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner, MESA_htable_handle map_tablename2id, void* logger) { struct db_group_rule_t db_group_rule; struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; int ret=0; + char virtual_table_name[MAX_TABLE_NAME_LEN]={0}; memset(&db_group_rule, 0, sizeof(db_group_rule)); - ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%d", &(db_group_rule.group_id), + ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%d\t%s", &(db_group_rule.group_id), &(db_group_rule.parent_id), &(db_group_rule.is_valid), &(db_group_rule.not_flag), - &(db_group_rule.parent_type)); - if(ret!=3&&ret!=4&&ret!=5) + &(db_group_rule.parent_type), + virtual_table_name); + if(ret!=3&&ret!=4&&ret!=5&&ret!=6) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , "update error, invalid format of group table %s:%s", @@ -2679,6 +2682,19 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru table->udpate_err_cnt++; return; } + if(strlen(virtual_table_name)>0) + { + ret=map_str2int(map_tablename2id, virtual_table_name, &(db_group_rule.virtual_table_id)); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + "update error, unknown virutal table name: %s of group table %s:%s.", + virtual_table_name, + table->table_name[table->updating_name], table_line); + table->udpate_err_cnt++; + return; + } + } if(db_group_rule.is_valid==FALSE) { ret=del_group_rule(table, &db_group_rule, scanner, logger); @@ -2720,14 +2736,6 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru return; } -void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct Maat_scanner *scanner,void* logger) -{ - char virtual_group_line[256]; - snprintf(virtual_group_line,sizeof(virtual_group_line), - "%d\t%d\t%d",region_id,compile_id,is_valid); - update_group_rule(table, virtual_group_line,scanner,logger); - return; -} void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) { struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); @@ -2828,16 +2836,6 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc table->udpate_err_cnt++; goto error_out; } - if(group_mode_on==FALSE)//for compatible old version - { - compatible_group_udpate(table - ,maat_str_rule->region_id - ,maat_str_rule->group_id - ,maat_str_rule->is_valid - ,scanner - ,logger); - maat_str_rule->group_id=maat_str_rule->region_id; - } if(maat_str_rule->is_valid==FALSE) { @@ -3199,16 +3197,6 @@ void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struc table->udpate_err_cnt++; goto error_out; } - if(group_mode_on==FALSE)//for compatible old version - { - compatible_group_udpate(table, - ip_rule->region_id, - ip_rule->group_id, - ip_rule->is_valid, - scanner, - logger); - ip_rule->group_id=ip_rule->region_id; - } if(ip_rule->is_valid==FALSE) { ret=del_region_rule(table, @@ -3274,16 +3262,6 @@ void update_intval_rule(struct Maat_table_desc* table,const char* table_line,str table->udpate_err_cnt++; goto error_out; } - if(group_mode_on==FALSE)//for compatible old version - { - compatible_group_udpate(table - ,intval_rule->region_id - ,intval_rule->group_id - ,intval_rule->is_valid - ,scanner - ,logger); - intval_rule->group_id=intval_rule->region_id; - } if(intval_rule->is_valid==FALSE) { @@ -3454,16 +3432,6 @@ void update_digest_rule(struct Maat_table_desc* table,const char* table_line,str table->udpate_err_cnt++; goto error_out; } - if(group_mode_on==FALSE)//for compatible old version - { - compatible_group_udpate(table - ,digest_rule->region_id - ,digest_rule->group_id - ,digest_rule->is_valid - ,scanner - ,logger); - digest_rule->group_id=digest_rule->region_id; - } if(digest_rule->is_valid==FALSE) { @@ -4057,7 +4025,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) update_compile_rule(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); break; case TABLE_TYPE_GROUP: - update_group_rule(feather->p_table_info[table_id], line, scanner,feather->logger); + update_group_rule(feather->p_table_info[table_id], line, scanner, feather->map_tablename2id, feather->logger); break; case TABLE_TYPE_PLUGIN: update_plugin_table(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index dcb31a0..e302190 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -114,6 +114,7 @@ struct db_group_rule_t int is_valid; int not_flag; int parent_type; //PARENT_TYPE_**, 0:compile, 1: group. + int virtual_table_id; }; struct op_expr_t { @@ -141,6 +142,7 @@ struct Maat_compile_group_relation long long magic_num; struct Maat_compile_rule *compile; dynamic_array_t *groups; //element is struct Maat_group_inner* + int virtual_table_id[MAX_ITEMS_PER_BOOL_EXPR]; char not_flag[MAX_ITEMS_PER_BOOL_EXPR]; int compile_id;//equal to compile->m_rule.config_id int group_boundary; @@ -161,7 +163,7 @@ struct Maat_group_inner char* group_name; int has_compile_neighbors; int vertex_id; - int top_group_cnt; + size_t top_group_cnt; long long* top_groups; dynamic_array_t *regions; pthread_mutex_t mutex; @@ -303,7 +305,7 @@ struct Maat_scanner igraph_t group_graph; int grp_vertex_id_generator; int most_popular_sub_group; - long long max_presented_top_group_cnt; + unsigned long long max_presented_top_group_cnt; unsigned int district_num; unsigned int cfg_num; diff --git a/src/inc_internal/Maat_table_description.h b/src/inc_internal/Maat_table_description.h index b924c34..249b3ac 100644 --- a/src/inc_internal/Maat_table_description.h +++ b/src/inc_internal/Maat_table_description.h @@ -23,10 +23,10 @@ enum MAAT_TABLE_TYPE TABLE_TYPE_DIGEST, TABLE_TYPE_EXPR_PLUS, TABLE_TYPE_SIMILARITY, + TABLE_TYPE_VIRTUAL, TABLE_TYPE_GROUP, TABLE_TYPE_COMPILE, TABLE_TYPE_PLUGIN - }; struct compile_ex_data_idx @@ -90,7 +90,10 @@ struct ip_table_desc int ipv4_rule_cnt; int ipv6_rule_cnt; }; - +struct virtual_table_desc +{ + int real_table_id; +}; struct Maat_table_desc { int table_id; @@ -103,7 +106,8 @@ struct Maat_table_desc struct compile_table_desc compile; struct expr_table_desc expr; struct ip_table_desc ip; - struct plugin_table_desc plugin; + struct plugin_table_desc plugin; + struct virtual_table_desc virtual_table;; void* others;//group, interval and digest don't have sperate description info. }; //for stat>>>>>>>>