gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

不再兼容无group表的情况，重构region_compile函数。

Browse Source
This commit is contained in:
zhengchao
2019-07-23 12:40:05 +06:00
parent eb7912a854
commit 24b2b4c1b6
4 changed files with 97 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
src/entry/Maat_api.cpp
View File

@@ -158,12 +158,49 @@ static int compare_compile_inner(const void *a, const void *b)
return (rb->compile_id-ra->compile_id);
}
}
int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num)
struct scan_region_hit_wraper
{
void* elem_array;
size_t elem_size;
size_t n_elem;
size_t group_offset;
int virtual_table_id;
int is_last_region;
};
void scan_region_hit_wraper_build_with_rulescan(struct scan_region_hit_wraper* region_hit, scan_result_t* rulescan_rslt, size_t n_rslt, int is_last_region, int virual_table_id)
{
memset(&region_hit, 0, sizeof(region_hit));
region_hit->elem_array=rulescan_rslt;
region_hit->n_elem=n_rslt;
region_hit->elem_size=sizeof(scan_result_t);
region_hit->group_offset=offsetof(scan_result_t, tag);
region_hit->is_last_region=is_last_region;
region_hit->virtual_table_id=virual_table_id;
return;
}
void scan_region_hit_wraper_build_with_GIE(struct scan_region_hit_wraper* region_hit, GIE_result_t* GIE_rslt, size_t n_rslt, int is_last_region, int virual_table_id)
{
memset(&region_hit, 0, sizeof(region_hit));
region_hit->elem_array=GIE_rslt;
region_hit->n_elem=n_rslt;
region_hit->elem_size=sizeof(GIE_result_t);
region_hit->group_offset=offsetof(GIE_result_t, tag);
region_hit->is_last_region=is_last_region;
region_hit->virtual_table_id=virual_table_id;
return;
}
int region_compile(_Maat_feather_t*feather, struct _INNER_scan_status_t *_mid, const struct scan_region_hit_wraper* region_hit_wraper, struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num)
{
int is_last_region=region_hit_wraper->is_last_region;
void* region_hit=region_hit_wraper->elem_array;
size_t region_type_size=region_hit_wraper->elem_size;
size_t group_offset=region_hit_wraper->group_offset;
size_t region_hit_num=region_hit_wraper->n_elem;
int scan_ret=0, result_cnt=0;
int ret=0, i=0, j=0;
int ret=0;
size_t i=0, j=0;
size_t r_in_c_cnt=0;
unsigned char has_not_flag=0;
struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler;
@@ -206,7 +243,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation**),
compare_compile_inner);
}
for(i=0;i<scan_ret&&result_cnt<size;i++)
for(i=0; i<(unsigned int)scan_ret&&result_cnt<size; i++)
{
relation=relation_array[i];
if(relation==NULL)
@@ -1341,6 +1378,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
struct expr_table_desc* expr_desc=NULL;
struct timespec start,end;
Maat_scanner* my_scanner=NULL;
struct scan_region_hit_wraper region_hit;
if(data==NULL||data_len<=0)
{
return 0;
@@ -1421,11 +1459,10 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{
alignment_int64_array_add(table_rt->hit_cnt, thread_num, 1);
}
scan_region_hit_wraper_build_with_rulescan(&region_hit, region_result, hit_region_cnt, _mid->is_last_region, 0);
_mid=grab_mid(mid,_feather,thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
hit_region_cnt,
&region_hit,
result,compile_result,rule_num,
thread_num);
assert(_mid->is_last_region<2);
@@ -1486,6 +1523,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
intval_scan_data.int_data=intval;
Maat_table_desc* p_table=NULL;
struct timespec start,end;
struct scan_region_hit_wraper region_hit;
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
@@ -1525,10 +1563,9 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
}
_mid=grab_mid(mid, _feather, thread_num, 1);
scan_region_hit_wraper_build_with_rulescan(&region_hit, region_result, region_ret, _mid->is_last_region, 0);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
region_ret,
&region_hit,
result,compile_result,rule_num,
thread_num);
assert(_mid->is_last_region<2);
@@ -1647,11 +1684,10 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
}
_mid=grab_mid(mid, _feather, thread_num, 1);
struct scan_region_hit_wraper region_hit;
scan_region_hit_wraper_build_with_rulescan(&region_hit, region_result, region_ret, _mid->is_last_region, 0);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
region_ret,
&region_hit,
result,compile_result,rule_num,
thread_num);
assert(_mid->is_last_region<2);
@@ -1867,10 +1903,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1);
}
_mid=grab_mid(mid, sp->feather,sp->thread_num, 1);
struct scan_region_hit_wraper region_hit;
scan_region_hit_wraper_build_with_rulescan(&region_hit, region_result, region_ret, _mid->is_last_region, 0);
compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
hit_region_cnt,
&region_hit,
result,compile_result,rule_num,
sp->thread_num);
assert(_mid->is_last_region<2);
@@ -2040,7 +2077,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
int do_query=0;
GIE_result_t query_result[MAX_SCANNER_HIT_NUM];
GIE_result_t region_result[MAX_SCANNER_HIT_NUM];
int hit_region_cnt=0,compile_ret=0;
_compile_result_t compile_result[rule_num];//dynamic array
if(data==NULL||data_len<=0)
@@ -2084,7 +2121,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
if(GIE_handle!=NULL)
{
hit_region_cnt=GIE_query(GIE_handle, digest_buff,(int)strlen(digest_buff), query_result, MAX_SCANNER_HIT_NUM);
hit_region_cnt=GIE_query(GIE_handle, digest_buff,(int)strlen(digest_buff), region_result, MAX_SCANNER_HIT_NUM);
}
free(digest_buff);
@@ -2102,10 +2139,10 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1);
}
_mid=grab_mid(mid,sp->feather, sp->thread_num,1);
struct scan_region_hit_wraper region_hit;
scan_region_hit_wraper_build_with_GIE(&region_hit, region_result, hit_region_cnt, _mid->is_last_region, 0);
compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region,
query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag),
hit_region_cnt,
&region_hit,
result,compile_result,rule_num,
sp->thread_num);
assert(_mid->is_last_region<2);
@@ -2268,10 +2305,10 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
{
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
struct scan_region_hit_wraper region_hit;
scan_region_hit_wraper_build_with_GIE(&region_hit, region_result, hit_region_cnt, _mid->is_last_region, 0);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
region_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag),
hit_region_cnt,
&region_hit,
result,compile_result,rule_num,
thread_num);
assert(_mid->is_last_region<2);

80
src/entry/Maat_rule.cpp
View File

@@ -941,7 +941,7 @@ void make_group_set(struct Maat_compile_group_relation* relation, struct bool_ex
{
continue;
}
a_set->items[j].item_id=group->group_id;
a_set->items[j].item_id=(unsigned long long)relation->virtual_table_id[j]<<32|group->group_id;
a_set->items[j].not_flag=relation->not_flag[j];
if(a_set->items[j].not_flag)
{
@@ -1894,7 +1894,7 @@ unsigned int del_region_from_group(struct Maat_group_inner* group,int region_id,
return i;
}
int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maat_group_inner* a_rule_group, int not_flag)
int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maat_group_inner* a_rule_group, int virual_table_id, int not_flag)
{
int i=0,ret=-1;
int write_pos=-1;
@@ -1945,6 +1945,7 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa
{
relation->not_flag[write_pos]=0;
}
relation->virtual_table_id[write_pos]=virual_table_id;
relation->group_cnt++;
a_rule_group->ref_by_parent_cnt++;
ret=1;
@@ -2500,7 +2501,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
{
compile_rule=create_compile_group_relation(db_group_rule->parent_id, scanner);
}
ret=add_group_to_compile(compile_rule, group_rule, db_group_rule->not_flag);
ret=add_group_to_compile(compile_rule, group_rule, db_group_rule->virtual_table_id, db_group_rule->not_flag);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
@@ -2648,18 +2649,20 @@ int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_
scanner->to_update_compile_cnt++;
return 1;
}
void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger)
void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner, MESA_htable_handle map_tablename2id, void* logger)
{
struct db_group_rule_t db_group_rule;
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
int ret=0;
char virtual_table_name[MAX_TABLE_NAME_LEN]={0};
memset(&db_group_rule, 0, sizeof(db_group_rule));
ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%d", &(db_group_rule.group_id),
ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%d\t%s", &(db_group_rule.group_id),
&(db_group_rule.parent_id),
&(db_group_rule.is_valid),
&(db_group_rule.not_flag),
&(db_group_rule.parent_type));
if(ret!=3&&ret!=4&&ret!=5)
&(db_group_rule.parent_type),
virtual_table_name);
if(ret!=3&&ret!=4&&ret!=5&&ret!=6)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"update error, invalid format of group table %s:%s",
@@ -2679,6 +2682,19 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru
table->udpate_err_cnt++;
return;
}
if(strlen(virtual_table_name)>0)
{
ret=map_str2int(map_tablename2id, virtual_table_name, &(db_group_rule.virtual_table_id));
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"update error, unknown virutal table name: %s of group table %s:%s.",
virtual_table_name,
table->table_name[table->updating_name], table_line);
table->udpate_err_cnt++;
return;
}
}
if(db_group_rule.is_valid==FALSE)
{
ret=del_group_rule(table, &db_group_rule, scanner, logger);
@@ -2720,14 +2736,6 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru
return;
}
void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct Maat_scanner *scanner,void* logger)
{
char virtual_group_line[256];
snprintf(virtual_group_line,sizeof(virtual_group_line),
"%d\t%d\t%d",region_id,compile_id,is_valid);
update_group_rule(table, virtual_group_line,scanner,logger);
return;
}
void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on)
{
struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1);
@@ -2828,16 +2836,6 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc
table->udpate_err_cnt++;
goto error_out;
}
if(group_mode_on==FALSE)//for compatible old version
{
compatible_group_udpate(table
,maat_str_rule->region_id
,maat_str_rule->group_id
,maat_str_rule->is_valid
,scanner
,logger);
maat_str_rule->group_id=maat_str_rule->region_id;
}
if(maat_str_rule->is_valid==FALSE)
{
@@ -3199,16 +3197,6 @@ void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struc
table->udpate_err_cnt++;
goto error_out;
}
if(group_mode_on==FALSE)//for compatible old version
{
compatible_group_udpate(table,
ip_rule->region_id,
ip_rule->group_id,
ip_rule->is_valid,
scanner,
logger);
ip_rule->group_id=ip_rule->region_id;
}
if(ip_rule->is_valid==FALSE)
{
ret=del_region_rule(table,
@@ -3274,16 +3262,6 @@ void update_intval_rule(struct Maat_table_desc* table,const char* table_line,str
table->udpate_err_cnt++;
goto error_out;
}
if(group_mode_on==FALSE)//for compatible old version
{
compatible_group_udpate(table
,intval_rule->region_id
,intval_rule->group_id
,intval_rule->is_valid
,scanner
,logger);
intval_rule->group_id=intval_rule->region_id;
}
if(intval_rule->is_valid==FALSE)
{
@@ -3454,16 +3432,6 @@ void update_digest_rule(struct Maat_table_desc* table,const char* table_line,str
table->udpate_err_cnt++;
goto error_out;
}
if(group_mode_on==FALSE)//for compatible old version
{
compatible_group_udpate(table
,digest_rule->region_id
,digest_rule->group_id
,digest_rule->is_valid
,scanner
,logger);
digest_rule->group_id=digest_rule->region_id;
}
if(digest_rule->is_valid==FALSE)
{
@@ -4057,7 +4025,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
update_compile_rule(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
break;
case TABLE_TYPE_GROUP:
update_group_rule(feather->p_table_info[table_id], line, scanner,feather->logger);
update_group_rule(feather->p_table_info[table_id], line, scanner, feather->map_tablename2id, feather->logger);
break;
case TABLE_TYPE_PLUGIN:
update_plugin_table(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger);

6
src/inc_internal/Maat_rule_internal.h
View File

@@ -114,6 +114,7 @@ struct db_group_rule_t
int is_valid;
int not_flag;
int parent_type; //PARENT_TYPE_**, 0:compile, 1: group.
int virtual_table_id;
};
struct op_expr_t
{
@@ -141,6 +142,7 @@ struct Maat_compile_group_relation
long long magic_num;
struct Maat_compile_rule *compile;
dynamic_array_t *groups; //element is struct Maat_group_inner*
int virtual_table_id[MAX_ITEMS_PER_BOOL_EXPR];
char not_flag[MAX_ITEMS_PER_BOOL_EXPR];
int compile_id;//equal to compile->m_rule.config_id
int group_boundary;
@@ -161,7 +163,7 @@ struct Maat_group_inner
char* group_name;
int has_compile_neighbors;
int vertex_id;
int top_group_cnt;
size_t top_group_cnt;
long long* top_groups;
dynamic_array_t *regions;
pthread_mutex_t mutex;
@@ -303,7 +305,7 @@ struct Maat_scanner
igraph_t group_graph;
int grp_vertex_id_generator;
int most_popular_sub_group;
long long max_presented_top_group_cnt;
unsigned long long max_presented_top_group_cnt;
unsigned int district_num;
unsigned int cfg_num;

8
src/inc_internal/Maat_table_description.h
View File

@@ -23,10 +23,10 @@ enum MAAT_TABLE_TYPE
TABLE_TYPE_DIGEST,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_VIRTUAL,
TABLE_TYPE_GROUP,
TABLE_TYPE_COMPILE,
TABLE_TYPE_PLUGIN
};
struct compile_ex_data_idx
@@ -90,7 +90,10 @@ struct ip_table_desc
int ipv4_rule_cnt;
int ipv6_rule_cnt;
};
struct virtual_table_desc
{
int real_table_id;
};
struct Maat_table_desc
{
int table_id;
@@ -104,6 +107,7 @@ struct Maat_table_desc
struct expr_table_desc expr;
struct ip_table_desc ip;
struct plugin_table_desc plugin;
struct virtual_table_desc virtual_table;;
void* others;//group, interval and digest don't have sperate description info.
};
//for stat>>>>>>>>