From 24b2b4c1b6f1b36dbccc2c02f0b2549dc219eb77 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 23 Jul 2019 12:40:05 +0600 Subject: [PATCH 01/29] =?UTF-8?q?=E4=B8=8D=E5=86=8D=E5=85=BC=E5=AE=B9?= =?UTF-8?q?=E6=97=A0group=E8=A1=A8=E7=9A=84=E6=83=85=E5=86=B5=EF=BC=8C?= =?UTF-8?q?=E9=87=8D=E6=9E=84region=5Fcompile=E5=87=BD=E6=95=B0=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 87 ++++++++++++++++------- src/entry/Maat_rule.cpp | 80 +++++++-------------- src/inc_internal/Maat_rule_internal.h | 6 +- src/inc_internal/Maat_table_description.h | 10 ++- 4 files changed, 97 insertions(+), 86 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index e332f45..d072527 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -158,12 +158,49 @@ static int compare_compile_inner(const void *a, const void *b) return (rb->compile_id-ra->compile_id); } } - -int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num) +struct scan_region_hit_wraper { + void* elem_array; + size_t elem_size; + size_t n_elem; + size_t group_offset; + int virtual_table_id; + int is_last_region; +}; +void scan_region_hit_wraper_build_with_rulescan(struct scan_region_hit_wraper* region_hit, scan_result_t* rulescan_rslt, size_t n_rslt, int is_last_region, int virual_table_id) +{ + memset(®ion_hit, 0, sizeof(region_hit)); + region_hit->elem_array=rulescan_rslt; + region_hit->n_elem=n_rslt; + region_hit->elem_size=sizeof(scan_result_t); + region_hit->group_offset=offsetof(scan_result_t, tag); + region_hit->is_last_region=is_last_region; + region_hit->virtual_table_id=virual_table_id; + return; +} +void scan_region_hit_wraper_build_with_GIE(struct scan_region_hit_wraper* region_hit, GIE_result_t* GIE_rslt, size_t n_rslt, int is_last_region, int virual_table_id) +{ + memset(®ion_hit, 0, sizeof(region_hit)); + region_hit->elem_array=GIE_rslt; + region_hit->n_elem=n_rslt; + region_hit->elem_size=sizeof(GIE_result_t); + region_hit->group_offset=offsetof(GIE_result_t, tag); + region_hit->is_last_region=is_last_region; + region_hit->virtual_table_id=virual_table_id; + return; +} + +int region_compile(_Maat_feather_t*feather, struct _INNER_scan_status_t *_mid, const struct scan_region_hit_wraper* region_hit_wraper, struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num) +{ + int is_last_region=region_hit_wraper->is_last_region; + void* region_hit=region_hit_wraper->elem_array; + size_t region_type_size=region_hit_wraper->elem_size; + size_t group_offset=region_hit_wraper->group_offset; + size_t region_hit_num=region_hit_wraper->n_elem; int scan_ret=0, result_cnt=0; - int ret=0, i=0, j=0; + int ret=0; + size_t i=0, j=0; size_t r_in_c_cnt=0; unsigned char has_not_flag=0; struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler; @@ -206,7 +243,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation**), compare_compile_inner); } - for(i=0;ihit_cnt, thread_num, 1); } + scan_region_hit_wraper_build_with_rulescan(®ion_hit, region_result, hit_region_cnt, _mid->is_last_region, 0); _mid=grab_mid(mid,_feather,thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), - hit_region_cnt, + ®ion_hit, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); @@ -1486,6 +1523,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id intval_scan_data.int_data=intval; Maat_table_desc* p_table=NULL; struct timespec start,end; + struct scan_region_hit_wraper region_hit; if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&start); @@ -1525,10 +1563,9 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); } _mid=grab_mid(mid, _feather, thread_num, 1); + scan_region_hit_wraper_build_with_rulescan(®ion_hit, region_result, region_ret, _mid->is_last_region, 0); compile_ret=region_compile(_feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), - region_ret, + ®ion_hit, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); @@ -1647,11 +1684,10 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); } _mid=grab_mid(mid, _feather, thread_num, 1); - + struct scan_region_hit_wraper region_hit; + scan_region_hit_wraper_build_with_rulescan(®ion_hit, region_result, region_ret, _mid->is_last_region, 0); compile_ret=region_compile(_feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), - region_ret, + ®ion_hit, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); @@ -1867,10 +1903,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1); } _mid=grab_mid(mid, sp->feather,sp->thread_num, 1); + struct scan_region_hit_wraper region_hit; + scan_region_hit_wraper_build_with_rulescan(®ion_hit, region_result, region_ret, _mid->is_last_region, 0); + compile_ret=region_compile(sp->feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), - hit_region_cnt, + ®ion_hit, result,compile_result,rule_num, sp->thread_num); assert(_mid->is_last_region<2); @@ -2040,7 +2077,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); int do_query=0; - GIE_result_t query_result[MAX_SCANNER_HIT_NUM]; + GIE_result_t region_result[MAX_SCANNER_HIT_NUM]; int hit_region_cnt=0,compile_ret=0; _compile_result_t compile_result[rule_num];//dynamic array if(data==NULL||data_len<=0) @@ -2084,7 +2121,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int if(GIE_handle!=NULL) { - hit_region_cnt=GIE_query(GIE_handle, digest_buff,(int)strlen(digest_buff), query_result, MAX_SCANNER_HIT_NUM); + hit_region_cnt=GIE_query(GIE_handle, digest_buff,(int)strlen(digest_buff), region_result, MAX_SCANNER_HIT_NUM); } free(digest_buff); @@ -2102,10 +2139,10 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1); } _mid=grab_mid(mid,sp->feather, sp->thread_num,1); + struct scan_region_hit_wraper region_hit; + scan_region_hit_wraper_build_with_GIE(®ion_hit, region_result, hit_region_cnt, _mid->is_last_region, 0); compile_ret=region_compile(sp->feather,_mid->inner, - _mid->is_last_region, - query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag), - hit_region_cnt, + ®ion_hit, result,compile_result,rule_num, sp->thread_num); assert(_mid->is_last_region<2); @@ -2268,10 +2305,10 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); + struct scan_region_hit_wraper region_hit; + scan_region_hit_wraper_build_with_GIE(®ion_hit, region_result, hit_region_cnt, _mid->is_last_region, 0); compile_ret=region_compile(_feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag), - hit_region_cnt, + ®ion_hit, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 0dd5571..687cd69 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -941,7 +941,7 @@ void make_group_set(struct Maat_compile_group_relation* relation, struct bool_ex { continue; } - a_set->items[j].item_id=group->group_id; + a_set->items[j].item_id=(unsigned long long)relation->virtual_table_id[j]<<32|group->group_id; a_set->items[j].not_flag=relation->not_flag[j]; if(a_set->items[j].not_flag) { @@ -1894,7 +1894,7 @@ unsigned int del_region_from_group(struct Maat_group_inner* group,int region_id, return i; } -int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maat_group_inner* a_rule_group, int not_flag) +int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maat_group_inner* a_rule_group, int virual_table_id, int not_flag) { int i=0,ret=-1; int write_pos=-1; @@ -1945,6 +1945,7 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa { relation->not_flag[write_pos]=0; } + relation->virtual_table_id[write_pos]=virual_table_id; relation->group_cnt++; a_rule_group->ref_by_parent_cnt++; ret=1; @@ -2500,7 +2501,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro { compile_rule=create_compile_group_relation(db_group_rule->parent_id, scanner); } - ret=add_group_to_compile(compile_rule, group_rule, db_group_rule->not_flag); + ret=add_group_to_compile(compile_rule, group_rule, db_group_rule->virtual_table_id, db_group_rule->not_flag); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, @@ -2648,18 +2649,20 @@ int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_ scanner->to_update_compile_cnt++; return 1; } -void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger) +void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner, MESA_htable_handle map_tablename2id, void* logger) { struct db_group_rule_t db_group_rule; struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; int ret=0; + char virtual_table_name[MAX_TABLE_NAME_LEN]={0}; memset(&db_group_rule, 0, sizeof(db_group_rule)); - ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%d", &(db_group_rule.group_id), + ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%d\t%s", &(db_group_rule.group_id), &(db_group_rule.parent_id), &(db_group_rule.is_valid), &(db_group_rule.not_flag), - &(db_group_rule.parent_type)); - if(ret!=3&&ret!=4&&ret!=5) + &(db_group_rule.parent_type), + virtual_table_name); + if(ret!=3&&ret!=4&&ret!=5&&ret!=6) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , "update error, invalid format of group table %s:%s", @@ -2679,6 +2682,19 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru table->udpate_err_cnt++; return; } + if(strlen(virtual_table_name)>0) + { + ret=map_str2int(map_tablename2id, virtual_table_name, &(db_group_rule.virtual_table_id)); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + "update error, unknown virutal table name: %s of group table %s:%s.", + virtual_table_name, + table->table_name[table->updating_name], table_line); + table->udpate_err_cnt++; + return; + } + } if(db_group_rule.is_valid==FALSE) { ret=del_group_rule(table, &db_group_rule, scanner, logger); @@ -2720,14 +2736,6 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru return; } -void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct Maat_scanner *scanner,void* logger) -{ - char virtual_group_line[256]; - snprintf(virtual_group_line,sizeof(virtual_group_line), - "%d\t%d\t%d",region_id,compile_id,is_valid); - update_group_rule(table, virtual_group_line,scanner,logger); - return; -} void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) { struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); @@ -2828,16 +2836,6 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc table->udpate_err_cnt++; goto error_out; } - if(group_mode_on==FALSE)//for compatible old version - { - compatible_group_udpate(table - ,maat_str_rule->region_id - ,maat_str_rule->group_id - ,maat_str_rule->is_valid - ,scanner - ,logger); - maat_str_rule->group_id=maat_str_rule->region_id; - } if(maat_str_rule->is_valid==FALSE) { @@ -3199,16 +3197,6 @@ void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struc table->udpate_err_cnt++; goto error_out; } - if(group_mode_on==FALSE)//for compatible old version - { - compatible_group_udpate(table, - ip_rule->region_id, - ip_rule->group_id, - ip_rule->is_valid, - scanner, - logger); - ip_rule->group_id=ip_rule->region_id; - } if(ip_rule->is_valid==FALSE) { ret=del_region_rule(table, @@ -3274,16 +3262,6 @@ void update_intval_rule(struct Maat_table_desc* table,const char* table_line,str table->udpate_err_cnt++; goto error_out; } - if(group_mode_on==FALSE)//for compatible old version - { - compatible_group_udpate(table - ,intval_rule->region_id - ,intval_rule->group_id - ,intval_rule->is_valid - ,scanner - ,logger); - intval_rule->group_id=intval_rule->region_id; - } if(intval_rule->is_valid==FALSE) { @@ -3454,16 +3432,6 @@ void update_digest_rule(struct Maat_table_desc* table,const char* table_line,str table->udpate_err_cnt++; goto error_out; } - if(group_mode_on==FALSE)//for compatible old version - { - compatible_group_udpate(table - ,digest_rule->region_id - ,digest_rule->group_id - ,digest_rule->is_valid - ,scanner - ,logger); - digest_rule->group_id=digest_rule->region_id; - } if(digest_rule->is_valid==FALSE) { @@ -4057,7 +4025,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) update_compile_rule(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); break; case TABLE_TYPE_GROUP: - update_group_rule(feather->p_table_info[table_id], line, scanner,feather->logger); + update_group_rule(feather->p_table_info[table_id], line, scanner, feather->map_tablename2id, feather->logger); break; case TABLE_TYPE_PLUGIN: update_plugin_table(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index dcb31a0..e302190 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -114,6 +114,7 @@ struct db_group_rule_t int is_valid; int not_flag; int parent_type; //PARENT_TYPE_**, 0:compile, 1: group. + int virtual_table_id; }; struct op_expr_t { @@ -141,6 +142,7 @@ struct Maat_compile_group_relation long long magic_num; struct Maat_compile_rule *compile; dynamic_array_t *groups; //element is struct Maat_group_inner* + int virtual_table_id[MAX_ITEMS_PER_BOOL_EXPR]; char not_flag[MAX_ITEMS_PER_BOOL_EXPR]; int compile_id;//equal to compile->m_rule.config_id int group_boundary; @@ -161,7 +163,7 @@ struct Maat_group_inner char* group_name; int has_compile_neighbors; int vertex_id; - int top_group_cnt; + size_t top_group_cnt; long long* top_groups; dynamic_array_t *regions; pthread_mutex_t mutex; @@ -303,7 +305,7 @@ struct Maat_scanner igraph_t group_graph; int grp_vertex_id_generator; int most_popular_sub_group; - long long max_presented_top_group_cnt; + unsigned long long max_presented_top_group_cnt; unsigned int district_num; unsigned int cfg_num; diff --git a/src/inc_internal/Maat_table_description.h b/src/inc_internal/Maat_table_description.h index b924c34..249b3ac 100644 --- a/src/inc_internal/Maat_table_description.h +++ b/src/inc_internal/Maat_table_description.h @@ -23,10 +23,10 @@ enum MAAT_TABLE_TYPE TABLE_TYPE_DIGEST, TABLE_TYPE_EXPR_PLUS, TABLE_TYPE_SIMILARITY, + TABLE_TYPE_VIRTUAL, TABLE_TYPE_GROUP, TABLE_TYPE_COMPILE, TABLE_TYPE_PLUGIN - }; struct compile_ex_data_idx @@ -90,7 +90,10 @@ struct ip_table_desc int ipv4_rule_cnt; int ipv6_rule_cnt; }; - +struct virtual_table_desc +{ + int real_table_id; +}; struct Maat_table_desc { int table_id; @@ -103,7 +106,8 @@ struct Maat_table_desc struct compile_table_desc compile; struct expr_table_desc expr; struct ip_table_desc ip; - struct plugin_table_desc plugin; + struct plugin_table_desc plugin; + struct virtual_table_desc virtual_table;; void* others;//group, interval and digest don't have sperate description info. }; //for stat>>>>>>>> From c189b90e6d6276d40dfa7d1ba69d40cedc4cb983 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 23 Jul 2019 21:26:05 +0600 Subject: [PATCH 02/29] =?UTF-8?q?=E9=87=8D=E6=9E=84=E5=90=8Eregion=5Fcompi?= =?UTF-8?q?le=E5=90=8E=EF=BC=8C=E5=9B=9E=E5=BD=92=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E9=80=9A=E8=BF=87=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index d072527..62977c5 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -169,7 +169,7 @@ struct scan_region_hit_wraper }; void scan_region_hit_wraper_build_with_rulescan(struct scan_region_hit_wraper* region_hit, scan_result_t* rulescan_rslt, size_t n_rslt, int is_last_region, int virual_table_id) { - memset(®ion_hit, 0, sizeof(region_hit)); + memset(region_hit, 0, sizeof(struct scan_region_hit_wraper)); region_hit->elem_array=rulescan_rslt; region_hit->n_elem=n_rslt; region_hit->elem_size=sizeof(scan_result_t); @@ -180,7 +180,7 @@ void scan_region_hit_wraper_build_with_rulescan(struct scan_region_hit_wraper* r } void scan_region_hit_wraper_build_with_GIE(struct scan_region_hit_wraper* region_hit, GIE_result_t* GIE_rslt, size_t n_rslt, int is_last_region, int virual_table_id) { - memset(®ion_hit, 0, sizeof(region_hit)); + memset(region_hit, 0, sizeof(struct scan_region_hit_wraper)); region_hit->elem_array=GIE_rslt; region_hit->n_elem=n_rslt; region_hit->elem_size=sizeof(GIE_result_t); @@ -1378,7 +1378,6 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id struct expr_table_desc* expr_desc=NULL; struct timespec start,end; Maat_scanner* my_scanner=NULL; - struct scan_region_hit_wraper region_hit; if(data==NULL||data_len<=0) { return 0; @@ -1459,10 +1458,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id { alignment_int64_array_add(table_rt->hit_cnt, thread_num, 1); } - scan_region_hit_wraper_build_with_rulescan(®ion_hit, region_result, hit_region_cnt, _mid->is_last_region, 0); - _mid=grab_mid(mid,_feather,thread_num, 1); + _mid=grab_mid(mid, _feather, thread_num, 1); + struct scan_region_hit_wraper region_hit_wraper; + scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, 0); compile_ret=region_compile(_feather,_mid->inner, - ®ion_hit, + ®ion_hit_wraper, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); @@ -1523,7 +1523,6 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id intval_scan_data.int_data=intval; Maat_table_desc* p_table=NULL; struct timespec start,end; - struct scan_region_hit_wraper region_hit; if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&start); @@ -1563,9 +1562,10 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); } _mid=grab_mid(mid, _feather, thread_num, 1); - scan_region_hit_wraper_build_with_rulescan(®ion_hit, region_result, region_ret, _mid->is_last_region, 0); + struct scan_region_hit_wraper region_hit_wraper; + scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_ret, _mid->is_last_region, 0); compile_ret=region_compile(_feather,_mid->inner, - ®ion_hit, + ®ion_hit_wraper, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); @@ -1684,10 +1684,10 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); } _mid=grab_mid(mid, _feather, thread_num, 1); - struct scan_region_hit_wraper region_hit; - scan_region_hit_wraper_build_with_rulescan(®ion_hit, region_result, region_ret, _mid->is_last_region, 0); + struct scan_region_hit_wraper region_hit_wraper; + scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_ret, _mid->is_last_region, 0); compile_ret=region_compile(_feather,_mid->inner, - ®ion_hit, + ®ion_hit_wraper, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); @@ -1903,11 +1903,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1); } _mid=grab_mid(mid, sp->feather,sp->thread_num, 1); - struct scan_region_hit_wraper region_hit; - scan_region_hit_wraper_build_with_rulescan(®ion_hit, region_result, region_ret, _mid->is_last_region, 0); + struct scan_region_hit_wraper region_hit_wraper; + scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, 0); compile_ret=region_compile(sp->feather,_mid->inner, - ®ion_hit, + ®ion_hit_wraper, result,compile_result,rule_num, sp->thread_num); assert(_mid->is_last_region<2); @@ -2139,10 +2139,10 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1); } _mid=grab_mid(mid,sp->feather, sp->thread_num,1); - struct scan_region_hit_wraper region_hit; - scan_region_hit_wraper_build_with_GIE(®ion_hit, region_result, hit_region_cnt, _mid->is_last_region, 0); + struct scan_region_hit_wraper region_hit_wraper; + scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, 0); compile_ret=region_compile(sp->feather,_mid->inner, - ®ion_hit, + ®ion_hit_wraper, result,compile_result,rule_num, sp->thread_num); assert(_mid->is_last_region<2); @@ -2305,10 +2305,10 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); - struct scan_region_hit_wraper region_hit; - scan_region_hit_wraper_build_with_GIE(®ion_hit, region_result, hit_region_cnt, _mid->is_last_region, 0); + struct scan_region_hit_wraper region_hit_wraper; + scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, 0); compile_ret=region_compile(_feather,_mid->inner, - ®ion_hit, + ®ion_hit_wraper, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); From 2909cb1997cc30da4f9d73dced95d527b5360c91 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Thu, 25 Jul 2019 14:49:11 +0600 Subject: [PATCH 03/29] =?UTF-8?q?=E9=87=8D=E6=9E=84Maat=20table=E7=9B=B8?= =?UTF-8?q?=E5=85=B3=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/CMakeLists.txt | 2 +- src/entry/Maat_api.cpp | 149 +++--- src/entry/Maat_rule.cpp | 361 +-------------- src/entry/Maat_table.cpp | 430 ++++++++++++++++++ src/entry/Maat_utils.cpp | 10 + src/entry/json2iris.cpp | 2 +- src/inc_internal/Maat_limits.h | 12 + src/inc_internal/Maat_rule_internal.h | 24 +- ...{Maat_table_description.h => Maat_table.h} | 8 + src/inc_internal/Maat_utils.h | 3 + 10 files changed, 562 insertions(+), 439 deletions(-) create mode 100644 src/entry/Maat_table.cpp create mode 100644 src/inc_internal/Maat_limits.h rename src/inc_internal/{Maat_table_description.h => Maat_table.h} (84%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 483ddf3..abce127 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) -set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp) +set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) include_directories(/opt/MESA/include/MESA/) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 62977c5..494055c 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -16,9 +16,9 @@ #include "rulescan.h" #include "json2iris.h" -struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int table_id, enum MAAT_TABLE_TYPE expect_type) +struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id) { - struct Maat_table_desc *p_table=NULL; + struct Maat_table_desc *p_table=NULL, *p_real_table=NULL; if(table_id>MAX_TABLE_NUM) { return NULL; @@ -32,7 +32,17 @@ struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int tab { return NULL; } - if(p_table->table_type!=expect_type) + if(p_table->table_type==TABLE_TYPE_VIRTUAL) + { + p_real_table=_feather->p_table_info[p_table->virtual_table.real_table_id]; + *virutal_table_id=table_id; + } + else + { + p_real_table=p_table; + *virutal_table_id=0; + } + if(p_real_table->table_type!=expect_type) { if((expect_type==TABLE_TYPE_EXPR && p_table->table_type!=TABLE_TYPE_EXPR_PLUS)|| (expect_type==TABLE_TYPE_IP && p_table->table_type!=TABLE_TYPE_IP_PLUS)) @@ -40,7 +50,7 @@ struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int tab return NULL; } } - return p_table; + return p_real_table; } inline void INC_SCANNER_REF(Maat_scanner*scanner,int thread_num) { @@ -537,40 +547,18 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* return NULL; } _Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1); - feather->table_cnt=read_table_description(feather->p_table_info, MAX_TABLE_NUM, table_info_path, max_thread_num, logger); + feather->table_cnt=Maat_table_read_table_info(feather->p_table_info, MAX_TABLE_NUM, table_info_path, logger); if(feather->table_cnt==0) { - free(feather); - return NULL; + goto failed; } - feather->map_tablename2id=map_create(); - int i=0,j=0,ret=0; - for(i=0;imap_tablename2id=Maat_table_build(feather->p_table_info, MAX_TABLE_NUM, + feather->compile_tn, sizeof(feather->compile_tn), + feather->group_tn, sizeof(feather->group_tn), + logger); + if(feather->map_tablename2id==NULL) { - if(feather->p_table_info[i]!=NULL) - { - if(feather->p_table_info[i]->table_type==TABLE_TYPE_GROUP) - { - feather->GROUP_MODE_ON=1; - strncpy(feather->group_tn,feather->p_table_info[i]->table_name[0],sizeof(feather->group_tn)); - } - if(feather->p_table_info[i]->table_type==TABLE_TYPE_COMPILE) - { - strncpy(feather->compile_tn,feather->p_table_info[i]->table_name[0],sizeof(feather->compile_tn)); - } - for(j=0;jp_table_info[i]->conj_cnt;j++) - { - ret=map_register(feather->map_tablename2id,feather->p_table_info[i]->table_name[j],feather->p_table_info[i]->table_id); - if(ret<0) - { - MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module , - "Duplicate table name %s of table id %d" - ,feather->p_table_info[i]->table_name[j] - ,feather->p_table_info[i]->table_id); - continue; - } - } - } + goto failed; } feather->logger=logger; feather->scan_thread_num=max_thread_num; @@ -594,6 +582,9 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* pthread_mutex_init(&(feather->background_update_mutex),NULL); snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path); return feather; +failed: + free(feather); + return NULL; } int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size) { @@ -1369,6 +1360,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id { int region_ret=0,compile_ret=0,hit_region_cnt=0; unsigned int sub_type=0; + int virtual_table_id=0; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _OUTER_scan_status_t* _mid=(struct _OUTER_scan_status_t*)(*mid); @@ -1392,7 +1384,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id clock_gettime(CLOCK_MONOTONIC,&start); } _mid=grab_mid(mid,_feather, thread_num, 0); - p_table=acqurie_table(_feather, table_id,TABLE_TYPE_EXPR); + p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1406,11 +1398,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } if(expr_desc->do_charset_merge==1) { - sub_type=make_sub_type(table_id,CHARSET_NONE,0); + sub_type=make_sub_type(p_table->table_id, CHARSET_NONE,0); } else { - sub_type=make_sub_type(table_id,charset,0); + sub_type=make_sub_type(p_table->table_id, charset,0); } alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); scan_data_t scan_data; @@ -1418,7 +1410,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id scan_data.text_data.tlen=data_len; scan_data.text_data.toffset=0; - struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id]; if(table_rt->origin_rule_num==0) { return 0; @@ -1440,7 +1432,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id if(table_rt->expr.regex_rule_cnt>0) { scan_data.rule_type=RULETYPE_REG; - scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0); + scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE,0); region_ret=rulescan_search(my_scanner->region, thread_num, &scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt); if(region_ret>0) { @@ -1460,7 +1452,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } _mid=grab_mid(mid, _feather, thread_num, 1); struct scan_region_hit_wraper region_hit_wraper; - scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, 0); + scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, virtual_table_id); compile_ret=region_compile(_feather,_mid->inner, ®ion_hit_wraper, result,compile_result,rule_num, @@ -1518,16 +1510,15 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id _compile_result_t compile_result[rule_num]; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct Maat_scanner* my_scanner=NULL; - intval_scan_data.rule_type=RULETYPE_INT; - intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); - intval_scan_data.int_data=intval; + Maat_table_desc* p_table=NULL; struct timespec start,end; if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&start); } - p_table=acqurie_table(_feather,table_id,TABLE_TYPE_INTERVAL); + int virutal_table_id=0; + p_table=acqurie_table(_feather, table_id, TABLE_TYPE_INTERVAL, &virutal_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1538,11 +1529,15 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id { return 0; } - struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id]; if(table_rt->origin_rule_num==0) { return 0; } + intval_scan_data.rule_type=RULETYPE_INT; + intval_scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE, 0); + intval_scan_data.int_data=intval; + alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; @@ -1563,7 +1558,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id } _mid=grab_mid(mid, _feather, thread_num, 1); struct scan_region_hit_wraper region_hit_wraper; - scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_ret, _mid->is_last_region, 0); + scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_ret, _mid->is_last_region, virutal_table_id); compile_ret=region_compile(_feather,_mid->inner, ®ion_hit_wraper, result,compile_result,rule_num, @@ -1613,7 +1608,8 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id { clock_gettime(CLOCK_MONOTONIC,&start); } - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_IP); + int virtual_table_id=0; + p_table=acqurie_table(_feather, table_id, TABLE_TYPE_IP, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1624,7 +1620,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id { return 0; } - struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id]; if(table_rt->origin_rule_num==0) { return 0; @@ -1641,7 +1637,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); ip_scan_data.rule_type=RULETYPE_IPv4; - ip_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); + ip_scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE, 0); switch(addr->addrtype) { case ADDR_TYPE_IPV4: @@ -1685,7 +1681,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id } _mid=grab_mid(mid, _feather, thread_num, 1); struct scan_region_hit_wraper region_hit_wraper; - scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_ret, _mid->is_last_region, 0); + scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_ret, _mid->is_last_region, virtual_table_id); compile_ret=region_compile(_feather,_mid->inner, ®ion_hit_wraper, result,compile_result,rule_num, @@ -1730,8 +1726,9 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, struct Maat_scanner* scanner=NULL; struct Maat_table_desc *p_table=NULL; + int virtual_table_id=0; assert(thread_num<_feather->scan_thread_num); - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR); + p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1742,6 +1739,8 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, struct _stream_para_t* sp=ALLOC(struct _stream_para_t ,1); scanner=_feather->scanner; sp->feather=_feather; + sp->p_real_table=p_table; + sp->virtual_table_id=virtual_table_id; sp->version=_feather->maat_version; sp->process_offset=0; sp->rs_stream_para=NULL; @@ -1749,14 +1748,13 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, { return sp; } - struct Maat_table_runtime* table_rt=scanner->table_rt[table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[sp->p_real_table->table_id]; if(table_rt->origin_rule_num==0) { return sp; } INC_SCANNER_REF(scanner, thread_num); - sp->table_id=table_id; sp->thread_num=thread_num; sp->max_cross_size=expr_desc->cross_cache_size; sp->caching_size=0; @@ -1793,16 +1791,15 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para scan_result_t *region_result; _compile_result_t compile_result[rule_num];//dynamic array scan_data_t region_scan_data; - Maat_table_desc* p_table=NULL; - struct timespec start,end; + struct timespec start,end; if(data==NULL||data_len<=0||scanner==NULL) { return 0; - } - struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id]; + } + struct Maat_table_runtime* table_rt=scanner->table_rt[sp->p_real_table->table_id]; if(sp->feather->perf_on==1) { - clock_gettime(CLOCK_MONOTONIC,&start); + clock_gettime(CLOCK_MONOTONIC, &start); } _mid=grab_mid(mid, sp->feather, sp->thread_num,0); @@ -1810,11 +1807,10 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para { return 0; } - p_table=sp->feather->p_table_info[sp->table_id]; - //table rule num is already judged in Maat_stream_scan_string_start + //table rule num is already in Maat_stream_scan_string_start - if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) + if(sp->p_real_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) { sp->feather->scan_err_cnt++; return -1; @@ -1824,11 +1820,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para *detail_ret=0; if(sp->do_merge==1) { - sub_type=make_sub_type(sp->table_id,CHARSET_NONE,0); + sub_type=make_sub_type(sp->p_real_table->table_id, CHARSET_NONE, 0); } else { - sub_type=make_sub_type(sp->table_id,charset,0); + sub_type=make_sub_type(sp->p_real_table->table_id, charset, 0); } if(sp->max_cross_size>0&&sp->caching_size>0) { @@ -1880,7 +1876,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para if(sp->do_regex==1) { region_scan_data.rule_type=RULETYPE_REG; - region_scan_data.sub_type=make_sub_type(sp->table_id,CHARSET_NONE,0); + region_scan_data.sub_type=make_sub_type(sp->p_real_table->table_id, CHARSET_NONE,0); region_ret=rulescan_searchstream(sp->rs_stream_para, ®ion_scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt); if(region_ret<0) { @@ -1892,7 +1888,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para hit_region_cnt+=region_ret; } } - if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS) + if(hit_region_cnt>0&&sp->p_real_table->table_type==TABLE_TYPE_EXPR_PLUS) { hit_region_cnt=match_district(_mid, region_result, hit_region_cnt, scanner); } @@ -1904,7 +1900,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para } _mid=grab_mid(mid, sp->feather,sp->thread_num, 1); struct scan_region_hit_wraper region_hit_wraper; - scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, 0); + scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, sp->virtual_table_id); compile_ret=region_compile(sp->feather,_mid->inner, ®ion_hit_wraper, @@ -1973,7 +1969,7 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para) struct Maat_table_runtime* table_rt=NULL; if(scanner!=NULL) { - table_rt=scanner->table_rt[sp->table_id]; + table_rt=scanner->table_rt[sp->p_real_table->table_id]; alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1); } @@ -2014,7 +2010,8 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, struct Maat_scanner* scanner=NULL; sfh_instance_t * tmp_fuzzy_handle=NULL; struct Maat_table_desc *p_table=NULL; - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST); + int virtual_table_id=0; + p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -2023,6 +2020,8 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, struct _stream_para_t* sp=ALLOC(struct _stream_para_t, 1); scanner=_feather->scanner; sp->feather=_feather; + sp->p_real_table=p_table; + sp->virtual_table_id=virtual_table_id; sp->version=_feather->maat_version; sp->process_offset=0; if(scanner==NULL) @@ -2038,7 +2037,6 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, } INC_SCANNER_REF(scanner, thread_num); - sp->table_id=table_id; sp->thread_num=thread_num; sp->total_len=total_len; sp->fuzzy_hash_handle=tmp_fuzzy_handle; @@ -2088,7 +2086,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { return 0; } - struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; + struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->p_real_table->table_id]; GIE_handle_t* GIE_handle=table_rt->similar.gie_handle; unsigned long long digest_len=0; char* digest_buff=NULL; @@ -2140,7 +2138,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int } _mid=grab_mid(mid,sp->feather, sp->thread_num,1); struct scan_region_hit_wraper region_hit_wraper; - scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, 0); + scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, sp->virtual_table_id); compile_ret=region_compile(sp->feather,_mid->inner, ®ion_hit_wraper, result,compile_result,rule_num, @@ -2171,7 +2169,7 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct Maat_scanner* scanner=sp->feather->scanner; - struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; + struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->p_real_table->table_id]; alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1); if(scanner!=NULL) { @@ -2274,7 +2272,8 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { clock_gettime(CLOCK_MONOTONIC,&start); } - p_table=acqurie_table(_feather,table_id,TABLE_TYPE_SIMILARITY); + int virtual_table_id=0; + p_table=acqurie_table(_feather, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -2285,7 +2284,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { return 0; } - struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id]; if(table_rt->origin_rule_num==0) { return 0; @@ -2306,7 +2305,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); struct scan_region_hit_wraper region_hit_wraper; - scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, 0); + scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, virtual_table_id); compile_ret=region_compile(_feather,_mid->inner, ®ion_hit_wraper, result,compile_result,rule_num, diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 687cd69..7f091f5 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -34,8 +34,6 @@ int MAAT_FRAME_VERSION_2_7_20190629=1; -const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", - "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type) { @@ -67,8 +65,8 @@ int is_valid_match_method(enum MAAT_MATCH_METHOD match_method) iconv_t maat_iconv_open(struct Maat_scanner* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from) { - const char *from_s=CHARSET_STRING[from]; - const char *to_s=CHARSET_STRING[to]; + const char *from_s=charset_get_name(from); + const char *to_s=charset_get_name(to); iconv_t cd; if(from==CHARSET_GBK&&to==CHARSET_BIG5) { @@ -523,323 +521,6 @@ void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_ } -int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map) -{ - int j=0,ret[4]={0}; - char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0}; - char *token=NULL,*sub_token=NULL,*saveptr; - struct expr_table_desc* p=&(table->expr); - sscanf(line,"%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id) - ,table->table_name[0] - ,table_type - ,src_charset - ,dst_charset - ,merge - ,&(p->cross_cache_size) - ,quick_str_scan); - memset(ret,0,sizeof(ret)); - ret[0]=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type)); - ret[1]=map_str2int(string2int_map,str_tolower(src_charset),(int*)&(p->src_charset)); - ret[2]=map_str2int(string2int_map,str_tolower(merge),&(p->do_charset_merge)); - if(strlen(quick_str_scan)>0) - { - ret[3]=map_str2int(string2int_map,str_tolower(quick_str_scan),&(p->quick_expr_switch)); - } - memset(quick_str_scan,0,sizeof(quick_str_scan)); - - for(j=0;j<4;j++) - { - if(ret[j]<0) - { - return -1; - } - } - j=0; - for (token = dst_charset; ; token= NULL) - { - sub_token= strtok_r(token,"/", &saveptr); - if (sub_token == NULL) - break; - ret[3]=map_str2int(string2int_map,str_tolower(sub_token),(int*)&(p->dst_charset[j])); - if(ret[3]>0) - { - if(p->dst_charset[j]==p->src_charset) - { - p->src_charset_in_dst=TRUE; - } - j++; - } - else - { - return -1; - } - - } - return 0; -} -Maat_table_desc* table_info_new(int max_thread_num) -{ - struct Maat_table_desc*p=ALLOC(struct Maat_table_desc, 1); - p->conj_cnt=1; - return p; -} -void table_info_free(struct Maat_table_desc*p) -{ - free(p); - return; -} -int _read_integer_arrary(char* string, int *array, int size) -{ - char *token=NULL,*sub_token=NULL,*saveptr; - int i=0; - for (token = string, i=0; iplugin); - copy_line=_maat_strdup(line); - ret=get_column_pos(copy_line, COLUMN_PLUGIN_DESCR_JSON, &offset, &len); - if(i<0) - { - goto error_out; - } - if(offset+lenvalid_flag_column)); - if(ret==0||ret==EOF) - { - plugin_desc->valid_flag_column=-1; - } - free(copy_line); - return 0; - } - json=cJSON_Parse(plug_info); - if(!json) - { - goto error_out; - } - tmp=cJSON_GetObjectItem(json, "key"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - plugin_desc->key_column=tmp->valueint; - } - tmp=cJSON_GetObjectItem(json, "valid"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - plugin_desc->valid_flag_column=tmp->valueint; - } - tmp=cJSON_GetObjectItem(json, "tag"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - plugin_desc->rule_tag_column=tmp->valueint; - } - tmp=cJSON_GetObjectItem(json, "estimate_size"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - plugin_desc->estimate_size=tmp->valueint; - } - tmp=cJSON_GetObjectItem(json, "foreign"); - if(tmp!=NULL) - { - if(tmp->type==cJSON_String) - { - plugin_desc->n_foreign=_read_integer_arrary(tmp->valuestring, plugin_desc->foreign_columns, MAX_FOREIGN_CLMN_NUM); - } - else if(tmp->type==cJSON_Array) - { - plugin_desc->n_foreign= cJSON_GetArraySize(tmp); - for(i=0;in_foreign; i++) - { - array_item=cJSON_GetArrayItem(tmp, i); - assert(array_item->type==cJSON_Number); - plugin_desc->foreign_columns[i]=array_item->valueint; - } - } - } - cJSON_Delete(json); - - free(copy_line); - return 0; -error_out: - free(copy_line); - return -1; -} -int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) -{ - FILE*fp=NULL; - char line[MAX_TABLE_LINE_SIZE]; - int i=0,ret=0,table_cnt=0; - char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; - MESA_htable_handle string2int_map=NULL;; - struct Maat_table_desc*p=NULL; - struct Maat_table_desc*conj_table=NULL; - fp=fopen(table_info_path,"r"); - if(fp==NULL) - { - fprintf(stderr,"Maat read table info %s error.\n",table_info_path); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s failed: %s.\n", table_info_path, strerror(errno)); - return 0; - } - - string2int_map=map_create(); - map_register(string2int_map,"expr", TABLE_TYPE_EXPR); - map_register(string2int_map,"ip", TABLE_TYPE_IP); - map_register(string2int_map,"ip_plus", TABLE_TYPE_IP_PLUS); - map_register(string2int_map,"compile", TABLE_TYPE_COMPILE); - map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN); - map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL); - map_register(string2int_map,"digest", TABLE_TYPE_DIGEST); - map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS); - map_register(string2int_map,"group", TABLE_TYPE_GROUP); - map_register(string2int_map,"similar", TABLE_TYPE_SIMILARITY); - map_register(string2int_map,"quickoff",0); - map_register(string2int_map,"quickon",1); - map_register(string2int_map,"escape",USER_REGION_ENCODE_ESCAPE); -// map_register(string2int_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet - - for(i=0;i0) - { - map_register(string2int_map,CHARSET_STRING[i], i); - } - else - { - break; - } - } - - map_register(string2int_map,"yes", 1); - map_register(string2int_map,"no", 0); - - - i=0; - while(NULL!=fgets(line,sizeof(line),fp)) - { - i++; - - if(line[0]=='#'||line[0]==' '||line[0]=='\t'||strlen(line)<4) - { - continue; - } - p=table_info_new(max_thread_num); - - ret=sscanf(line,"%d\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id) - ,p->table_name[0] - ,table_type_str - ,not_care); - if(ret<3) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error: not enough column.",table_info_path,i); - continue; - } - ret=map_str2int(string2int_map,str_tolower(table_type_str),(int*)&(p->table_type)); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:invalid table type.",table_info_path,i); - goto invalid_table; - } - switch(p->table_type) - { - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - ret=read_expr_table_info(line, p, string2int_map); - if(ret<0) - { - fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal column.",table_info_path,i); - goto invalid_table; - } - break; - case TABLE_TYPE_PLUGIN: - ret=read_plugin_table_description(line, p); - if(ret<0) - { - fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal plugin info.",table_info_path,i); - goto invalid_table; - } - break; - case TABLE_TYPE_COMPILE: - ret=sscanf(not_care,"%[a-z0-9]",tmp_str); - if(ret>0) - { - ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding)); - } - if(ret!=1) - { - p->compile.user_region_encoding=USER_REGION_ENCODE_NONE; - } - default: - break; - } - - if(p->table_id>=num) - { - fprintf(stderr,"Maat read table info %s:%d error: table id %uh > %d.\n",table_info_path,i,p->table_id,num); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,num); - - goto invalid_table; - } - if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table; - { - conj_table=p_table_info[p->table_id]; - if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Maat read table info %s line %d error:reach tableid %d conjunction upper limit." - ,table_info_path,i,p->table_id); - goto invalid_table; - } - memcpy(conj_table->table_name[conj_table->conj_cnt],p->table_name[0],MAX_TABLE_NAME_LEN); - conj_table->conj_cnt++; - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module, - "Maat read table info %s:%d:conjunction %s with %s (id=%d,total=%d)." - ,table_info_path,i,p->table_name[0] - ,conj_table->table_name[0],conj_table->table_id,conj_table->conj_cnt); - //use goto to free the conjunctioned table_info - goto invalid_table; - } - - p_table_info[p->table_id]=p; - table_cnt++; - continue; -invalid_table: - table_info_free(p); - p=NULL; - } - fclose(fp); - map_destroy(string2int_map); - return table_cnt; -} struct Maat_group_inner* create_group_rule(int group_id, int table_id, struct Maat_scanner *scanner) { int ret=0; @@ -2222,10 +1903,11 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "Table %s region cfg %d charset convert from %s to %s failed.",table->table_name - ,db_rule->region_id - ,CHARSET_STRING[expr_desc->src_charset] - ,CHARSET_STRING[dst_charset]); + "Table %s region cfg %d charset convert from %s to %s failed.", + table->table_name, + db_rule->region_id, + charset_get_name(expr_desc->src_charset), + charset_get_name(dst_charset)); free(region_string); op_expr->convert_failed++; expr_desc->iconv_err_cnt++; @@ -2736,7 +2418,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru return; } -void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) +void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger) { struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); int ret=0,db_hexbin=0,rule_type=0; @@ -3024,7 +2706,7 @@ int ip_format2range(int ip_type, enum MAAT_IP_FORMAT format, const char* ip1, co } return 0; } -void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger, int group_mode_on) +void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger) { struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); char src_ip1[40]={0}, src_ip2[40]={0}, dst_ip1[40]={0}, dst_ip2[40]={0}; @@ -3233,7 +2915,7 @@ error_out: ip_rule=NULL; } -void update_intval_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) +void update_intval_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger) { struct db_intval_rule* intval_rule=ALLOC(struct db_intval_rule, 1); struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; @@ -3385,7 +3067,7 @@ error_out: return; } -void update_digest_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) +void update_digest_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger) { struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; struct db_digest_rule* digest_rule=ALLOC(struct db_digest_rule, 1); @@ -4008,18 +3690,18 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - update_expr_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); + update_expr_rule(feather->p_table_info[table_id], line, scanner, feather->logger); break; case TABLE_TYPE_IP: case TABLE_TYPE_IP_PLUS: - update_ip_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); + update_ip_rule(feather->p_table_info[table_id], line, scanner, feather->logger); break; case TABLE_TYPE_INTERVAL: - update_intval_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); + update_intval_rule(feather->p_table_info[table_id], line, scanner,feather->logger); break; case TABLE_TYPE_DIGEST: case TABLE_TYPE_SIMILARITY: - update_digest_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); + update_digest_rule(feather->p_table_info[table_id], line, scanner,feather->logger); break; case TABLE_TYPE_COMPILE: update_compile_rule(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); @@ -4207,16 +3889,8 @@ void *thread_rule_monitor(void *arg) garbage_bury(feather->garbage_q,0,feather->logger); assert(0==MESA_lqueue_get_count(feather->garbage_q)); MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL); - int i=0; - for(i=0;ip_table_info[i]==NULL) - { - continue; - } - table_info_free(feather->p_table_info[i]); - feather->p_table_info[i]=NULL; - } + Maat_table_clear(feather->p_table_info, MAX_TABLE_NUM); + alignment_int64_array_free(feather->thread_call_cnt); alignment_int64_array_free(feather->inner_mid_cnt); alignment_int64_array_free(feather->outer_mid_cnt); @@ -4235,6 +3909,7 @@ void *thread_rule_monitor(void *arg) feather->mr_ctx.write_ctx=NULL; } } + int i=0; for(i=0; in_tags; i++) { free(feather->accept_tags[i].tag_name); diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table.cpp new file mode 100644 index 0000000..bfc45e7 --- /dev/null +++ b/src/entry/Maat_table.cpp @@ -0,0 +1,430 @@ +#include "Maat_table.h" +#include "map_str2int.h" +#include "Maat_utils.h" +#include "cJSON.h" + +#include +#include +#include +#include +#include + + +int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map) +{ + int j=0,ret[4]={0}; + char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0}; + char *token=NULL,*sub_token=NULL,*saveptr; + struct expr_table_desc* p=&(table->expr); + sscanf(line,"%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id) + ,table->table_name[0] + ,table_type + ,src_charset + ,dst_charset + ,merge + ,&(p->cross_cache_size) + ,quick_str_scan); + memset(ret,0,sizeof(ret)); + ret[0]=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type)); + ret[1]=map_str2int(string2int_map,str_tolower(src_charset),(int*)&(p->src_charset)); + ret[2]=map_str2int(string2int_map,str_tolower(merge),&(p->do_charset_merge)); + if(strlen(quick_str_scan)>0) + { + ret[3]=map_str2int(string2int_map,str_tolower(quick_str_scan),&(p->quick_expr_switch)); + } + memset(quick_str_scan,0,sizeof(quick_str_scan)); + + for(j=0;j<4;j++) + { + if(ret[j]<0) + { + return -1; + } + } + j=0; + for (token = dst_charset; ; token= NULL) + { + sub_token= strtok_r(token,"/", &saveptr); + if (sub_token == NULL) + break; + ret[3]=map_str2int(string2int_map,str_tolower(sub_token),(int*)&(p->dst_charset[j])); + if(ret[3]>0) + { + if(p->dst_charset[j]==p->src_charset) + { + p->src_charset_in_dst=TRUE; + } + j++; + } + else + { + return -1; + } + + } + return 0; +} +int read_virtual_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map) +{ + int ret=0; + char table_type[16]; + ret=sscanf(line, "%d\t%s\t%s\t%s", &(table->table_id), + table->table_name[0], + table_type, + table->virtual_table.real_table_name); + if(ret!=4) + { + return -1; + } + ret=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type)); + if(ret<0) + { + return -1; + } + return 0; +} +Maat_table_desc* table_info_new(void) +{ + struct Maat_table_desc*p=ALLOC(struct Maat_table_desc, 1); + p->conj_cnt=1; + return p; +} +void table_info_free(struct Maat_table_desc*p) +{ + free(p); + return; +} +int _read_integer_arrary(char* string, int *array, int size) +{ + char *token=NULL,*sub_token=NULL,*saveptr; + int i=0; + for (token = string, i=0; iplugin); + copy_line=_maat_strdup(line); + ret=get_column_pos(copy_line, COLUMN_PLUGIN_DESCR_JSON, &offset, &len); + if(i<0) + { + goto error_out; + } + if(offset+lenvalid_flag_column)); + if(ret==0||ret==EOF) + { + plugin_desc->valid_flag_column=-1; + } + free(copy_line); + return 0; + } + json=cJSON_Parse(plug_info); + if(!json) + { + goto error_out; + } + tmp=cJSON_GetObjectItem(json, "key"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + plugin_desc->key_column=tmp->valueint; + } + tmp=cJSON_GetObjectItem(json, "valid"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + plugin_desc->valid_flag_column=tmp->valueint; + } + tmp=cJSON_GetObjectItem(json, "tag"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + plugin_desc->rule_tag_column=tmp->valueint; + } + tmp=cJSON_GetObjectItem(json, "estimate_size"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + plugin_desc->estimate_size=tmp->valueint; + } + tmp=cJSON_GetObjectItem(json, "foreign"); + if(tmp!=NULL) + { + if(tmp->type==cJSON_String) + { + plugin_desc->n_foreign=_read_integer_arrary(tmp->valuestring, plugin_desc->foreign_columns, MAX_FOREIGN_CLMN_NUM); + } + else if(tmp->type==cJSON_Array) + { + plugin_desc->n_foreign= cJSON_GetArraySize(tmp); + for(i=0;in_foreign; i++) + { + array_item=cJSON_GetArrayItem(tmp, i); + assert(array_item->type==cJSON_Number); + plugin_desc->foreign_columns[i]=array_item->valueint; + } + } + } + cJSON_Delete(json); + + free(copy_line); + return 0; +error_out: + free(copy_line); + return -1; +} +int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_table, const char* table_info_path, void* logger) +{ + FILE*fp=NULL; + char line[MAX_TABLE_LINE_SIZE]; + int i=0,ret=0,table_cnt=0; + char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; + MESA_htable_handle string2int_map=NULL;; + struct Maat_table_desc*p=NULL; + struct Maat_table_desc*conj_table=NULL; + fp=fopen(table_info_path,"r"); + if(fp==NULL) + { + fprintf(stderr,"Maat read table info %s error.\n",table_info_path); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s failed: %s.\n", table_info_path, strerror(errno)); + return 0; + } + + string2int_map=map_create(); + map_register(string2int_map,"expr", TABLE_TYPE_EXPR); + map_register(string2int_map,"ip", TABLE_TYPE_IP); + map_register(string2int_map,"ip_plus", TABLE_TYPE_IP_PLUS); + map_register(string2int_map,"compile", TABLE_TYPE_COMPILE); + map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN); + map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL); + map_register(string2int_map,"digest", TABLE_TYPE_DIGEST); + map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS); + map_register(string2int_map,"group", TABLE_TYPE_GROUP); + map_register(string2int_map,"similar", TABLE_TYPE_SIMILARITY); + map_register(string2int_map,"virtual", TABLE_TYPE_VIRTUAL); + map_register(string2int_map,"quickoff", 0); + map_register(string2int_map,"quickon", 1); + map_register(string2int_map,"escape", USER_REGION_ENCODE_ESCAPE); +// map_register(string2int_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet + + const char** charset_name_list=charset_get_all_name(); + for(i=0;i0) + { + map_register(string2int_map, charset_name_list[i], i); + } + else + { + break; + } + } + + map_register(string2int_map,"yes", 1); + map_register(string2int_map,"no", 0); + + + i=0; + while(NULL!=fgets(line,sizeof(line),fp)) + { + i++; + + if(line[0]=='#'||line[0]==' '||line[0]=='\t'||strlen(line)<4) + { + continue; + } + p=table_info_new(); + + ret=sscanf(line,"%d\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id) + ,p->table_name[0] + ,table_type_str + ,not_care); + if(ret<3) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error: not enough column.",table_info_path,i); + continue; + } + ret=map_str2int(string2int_map,str_tolower(table_type_str),(int*)&(p->table_type)); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:invalid table type.",table_info_path,i); + goto invalid_table; + } + switch(p->table_type) + { + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + ret=read_expr_table_info(line, p, string2int_map); + if(ret<0) + { + fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:illegal column.",table_info_path,i); + goto invalid_table; + } + break; + case TABLE_TYPE_PLUGIN: + ret=read_plugin_table_description(line, p); + if(ret<0) + { + fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:illegal plugin info.",table_info_path,i); + goto invalid_table; + } + break; + case TABLE_TYPE_VIRTUAL: + ret=read_virtual_table_info(line, p, string2int_map); + if(ret<0) + { + fprintf(stderr,"Maat read table info %s line %d error:illegal virtual info.\n",table_info_path,i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:illegal virtual info.",table_info_path,i); + goto invalid_table; + } + break; + case TABLE_TYPE_COMPILE: + ret=sscanf(not_care,"%[a-z0-9]",tmp_str); + if(ret>0) + { + ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding)); + } + if(ret!=1) + { + p->compile.user_region_encoding=USER_REGION_ENCODE_NONE; + } + default: + break; + } + + if((unsigned int)p->table_id>=n_table) + { + fprintf(stderr,"Maat read table info %s:%d error: table id %uh > %zu.\n",table_info_path,i,p->table_id,n_table); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,n_table); + + goto invalid_table; + } + if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table; + { + conj_table=p_table_info[p->table_id]; + if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Maat read table info %s line %d error:reach tableid %d conjunction upper limit." + ,table_info_path,i,p->table_id); + goto invalid_table; + } + memcpy(conj_table->table_name[conj_table->conj_cnt],p->table_name[0],MAX_TABLE_NAME_LEN); + conj_table->conj_cnt++; + MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module, + "Maat read table info %s:%d:conjunction %s with %s (id=%d,total=%d)." + ,table_info_path,i,p->table_name[0] + ,conj_table->table_name[0],conj_table->table_id,conj_table->conj_cnt); + //use goto to free the conjunctioned table_info + goto invalid_table; + } + + p_table_info[p->table_id]=p; + table_cnt++; + continue; +invalid_table: + table_info_free(p); + p=NULL; + } + fclose(fp); + map_destroy(string2int_map); + return table_cnt; +} +MESA_htable_handle Maat_table_build(struct Maat_table_desc** p_table_info, size_t n_table, char* compile_tn, size_t n_ctn, char* group_tn, size_t n_gtn, void* logger) +{ + MESA_htable_handle map_tablename2id=map_create(); + size_t i=0; + int j=0, ret=0; + for(i=0;itable_type) + { + case TABLE_TYPE_GROUP: + strncpy(group_tn, p_table_info[i]->table_name[0], n_gtn); + break; + case TABLE_TYPE_COMPILE: + strncpy(compile_tn, p_table_info[i]->table_name[0], n_ctn); + break; + case TABLE_TYPE_VIRTUAL: + ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id)); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Undefined real table %s, virtual table %s of table id %d.", + p_table_info[i]->virtual_table.real_table_name, + p_table_info[i]->table_name[j], + p_table_info[i]->table_id); + goto failed; + } + break; + default: + break; + } + + + for(j=0; jconj_cnt; j++) + { + ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Duplicate table %s of table id %d", + p_table_info[i]->table_name[j], + p_table_info[i]->table_id); + continue; + } + } + + } + return map_tablename2id; +failed: + map_destroy(map_tablename2id); + return NULL; +} +void Maat_table_clear(struct Maat_table_desc** p_table_info, size_t n_table) +{ + size_t i=0; + for(i=0;i #include @@ -27,17 +27,7 @@ extern const char *maat_module; #define mr_group_id_var "SEQUENCE_GROUP" -#define MAX_TABLE_NUM 256 -#define MAX_TABLE_LINE_SIZE (1024*16) -#define MAX_EXPR_KEYLEN 1024 -#define MAX_DISTRICT_LEN 64 -#define MAX_SCANNER_HIT_NUM 64 -#define MAX_GROUP_CACHE 128 - -#define MAX_FAILED_NUM 128 - -#define MAX_MAAT_STAT_NUM 64 typedef void* rule_scanner_t; @@ -210,11 +200,12 @@ struct iconv_handle_t struct _stream_para_t { struct _Maat_feather_t* feather; + const struct Maat_table_desc* p_real_table; + int virtual_table_id; int version; int thread_num; int max_cross_size; int caching_size; - unsigned short table_id; char do_merge; char do_expr:4; char do_regex:4; @@ -355,7 +346,6 @@ struct _Maat_feather_t MESA_lqueue_head garbage_q; int table_cnt; int DEFERRED_LOAD_ON; - int GROUP_MODE_ON; int REDIS_MODE_ON; enum data_source input_mode; union @@ -471,7 +461,6 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void make_group_set(struct Maat_compile_group_relation* compile_rule, struct bool_expr* a_set, unsigned char *has_not); -int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); void maat_start_cb(long long new_version,int update_type,void*u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para); void maat_finish_cb(void* u_para); @@ -517,6 +506,3 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m _Maat_feather_t* feather); - -#endif - diff --git a/src/inc_internal/Maat_table_description.h b/src/inc_internal/Maat_table.h similarity index 84% rename from src/inc_internal/Maat_table_description.h rename to src/inc_internal/Maat_table.h index 249b3ac..e60d887 100644 --- a/src/inc_internal/Maat_table_description.h +++ b/src/inc_internal/Maat_table.h @@ -1,5 +1,8 @@ #pragma once +#include #include "Maat_rule.h" +#include "Maat_limits.h" + #define MAX_COMPILE_EX_DATA_NUM 2 #define MAX_FOREIGN_CLMN_NUM 8 #define MAX_PLUGIN_PER_TABLE 32 @@ -7,6 +10,7 @@ #define MAX_CONJUNCTION_TABLE_NUM 8 #define MAX_TABLE_NAME_LEN 256 + enum USER_REGION_ENCODE { USER_REGION_ENCODE_NONE=0, @@ -93,6 +97,7 @@ struct ip_table_desc struct virtual_table_desc { int real_table_id; + char real_table_name[MAX_TABLE_NAME_LEN]; }; struct Maat_table_desc { @@ -115,4 +120,7 @@ struct Maat_table_desc unsigned long long unmatch_tag_cnt; int stat_line_id; }; +int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_table, const char* table_info_path, void* logger); +MESA_htable_handle Maat_table_build(struct Maat_table_desc** p_table_info, size_t n_table, char* compile_tn, size_t n_ctn, char* group_tn, size_t n_gtn, void* logger); +void Maat_table_clear(struct Maat_table_desc** p_table_info, size_t n_table); diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index e8d7114..167048d 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -1,4 +1,5 @@ #pragma once +#include "Maat_rule.h" #include #include #include @@ -69,5 +70,7 @@ int system_cmd_mv(const char* src_file,const char*dst_file); int system_cmd_cp(const char* src_file,const char*dst_file); char* md5_file(const char* filename, char* md5string); int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len); +const char** charset_get_all_name(void); +const char* charset_get_name(enum MAAT_CHARSET charset); From 4c4222a302f6642f7afaeb37a62946da49f17015 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Sun, 28 Jul 2019 11:45:57 +0600 Subject: [PATCH 04/29] =?UTF-8?q?=E5=B0=86scanner=E4=B8=AD=E7=9A=84?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E6=80=81=E6=95=B0=E6=8D=AE=E6=8B=86=E5=88=86?= =?UTF-8?q?=E5=88=B0Maat=5Ftable=5Fruntime.cpp=E4=B8=AD=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/CMakeLists.txt | 2 +- src/entry/Maat_api.cpp | 373 +++++--------------- src/entry/Maat_command.cpp | 41 +-- src/entry/Maat_rule.cpp | 404 ++++----------------- src/entry/Maat_stat.cpp | 19 +- src/entry/Maat_table.cpp | 482 ++++++++++++++++++++++---- src/entry/Maat_table_runtime.cpp | 427 +++++++++++++++++++++++ src/entry/Maat_utils.cpp | 8 +- src/inc_internal/Maat_limits.h | 1 - src/inc_internal/Maat_rule_internal.h | 61 +--- src/inc_internal/Maat_table.h | 82 +++-- src/inc_internal/Maat_table_runtime.h | 69 ++++ src/inc_internal/Maat_utils.h | 1 + 13 files changed, 1166 insertions(+), 804 deletions(-) create mode 100644 src/entry/Maat_table_runtime.cpp create mode 100644 src/inc_internal/Maat_table_runtime.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index abce127..1ef76dd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) -set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp) +set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) include_directories(/opt/MESA/include/MESA/) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 494055c..abcc421 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -16,42 +16,7 @@ #include "rulescan.h" #include "json2iris.h" -struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id) -{ - struct Maat_table_desc *p_table=NULL, *p_real_table=NULL; - if(table_id>MAX_TABLE_NUM) - { - return NULL; - } - if(_feather->p_table_info[table_id]==NULL) - { - return NULL; - } - p_table=_feather->p_table_info[table_id]; - if(p_table==NULL) - { - return NULL; - } - if(p_table->table_type==TABLE_TYPE_VIRTUAL) - { - p_real_table=_feather->p_table_info[p_table->virtual_table.real_table_id]; - *virutal_table_id=table_id; - } - else - { - p_real_table=p_table; - *virutal_table_id=0; - } - if(p_real_table->table_type!=expect_type) - { - if((expect_type==TABLE_TYPE_EXPR && p_table->table_type!=TABLE_TYPE_EXPR_PLUS)|| - (expect_type==TABLE_TYPE_IP && p_table->table_type!=TABLE_TYPE_IP_PLUS)) - { - return NULL; - } - } - return p_real_table; -} + inline void INC_SCANNER_REF(Maat_scanner*scanner,int thread_num) { alignment_int64_array_add(scanner->ref_cnt, thread_num, 1); @@ -547,19 +512,14 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* return NULL; } _Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1); - feather->table_cnt=Maat_table_read_table_info(feather->p_table_info, MAX_TABLE_NUM, table_info_path, logger); - if(feather->table_cnt==0) - { - goto failed; - } - feather->map_tablename2id=Maat_table_build(feather->p_table_info, MAX_TABLE_NUM, - feather->compile_tn, sizeof(feather->compile_tn), - feather->group_tn, sizeof(feather->group_tn), - logger); - if(feather->map_tablename2id==NULL) + feather->table_mgr=Maat_table_manager_create(table_info_path, logger); + if(feather->table_mgr==NULL) { goto failed; } + Maat_table_get_compile_table_name(feather->table_mgr, feather->compile_tn, sizeof(feather->compile_tn)); + Maat_table_get_group_table_name(feather->table_mgr, feather->group_tn, sizeof(feather->group_tn)); + feather->logger=logger; feather->scan_thread_num=max_thread_num; feather->garbage_q=MESA_lqueue_create(0,0); @@ -994,16 +954,7 @@ void Maat_burn_feather(Maat_feather_t feather) int Maat_table_register(Maat_feather_t feather,const char* table_name) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - int table_id=-1,ret=0; - ret=map_str2int(_feather->map_tablename2id, table_name, &table_id); - if(ret>0) - { - return table_id; - } - else - { - return -1; - } + return Maat_table_get_id_by_name(_feather->table_mgr, table_name); } int Maat_table_callback_register(Maat_feather_t feather,short table_id, Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para @@ -1012,52 +963,35 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, void* u_para) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - int idx=0,i=0; - struct Maat_table_desc *p_table=_feather->p_table_info[table_id]; - struct plugin_table_desc *plugin_desc=&(p_table->plugin); - struct plugin_runtime* plugin_aux=NULL; - const char* lines=NULL; - if(p_table==NULL) - { - return -1; - } - if(p_table->table_type!=TABLE_TYPE_PLUGIN) - { - return -1; - } - //plugin table register blocks background update. + int i=0,ret=0; + pthread_mutex_lock(&(_feather->background_update_mutex)); - idx=plugin_desc->cb_plug_cnt; - if(idx==MAX_PLUGIN_PER_TABLE) + ret=Maat_table_add_callback_func(_feather->table_mgr, table_id, start, update, finish, u_para); + if(ret<0) { pthread_mutex_unlock(&(_feather->background_update_mutex)); return -1; } - plugin_desc->cb_plug_cnt++; - plugin_desc->cb_plug[idx].start=start; - plugin_desc->cb_plug[idx].update=update; - plugin_desc->cb_plug[idx].finish=finish; - plugin_desc->cb_plug[idx].u_para=u_para; - if(_feather->scanner==NULL) - { - pthread_mutex_unlock(&(_feather->background_update_mutex)); - return 1; - } - plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin); - if(plugin_aux->cache_line_num>0) + + const char* line=NULL; + struct Maat_table_runtime* table_rt=NULL; + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + long long line_cnt=Maat_table_runtime_plugin_cached_line_count(table_rt); + + if(line_cnt>0) { if(start!=NULL) { start(MAAT_RULE_UPDATE_TYPE_FULL,u_para); } - for(i=0;icache_line_num;i++) + for(i=0; icache_lines,i); - if(lines==NULL) + line=Maat_table_runtime_plugin_get_cached_line(table_rt, i); + if(line==NULL) { break; } - update(table_id,lines,u_para); + update(table_id,line,u_para); } if(finish!=NULL) { @@ -1095,39 +1029,31 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table long argl, void *argp) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - int table_id=-1,ret=0, idx=-1; - ret=map_str2int(_feather->map_tablename2id, compile_table_name, &table_id); - if(ret<0) + int idx=-1; + + if(new_func==NULL || free_func==NULL || dup_func==NULL) { return -1; } - struct Maat_table_desc *p_table=_feather->p_table_info[table_id]; - if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL || dup_func==NULL) - { - return -1; - } - struct compile_table_desc* compile_desc=&(p_table->compile); pthread_mutex_lock(&(_feather->background_update_mutex)); - if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM) - { - ret=-1; - goto failed; - } - idx=compile_desc->ex_data_num; - compile_desc->ex_desc[idx].idx=idx; - compile_desc->ex_desc[idx].table_id=table_id; - compile_desc->ex_desc[idx].argl=argl; - compile_desc->ex_desc[idx].argp=argp; - compile_desc->ex_desc[idx].new_func=new_func; - compile_desc->ex_desc[idx].free_func=free_func; - compile_desc->ex_desc[idx].dup_func=dup_func; + idx=Maat_table_new_compile_rule_ex_index(_feather->table_mgr, compile_table_name, + new_func, + free_func, + dup_func, + argl, argp); - compile_desc->ex_data_num++; + if(idx<0) + { + pthread_mutex_unlock(&(_feather->background_update_mutex)); + return -1; + } + struct compile_ex_data_idx* compile_ex_desc=Maat_table_get_compile_rule_ex_desc(_feather->table_mgr, compile_table_name, idx); + if(_feather->scanner!=NULL) { - MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_desc->ex_desc+idx); + MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_ex_desc); } -failed: + pthread_mutex_unlock(&(_feather->background_update_mutex)); return idx; @@ -1155,129 +1081,7 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa pthread_rwlock_unlock(&(relation->rwlock)); return ad; } -struct wrap_plugin_EX_data -{ - MAAT_RULE_EX_DATA exdata; - const struct Maat_table_desc* ref_plugin_table; -}; -void wrap_plugin_EX_data_free(void *data) -{ - struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data; - const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc); - ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp); - wrap_data->ref_plugin_table=NULL; - free(wrap_data); - return; -} -MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index) -{ - MESA_htable_handle key2ex_hash=NULL; - unsigned int slot_size=1; - while(estimate_size!=0) - { - estimate_size=estimate_size>>1; - slot_size*=2; - } - if(slot_size==1) - { - slot_size=4096; - } - MESA_htable_create_args_t hargs; - memset(&hargs,0,sizeof(hargs)); - hargs.thread_safe=8; - hargs.hash_slot_size = slot_size; - hargs.max_elem_num = 0; - hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; - hargs.expire_time = 0; - hargs.key_comp = NULL; - hargs.key2index = NULL; //Not supported yet. - hargs.recursive = 1; - hargs.data_free = wrap_plugin_EX_data_free; - hargs.data_expire_with_condition = NULL; - key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); - MESA_htable_print_crtl(key2ex_hash, 0); - return key2ex_hash; -} - -int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, - MESA_htable_handle key2ex_hash, void *logger) -{ - size_t key_offset=0, key_len=0; - const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); - int ret=0; - ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data del error: cannot find column %d of %s", - plugin_desc->key_column, line); - return -1; - } - ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data del error: no such key %.*s of %s", - key_len, line+key_offset, line); - return -1; - } - return 0; -} - -int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, - MESA_htable_handle key2ex_hash, void *logger) -{ - char* key=NULL; - size_t key_offset=0, key_len=0; - MAAT_RULE_EX_DATA exdata=NULL; - struct wrap_plugin_EX_data* wrap_data=NULL; - const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); - int ret=0; - ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data add error: cannot find column %d of %s", - plugin_desc->key_column, line); - return -1; - } - key=ALLOC(char, key_len+1); - memcpy(key, line+key_offset, key_len); - plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata, - plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); - wrap_data=ALLOC(struct wrap_plugin_EX_data, 1); - wrap_data->exdata=exdata; - wrap_data->ref_plugin_table=plugin_table; - ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data); - free(key); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data add error: duplicated key %.*s of %s", - key_len, line+key_offset, line); - wrap_plugin_EX_data_free(wrap_data); - return -1; - } - return 0; -} -MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table, - struct dynamic_array_t* lines, size_t line_cnt, void* logger) -{ - MESA_htable_handle key2ex_hash=NULL; - size_t i=0; - const char* line=NULL; - const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); - - key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func); - - for(i=0; i< line_cnt; i++) - { - line=(const char*)dynamic_array_read(lines, i); - plugin_EX_data_new(plugin_table, line, key2ex_hash, logger); - } - return key2ex_hash; -} int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, Maat_plugin_EX_new_func_t* new_func, Maat_plugin_EX_free_func_t* free_func, @@ -1287,39 +1091,35 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; - struct plugin_table_desc* plugin_desc=&(table_desc->plugin); - struct Maat_table_runtime* table_rt=NULL; + int idx=-1; + if(new_func==NULL || free_func==NULL || dup_func==NULL ) { assert(0); MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); return -1; } - if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata - || plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0) - { - assert(0); - MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, - "%s failed: key or valid flag column are not specified", __FUNCTION__); - return -1; - } - pthread_mutex_lock(&(_feather->background_update_mutex)); - plugin_desc->ex_desc.new_func=new_func; - plugin_desc->ex_desc.free_func=free_func; - plugin_desc->ex_desc.dup_func=dup_func; - plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used. - plugin_desc->ex_desc.argl=argl; - plugin_desc->ex_desc.argp=argp; - plugin_desc->have_exdata=1; + idx=Maat_table_plugin_new_ex_index(_feather->table_mgr, table_id, + new_func, + free_func, + dup_func, + key2index_func, + argl, argp); + if(idx<0) + { + pthread_mutex_unlock(&(_feather->background_update_mutex)); + return -1; + } + struct Maat_table_desc *table_desc=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + struct Maat_table_runtime* table_rt=NULL; + + if(_feather->scanner!=NULL) { - table_rt=_feather->scanner->table_rt[table_id]; - assert(table_rt->plugin.key2ex_hash==NULL); - table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines, - table_rt->plugin.cache_line_num, _feather->logger); + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + Maat_table_runtime_plugin_new_ex_idx(table_rt, table_desc, _feather->logger); } pthread_mutex_unlock(&(_feather->background_update_mutex)); @@ -1328,28 +1128,16 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key) { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; - struct Maat_table_runtime *table_rt= NULL; - struct plugin_table_desc* plugin_desc=&(table_desc->plugin); - struct wrap_plugin_EX_data* wrap_data=NULL; + struct Maat_table_desc *table_desc=NULL; + struct Maat_table_runtime *table_rt=NULL; MAAT_RULE_EX_DATA exdata=NULL; - if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata==0) - { - assert(0); - return NULL; - } if(_feather->scanner==NULL) { return NULL; } - table_rt= _feather->scanner->table_rt[table_id]; - wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash, - (const unsigned char*)key, strlen(key)); - if(wrap_data!=NULL) - { - plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata), - plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); - } + table_desc=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key); return exdata; } @@ -1384,7 +1172,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id clock_gettime(CLOCK_MONOTONIC,&start); } _mid=grab_mid(mid,_feather, thread_num, 0); - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR, &virtual_table_id); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1410,7 +1198,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id scan_data.text_data.tlen=data_len; scan_data.text_data.toffset=0; - struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); if(table_rt->origin_rule_num==0) { return 0; @@ -1518,7 +1306,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id clock_gettime(CLOCK_MONOTONIC,&start); } int virutal_table_id=0; - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_INTERVAL, &virutal_table_id); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_INTERVAL, &virutal_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1529,7 +1317,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id { return 0; } - struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); if(table_rt->origin_rule_num==0) { return 0; @@ -1609,7 +1397,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id clock_gettime(CLOCK_MONOTONIC,&start); } int virtual_table_id=0; - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_IP, &virtual_table_id); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_IP, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1620,7 +1408,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id { return 0; } - struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); if(table_rt->origin_rule_num==0) { return 0; @@ -1728,7 +1516,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, struct Maat_table_desc *p_table=NULL; int virtual_table_id=0; assert(thread_num<_feather->scan_thread_num); - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR, &virtual_table_id); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1748,7 +1536,8 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, { return sp; } - struct Maat_table_runtime* table_rt=scanner->table_rt[sp->p_real_table->table_id]; + + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id); if(table_rt->origin_rule_num==0) { return sp; @@ -1796,7 +1585,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para { return 0; } - struct Maat_table_runtime* table_rt=scanner->table_rt[sp->p_real_table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id); if(sp->feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC, &start); @@ -1969,7 +1758,7 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para) struct Maat_table_runtime* table_rt=NULL; if(scanner!=NULL) { - table_rt=scanner->table_rt[sp->p_real_table->table_id]; + table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id); alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1); } @@ -2011,7 +1800,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, sfh_instance_t * tmp_fuzzy_handle=NULL; struct Maat_table_desc *p_table=NULL; int virtual_table_id=0; - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST, &virtual_table_id); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_DIGEST, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -2028,7 +1817,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, { return sp; } - struct Maat_table_runtime* table_rt=scanner->table_rt[table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_id); tmp_fuzzy_handle=SFH_instance(total_len); if(tmp_fuzzy_handle==NULL) { @@ -2086,7 +1875,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { return 0; } - struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->p_real_table->table_id]; + struct Maat_table_runtime *table_rt=Maat_table_runtime_get(sp->feather->scanner->table_rt_mgr, sp->p_real_table->table_id); GIE_handle_t* GIE_handle=table_rt->similar.gie_handle; unsigned long long digest_len=0; char* digest_buff=NULL; @@ -2169,7 +1958,7 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct Maat_scanner* scanner=sp->feather->scanner; - struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->p_real_table->table_id]; + struct Maat_table_runtime *table_rt=Maat_table_runtime_get(sp->feather->scanner->table_rt_mgr, sp->p_real_table->table_id); alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1); if(scanner!=NULL) { @@ -2273,7 +2062,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id clock_gettime(CLOCK_MONOTONIC,&start); } int virtual_table_id=0; - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -2284,7 +2073,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { return 0; } - struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); if(table_rt->origin_rule_num==0) { return 0; @@ -2356,7 +2145,7 @@ int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* valu } break; case MAAT_STATE_LAST_UPDATING_TABLE: - *int_val=_feather->is_last_plugin_table_updating; + *int_val=Maat_table_manager_is_last_plugin_table_updating(_feather->table_mgr); break; case MAAT_STATE_IN_UPDATING: if(size!=sizeof(int)) diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index a86d303..16d206b 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -915,7 +915,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _ continue; } region_cmd=&(group_cmd->regions[j]); - region_cmd->table_name=_maat_strdup(feather->p_table_info[region_inner->table_id]->table_name[0]); + region_cmd->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, region_inner->table_id)); region_cmd->region_id=region_inner->region_id; //NOTICE: region_type only avilable when OP_ADD, region_cmd->region_type=REGION_EXPR; @@ -1374,7 +1374,7 @@ error_out: int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) { - int i=0,j=0,ret=0; + int i=0, j=0; const char *table_name=NULL; int table_id=0; struct Maat_group_t* p_group=NULL; @@ -1399,8 +1399,8 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) { p_region=&(p_group->regions[j]); table_name=p_region->table_name; - ret=map_str2int(feather->map_tablename2id, table_name, &table_id); - if(ret<0) + table_id=Maat_table_get_id_by_name(feather->table_mgr, table_name); + if(table_id<0) { MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module ,"Unknown table %s of Maat_cmd_t[%d]->group[%d]->region[%d]." @@ -1409,7 +1409,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) return -1; } table_type=type_region2table(p_region); - if(table_type!=feather->p_table_info[table_id]->table_type) + if(table_type!=Maat_table_get_type_by_id(feather->table_mgr, table_id)) { MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module ,"Table %s not support region type %d of Maat_cmd_t[%d]->group[%d]->region[%d]." @@ -1419,7 +1419,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) return -1; } free((char*)p_region->table_name); - p_region->table_name=_maat_strdup(feather->p_table_info[table_id]->table_name[0]); + p_region->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, table_id)); } } return 0; @@ -1651,7 +1651,7 @@ void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n } int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger) { - int ret=0, table_id=0, i=0; + int i=0; int rule_with_foreign_key=0; struct Maat_table_desc* p_table=NULL; struct plugin_table_desc* plugin_desc=NULL; @@ -1661,14 +1661,13 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, { continue; } - ret=map_str2int(feather->map_tablename2id, rule_list[i].table_name, &table_id); - if(ret<0) + p_table=Maat_table_get_desc_by_name(feather->table_mgr, rule_list[i].table_name); + if(!p_table||p_table->table_type!=TABLE_TYPE_PLUGIN) { continue; } - p_table=feather->p_table_info[table_id]; plugin_desc= &(p_table->plugin); - if(p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->n_foreign==0) + if(plugin_desc->n_foreign==0) { continue; } @@ -1808,6 +1807,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m int update_type=CM_UPDATE_TYPE_INC; long long new_version=0; enum MAAT_TABLE_TYPE table_type; + struct Maat_table_desc* table_desc=NULL; const struct plugin_table_desc* plugin_desc=NULL; void* logger=feather->logger; @@ -1889,17 +1889,18 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m { continue; } - ret=map_str2int(feather->map_tablename2id,rule_list[i].table_name,&table_id); - if(ret<0)//Unrecognized table. + table_id=Maat_table_get_id_by_name(feather->table_mgr, rule_list[i].table_name); + if(table_id<0)//Unrecognized table. { continue; } - table_type=feather->p_table_info[table_id]->table_type; + table_type=Maat_table_get_type_by_id(feather->table_mgr, table_id); if(rule_list[i].op==MAAT_OP_DEL) { if(table_type==TABLE_TYPE_PLUGIN) { - plugin_desc=&(feather->p_table_info[table_id]->plugin); + table_desc=Maat_table_get_by_id(feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + plugin_desc=&(table_desc->plugin); valid_column=plugin_desc->valid_flag_column; } else @@ -2105,8 +2106,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru s_rule=(struct serial_rule_t *)calloc(sizeof(struct serial_rule_t),line_num); for(i=0;imap_tablename2id, line_rule[i]->table_name, &table_id); - if(ret<0) + table_id=Maat_table_get_id_by_name(_feather->table_mgr, line_rule[i]->table_name); + if(table_id<0) { MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command ,"Command set line id %d failed: unknown table %s." @@ -2115,9 +2116,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru ret=-1; goto error_out; } - p_table=_feather->p_table_info[table_id]; - plugin_desc=&(p_table->plugin); - if(TABLE_TYPE_PLUGIN!=p_table->table_type) + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + if(!p_table) { MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command ,"Command set line id %d failed: table %s is not a plugin table." @@ -2126,6 +2126,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru ret=-1; goto error_out; } + plugin_desc=&(p_table->plugin); if(op==MAAT_OP_ADD) { ret=get_valid_flag_offset(line_rule[i]->table_line diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 7f091f5..7c12f23 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -479,11 +480,7 @@ error_out: cJSON_Delete(json); return ret; } -int lqueue_destroy_cb(void *data, long data_len, void *arg) -{ - assert(0); - return 0; -} + void * HASH_fetch_by_id(MESA_htable_handle hash,int id) { return MESA_htable_search(hash,(unsigned char*)&(id),sizeof(id)); @@ -938,127 +935,10 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule) op_expr->rule_type=p_rule->rule_type; return; } -GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest, - short cfds_lvl,struct Maat_group_inner* tag) -{ - GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1); - int digest_len=0; - rule->id=id; - rule->operation=op; - if(digest!=NULL) - { - digest_len=strlen(digest); - rule->sfh=(char*)calloc(sizeof(char),digest_len+1); - memcpy(rule->sfh,digest,digest_len); - - } - rule->sfh_length=digest_len; - rule->cfds_lvl=cfds_lvl; - rule->tag=(void*)tag; - return rule; -} -void destroy_digest_rule(GIE_digest_t*rule) -{ - if(rule->sfh!=NULL) - { - free(rule->sfh); - rule->sfh=NULL; - } - free(rule); - rule=NULL; - return; -} - - -struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num) -{ - - struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1); - table_rt->table_type=table_desc->table_type; - switch(table_desc->table_type) - { - case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - table_rt->similar.update_q=MESA_lqueue_create(0,0); - break; - case TABLE_TYPE_PLUGIN: - table_rt->plugin.cache_lines=dynamic_array_create(1, 1024); - if(table_desc->plugin.have_exdata) - { - table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, - table_desc->plugin.ex_desc.key2index_func); - } - break; - default: - break; - } - - table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num); - table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); - table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num); - table_rt->stream_num=alignment_int64_array_alloc(max_thread_num); - table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num); - return table_rt; -} -void table_runtime_free(struct Maat_table_runtime* p) -{ - long q_cnt=0,data_size=0; - int i=0; - UNUSED int q_ret=0; - - GIE_digest_t* digest_rule=NULL; - if(p==NULL) - { - return; - } - switch(p->table_type) - { - case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - if(p->similar.gie_handle!=NULL) - { - GIE_destory(p->similar.gie_handle); - } - if(p->similar.update_q!=NULL) - { - q_cnt=MESA_lqueue_get_count(p->similar.update_q); - for(i=0;isimilar.update_q,&digest_rule,&data_size); - assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); - destroy_digest_rule(digest_rule); - } - MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL); - } - break; - case TABLE_TYPE_PLUGIN: - dynamic_array_destroy(p->plugin.cache_lines, free); - p->plugin.cache_lines=NULL; - if(p->plugin.key2ex_hash!=NULL) - { - MESA_htable_destroy(p->plugin.key2ex_hash, NULL); - } - default: - break; - } - - alignment_int64_array_free(p->scan_cnt); - alignment_int64_array_free(p->scan_cpu_time); - alignment_int64_array_free(p->input_bytes); - alignment_int64_array_free(p->stream_num); - alignment_int64_array_free(p->hit_cnt); - free(p); - return; -} struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *feather) { int scan_thread_num=feather->scan_thread_num; -// int rs_scan_type=feather->rule_scan_type; - struct Maat_table_desc ** pp_table_desc=feather->p_table_info; - struct Maat_table_runtime* table_rt=NULL; - int i=0; UNUSED int ret=0; MESA_htable_create_args_t hargs; @@ -1115,8 +995,8 @@ struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *f scanner->region_update_q=MESA_lqueue_create(0, 0); scanner->region=rulescan_initialize(scan_thread_num); - //For best performance test: - //1.Do NOT set this option,rulescan return no hit detail as default; + //For best scan performance: + //1.Do NOT set this option, rulescan return no hit detail as default; //2.Set necessary STR rule to QUICK; if(feather->rule_scan_type==1) { @@ -1130,16 +1010,8 @@ struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *f scanner->tomb_ref=feather->garbage_q; scanner->logger_ref=feather->logger; scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num); - - for(i=0;iscan_thread_num); - scanner->table_rt[i]=table_rt; - } + scanner->table_rt_mgr=Maat_table_runtime_manager_create(feather->table_mgr, feather->scan_thread_num); + scanner->max_table_num=Maat_table_manager_get_size(feather->table_mgr); return scanner; } @@ -1189,10 +1061,9 @@ void destroy_maat_scanner(struct Maat_scanner*scanner) } } } - for(i=0;itable_rt[i]); - } + Maat_table_rt_manager_destroy(scanner->table_rt_mgr); + scanner->table_rt_mgr=NULL; + igraph_destroy(&scanner->group_graph); free(scanner); return; @@ -1235,9 +1106,8 @@ struct _region_stat_t int ipv6_rule_cnt; }; }; -void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_stat, int size) +void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_stat, size_t size) { - assert(op_expr->table_idp_expr->operation==0)//add { @@ -1277,7 +1147,7 @@ void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_sta void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,void*logger,struct Maat_scanner* maat_scanner) { - long i=0,data_size=0; + long data_size=0, i=0; unsigned int j=0; int ret=0; unsigned int failed_ids[MAX_FAILED_NUM]; @@ -1288,7 +1158,8 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, const long q_cnt=MESA_lqueue_get_count(expr_queue); struct timespec start,end; unsigned long long update_interval=0; - struct _region_stat_t region_counter[MAX_TABLE_NUM]; + size_t max_table_num=maat_scanner->max_table_num; + struct _region_stat_t region_counter[max_table_num]; memset(region_counter, 0, sizeof(region_counter)); struct Maat_table_runtime* table_rt=NULL; if(q_cnt==0) @@ -1317,7 +1188,7 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, } } - count_rs_region(op_expr,region_counter,MAX_TABLE_NUM); + count_rs_region(op_expr, region_counter, max_table_num); destroy_op_expr(op_expr); op_expr=NULL; } @@ -1344,9 +1215,9 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, ,(double)update_interval ,update_interval); //update scanner's region cnt; - for(i=0;itable_rt[i]; + table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, i); if(table_rt==NULL) { continue; @@ -1384,52 +1255,6 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, free(to_update_expr); } -void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*logger,struct Maat_scanner* maat_scanner,int table_id) -{ - long i=0,data_size=0; - int ret=0; - GIE_digest_t* digest_rule=NULL; - GIE_digest_t** update_array=NULL; - UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; - const long q_cnt=MESA_lqueue_get_count(update_q); - if(q_cnt==0) - { - return; - } - struct Maat_table_runtime* table_rt=maat_scanner->table_rt[table_id]; - update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt); - for(i=0;ioperation==GIE_INSERT_OPT) - { - table_rt->origin_rule_num++; - } - else - { - table_rt->origin_rule_num--; - } - destroy_digest_rule(update_array[i]); - update_array[i]=NULL; - } - free(update_array); - update_array=NULL; - return; -} struct region_group_relation { int region_id; @@ -2049,34 +1874,25 @@ int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule* intval_ MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); return 0; } -int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule* db_digest_rule,struct Maat_scanner *scanner,void* logger) +int add_digest_rule(struct Maat_table_desc* table, struct db_digest_rule* db_rule, struct Maat_scanner *scanner,void* logger) { struct Maat_group_inner* group_rule=NULL; - GIE_digest_t* digest_rule=NULL; struct Maat_group_inner* u_para=NULL; - struct Maat_table_runtime * table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime * table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); int expr_id=0,district_id=-1; - group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); + group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id); if(group_rule==NULL) { - group_rule=create_group_rule(db_digest_rule->group_id, 0, scanner); + group_rule=create_group_rule(db_rule->group_id, 0, scanner); } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule, table->table_id, db_digest_rule->region_id, district_id, expr_id, TABLE_TYPE_DIGEST, scanner); + u_para=add_region_to_group(group_rule, table->table_id, db_rule->region_id, district_id, expr_id, TABLE_TYPE_DIGEST, scanner); if(u_para==NULL) { return -1; } - if(table->table_type==TABLE_TYPE_SIMILARITY) - { - db_digest_rule->digest_string=str_unescape(db_digest_rule->digest_string); - } - digest_rule=create_digest_rule(expr_id, GIE_INSERT_OPT - ,db_digest_rule->digest_string - ,db_digest_rule->confidence_degree - ,group_rule); - MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*)); + Maat_table_runtime_digest_add(table_rt, expr_id, db_rule->digest_string, db_rule->confidence_degree, group_rule); scanner->gie_update_q_size++; return 0; } @@ -2086,8 +1902,8 @@ int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0}; int expr_num=0; struct Maat_group_inner* group_rule=NULL; + struct Maat_table_runtime* table_rt=NULL; struct op_expr_t* op_expr=NULL; - GIE_digest_t* digest_rule=NULL; group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(maat_scanner->group_hash, group_id); if(group_rule==NULL) { @@ -2126,11 +1942,8 @@ int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int case TABLE_TYPE_SIMILARITY: case TABLE_TYPE_DIGEST: assert(expr_num==1); - digest_rule=create_digest_rule(expr_id[0], GIE_DELETE_OPT //del digest - ,NULL - ,0 - ,NULL); - MESA_lqueue_join_tail(maat_scanner->table_rt[table->table_id]->similar.update_q,&digest_rule, sizeof(void*)); + table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, table->table_id); + Maat_table_runtime_digest_del(table_rt, expr_id[0]); maat_scanner->gie_update_q_size++; break; default: @@ -2331,10 +2144,10 @@ int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_ scanner->to_update_compile_cnt++; return 1; } -void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner, MESA_htable_handle map_tablename2id, void* logger) +void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner, struct Maat_table_manager* table_mgr, void* logger) { struct db_group_rule_t db_group_rule; - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); int ret=0; char virtual_table_name[MAX_TABLE_NAME_LEN]={0}; memset(&db_group_rule, 0, sizeof(db_group_rule)); @@ -2364,10 +2177,10 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru table->udpate_err_cnt++; return; } - if(strlen(virtual_table_name)>0) + if(strlen(virtual_table_name)>0&&!strcasecmp(virtual_table_name, "null")) { - ret=map_str2int(map_tablename2id, virtual_table_name, &(db_group_rule.virtual_table_id)); - if(ret<0) + db_group_rule.virtual_table_id=Maat_table_get_id_by_name(table_mgr, virtual_table_name); + if(db_group_rule.virtual_table_id<0) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , "update error, unknown virutal table name: %s of group table %s:%s.", @@ -2381,7 +2194,8 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru { ret=del_group_rule(table, &db_group_rule, scanner, logger); //leave no trace when compatible_group_update calling - if(table->table_type==TABLE_TYPE_GROUP&&ret==1) + assert(table->table_type==TABLE_TYPE_GROUP); + if(ret==1) { table_rt->origin_rule_num--; assert(table_rt->origin_rule_num>=0); @@ -2422,7 +2236,7 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc { struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); int ret=0,db_hexbin=0,rule_type=0; - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); switch(table->table_type) { case TABLE_TYPE_EXPR: @@ -2711,7 +2525,7 @@ void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struc struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); char src_ip1[40]={0}, src_ip2[40]={0}, dst_ip1[40]={0}, dst_ip2[40]={0}; char saddr_format[16]={0}, sport_format[16]={0}, daddr_format[16]={0}, dport_format[16]={0}; - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); unsigned short src_port1=0, src_port2=0, dst_port1=0, dst_port2=0; int protocol=0,direction=0; int ret=0; @@ -2918,7 +2732,7 @@ error_out: void update_intval_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger) { struct db_intval_rule* intval_rule=ALLOC(struct db_intval_rule, 1); - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); int ret=0; ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id) ,&(intval_rule->group_id) @@ -2984,7 +2798,7 @@ error_out: void update_compile_rule(struct Maat_table_desc* table,const char* table_line ,struct Maat_scanner *scanner, const struct rule_tag* tags, int n_tags,void* logger) { struct compile_table_desc* compile_desc=&(table->compile); - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); struct Maat_compile_rule *p_compile=NULL; struct Maat_rule_head m_rule_tmp; @@ -3069,7 +2883,7 @@ error_out: void update_digest_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger) { - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); struct db_digest_rule* digest_rule=ALLOC(struct db_digest_rule, 1); int ret=0; char digest_buff[MAX_TABLE_LINE_SIZE]={'\0'}; @@ -3270,7 +3084,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Ma int i=0, ret=1, matched_tag=1; unsigned int len=strlen(table_line)+1; struct plugin_table_desc* plugin_desc=&(table->plugin); - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); char *p=NULL; char* copy=NULL; size_t is_valid_offset=0, valid_len=0; @@ -3413,11 +3227,7 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, struct bool_matcher *tmp1=NULL,*tmp2=NULL; MESA_htable_handle tmp_map=NULL; struct Maat_table_runtime* table_rt=NULL; - int i=0; - long q_cnt; - GIE_create_para_t para; - para.gram_value=7; - para.position_accuracy=10; + int i=0, ret=0; igraph_bool_t is_dag; igraph_is_dag(&(scanner->group_graph), &is_dag); if(!is_dag) @@ -3451,9 +3261,9 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, scanner->region_update_q, logger, scanner); - for(i=0;imax_table_num; i++) { - table_rt=scanner->table_rt[i]; + table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, i); if(table_rt==NULL) { continue; @@ -3462,30 +3272,13 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, { case TABLE_TYPE_DIGEST: case TABLE_TYPE_SIMILARITY: - q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q); - if(q_cnt==0) + + ret=Maat_table_runtime_digest_batch_udpate(table_rt); + if(ret<0) { - continue; + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "GIE_update error."); } - if(table_rt->similar.gie_handle==NULL) - { - if(table_rt->table_type==TABLE_TYPE_SIMILARITY) - { - para.ED_reexamine=1; - para.format=GIE_INPUT_FORMAT_PLAIN; - } - else - { - para.ED_reexamine=0; - para.format=GIE_INPUT_FORMAT_SFH; - } - table_rt->similar.gie_handle=GIE_create(¶); - } - digest_batch_update(table_rt->similar.gie_handle, - table_rt->similar.update_q, - logger, - scanner, - i); break; case TABLE_TYPE_PLUGIN: break; @@ -3509,12 +3302,9 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, } -void maat_start_cb(long long new_version,int update_type,void*u_para) +void maat_start_cb(long long new_version, int update_type, void*u_para) { - struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - struct Maat_table_desc* p_table=NULL; - struct plugin_table_desc* plugin_desc=NULL; - int i=0,j=0; + struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; feather->new_version=new_version; if(update_type==CM_UPDATE_TYPE_FULL) @@ -3532,26 +3322,7 @@ void maat_start_cb(long long new_version,int update_type,void*u_para) feather->maat_version,new_version); feather->maat_version=new_version; } - feather->active_plugin_table_num=0; - for(i=0;ip_table_info[i]; - plugin_desc=&(p_table->plugin); - if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->cb_plug_cnt==0) - { - continue; - } - - feather->active_plugin_table_num++; - - for(j=0;jcb_plug_cnt;j++) - { - if(plugin_desc->cb_plug[j].start!=NULL) - { - plugin_desc->cb_plug[j].start(update_type,plugin_desc->cb_plug[j].u_para); - } - } - } + Maat_table_manager_all_plugin_cb_start(feather->table_mgr, update_type); return; } long long scanner_rule_num(struct Maat_scanner *scanner) @@ -3559,9 +3330,9 @@ long long scanner_rule_num(struct Maat_scanner *scanner) long long total=0; struct Maat_table_runtime* table_rt=NULL; int i=0; - for(i=0;imax_table_num; i++) { - table_rt=scanner->table_rt[i]; + table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, i); if(table_rt!=NULL) { total+=table_rt->origin_rule_num; @@ -3572,42 +3343,9 @@ long long scanner_rule_num(struct Maat_scanner *scanner) void maat_finish_cb(void* u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - struct Maat_table_desc* p_table=NULL; - struct plugin_table_desc* plugin_desc=NULL; long expr_wait_q_cnt=0; - int i=0, j=0; - - int call_plugin_table_cnt=0; - for(i=0;ip_table_info[i]; - if(p_table==NULL) - { - continue; - } - switch(p_table->table_type) - { - case TABLE_TYPE_PLUGIN: - plugin_desc=&(p_table->plugin); - call_plugin_table_cnt++; - if(call_plugin_table_cnt==feather->active_plugin_table_num) - { - feather->is_last_plugin_table_updating=1; - } - for(j=0;jcb_plug_cnt;j++) - { - if(plugin_desc->cb_plug[j].finish!=NULL) - { - plugin_desc->cb_plug[j].finish(plugin_desc->cb_plug[j].u_para); - } - } - feather->is_last_plugin_table_updating=0; - break; - default: - break; - } - } + Maat_table_manager_all_plugin_cb_finish(feather->table_mgr); if(feather->update_tmp_scanner!=NULL) { @@ -3651,14 +3389,11 @@ void maat_finish_cb(void* u_para) feather->maat_version); } feather->new_version=-1; - feather->active_plugin_table_num=0; return; } int maat_update_cb(const char* table_name,const char* line,void *u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - int ret=-1,i=0; - int table_id=-1; Maat_scanner* scanner=NULL; struct Maat_table_desc* p_table=NULL; if(feather->update_tmp_scanner!=NULL) @@ -3670,47 +3405,39 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) scanner=feather->scanner; } // MESA_handle_runtime_log(feather->logger, RLOG_LV_DEBUG, maat_module, "Maat table %s input: %s", table_name, line); - ret=map_str2int(feather->map_tablename2id,table_name,&table_id); - if(ret<0) + p_table=Maat_table_get_desc_by_name(feather->table_mgr, table_name); + if(!p_table) { - MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s",table_name); + MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s", table_name); return -1; } - p_table=feather->p_table_info[table_id]; - for(i=0;iconj_cnt;i++) - { - if(0==memcmp(p_table->table_name[i],table_name,strlen(table_name))) - { - p_table->updating_name=i; - } - } - assert(i<=p_table->conj_cnt); + Maat_table_set_updating_name(p_table, table_name); - switch(feather->p_table_info[table_id]->table_type) + switch(p_table->table_type) { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - update_expr_rule(feather->p_table_info[table_id], line, scanner, feather->logger); + update_expr_rule(p_table, line, scanner, feather->logger); break; case TABLE_TYPE_IP: case TABLE_TYPE_IP_PLUS: - update_ip_rule(feather->p_table_info[table_id], line, scanner, feather->logger); + update_ip_rule(p_table, line, scanner, feather->logger); break; case TABLE_TYPE_INTERVAL: - update_intval_rule(feather->p_table_info[table_id], line, scanner,feather->logger); + update_intval_rule(p_table, line, scanner,feather->logger); break; case TABLE_TYPE_DIGEST: case TABLE_TYPE_SIMILARITY: - update_digest_rule(feather->p_table_info[table_id], line, scanner,feather->logger); + update_digest_rule(p_table, line, scanner,feather->logger); break; case TABLE_TYPE_COMPILE: - update_compile_rule(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); + update_compile_rule(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); break; case TABLE_TYPE_GROUP: - update_group_rule(feather->p_table_info[table_id], line, scanner, feather->map_tablename2id, feather->logger); + update_group_rule(p_table, line, scanner, feather->table_mgr, feather->logger); break; case TABLE_TYPE_PLUGIN: - update_plugin_table(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); + update_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); default: break; @@ -3884,12 +3611,11 @@ void *thread_rule_monitor(void *arg) } } - MESA_htable_destroy(feather->map_tablename2id,free); + Maat_table_manager_destroy(feather->table_mgr); destroy_maat_scanner(feather->scanner); garbage_bury(feather->garbage_q,0,feather->logger); assert(0==MESA_lqueue_get_count(feather->garbage_q)); MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL); - Maat_table_clear(feather->p_table_info, MAX_TABLE_NUM); alignment_int64_array_free(feather->thread_call_cnt); alignment_int64_array_free(feather->inner_mid_cnt); diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index f0573eb..bcf3bfb 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -1,4 +1,5 @@ #include "Maat_rule_internal.h" +#include "Maat_table.h" #include "alignment_int64.h" #include #include @@ -121,10 +122,11 @@ void maat_stat_init(struct _Maat_feather_t* feather) FS_STYLE_COLUMN, FS_CALC_SPEED, "hit_rate"); - feather->total_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT,"Sum"); - for(i=0;itotal_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT, "Sum"); + size_t max_table_num=Maat_table_manager_get_size(feather->table_mgr); + for(i=0; i<(int)max_table_num; i++) { - p_table=feather->p_table_info[i]; + p_table=Maat_table_get_by_id_raw(feather->table_mgr, i); if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN ||p_table->table_type==TABLE_TYPE_GROUP ||p_table->table_type==TABLE_TYPE_COMPILE) @@ -185,7 +187,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,Maat_table_manager_get_count(feather->table_mgr)); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0,FS_OP_SET,not_grp_hit_cnt); @@ -197,7 +200,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value); feather->update_err_cnt=0; feather->iconv_err_cnt=0; - for(i=0;itable_mgr); + for(i=0; i<(int)max_table_num; i++) { table_stream_num=0; table_scan_cnt=0; @@ -205,12 +209,13 @@ void maat_stat_output(struct _Maat_feather_t* feather) table_scan_cpu_time=0; table_hit_cnt=0; table_regex_ipv6_num=0; - p_table=feather->p_table_info[i]; + + p_table=Maat_table_get_by_id_raw(feather->table_mgr, i); if(p_table==NULL) { continue; } - table_rt=feather->scanner->table_rt[i]; + table_rt=Maat_table_runtime_get(feather->scanner->table_rt_mgr, i); switch(p_table->table_type) { case TABLE_TYPE_PLUGIN: diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table.cpp index bfc45e7..5cd8a64 100644 --- a/src/entry/Maat_table.cpp +++ b/src/entry/Maat_table.cpp @@ -9,6 +9,16 @@ #include #include +#define MAX_TABLE_NUM 256 + +struct Maat_table_manager +{ + struct Maat_table_desc* p_table_info[MAX_TABLE_NUM]; + size_t table_cnt; + MESA_htable_handle map_tablename2id; + int active_plugin_table_num; + int is_last_plugin_table_updating; +}; int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map) { @@ -192,11 +202,85 @@ error_out: free(copy_line); return -1; } -int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_table, const char* table_info_path, void* logger) + +static int Maat_table_build_map(struct Maat_table_manager* table_mgr, void* logger) { + struct Maat_table_desc** p_table_info=table_mgr->p_table_info; + size_t n_table=MAX_TABLE_NUM; + + MESA_htable_handle map_tablename2id=map_create(); + size_t i=0; + int j=0, ret=0; + for(i=0;itable_type) + { + case TABLE_TYPE_VIRTUAL: + ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id)); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Undefined real table %s, virtual table %s of table id %d.", + p_table_info[i]->virtual_table.real_table_name, + p_table_info[i]->table_name[j], + p_table_info[i]->table_id); + goto failed; + } + break; + default: + break; + } + + + for(j=0; jconj_cnt; j++) + { + ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Duplicate table %s of table id %d", + p_table_info[i]->table_name[j], + p_table_info[i]->table_id); + continue; + } + } + + } + table_mgr->map_tablename2id=map_tablename2id; + return 0; +failed: + map_destroy(map_tablename2id); + return -1; +} + +void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr) +{ + size_t i=0; + for(i=0;ip_table_info[i]==NULL) + { + continue; + } + table_info_free(table_mgr->p_table_info[i]); + table_mgr->p_table_info[i]=NULL; + } + MESA_htable_destroy(table_mgr->map_tablename2id, free); + free(table_mgr); + return; +} + +struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger) +{ + struct Maat_table_manager* table_mgr=NULL; FILE*fp=NULL; char line[MAX_TABLE_LINE_SIZE]; - int i=0,ret=0,table_cnt=0; + int i=0, ret=0; char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; MESA_htable_handle string2int_map=NULL;; struct Maat_table_desc*p=NULL; @@ -207,9 +291,12 @@ int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_t fprintf(stderr,"Maat read table info %s error.\n",table_info_path); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, "Maat read table info %s failed: %s.\n", table_info_path, strerror(errno)); - return 0; + return NULL; } - + table_mgr=ALLOC(struct Maat_table_manager, 1); + struct Maat_table_desc** p_table_info=table_mgr->p_table_info; + size_t n_table=MAX_TABLE_NUM; + string2int_map=map_create(); map_register(string2int_map,"expr", TABLE_TYPE_EXPR); map_register(string2int_map,"ip", TABLE_TYPE_IP); @@ -348,83 +435,350 @@ int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_t } p_table_info[p->table_id]=p; - table_cnt++; + table_mgr->table_cnt++; continue; invalid_table: table_info_free(p); p=NULL; } fclose(fp); - map_destroy(string2int_map); - return table_cnt; -} -MESA_htable_handle Maat_table_build(struct Maat_table_desc** p_table_info, size_t n_table, char* compile_tn, size_t n_ctn, char* group_tn, size_t n_gtn, void* logger) -{ - MESA_htable_handle map_tablename2id=map_create(); - size_t i=0; - int j=0, ret=0; - for(i=0;itable_cnt; +} +int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) +{ + int i=0; + for(i=0; i< MAX_TABLE_NUM; i++) + { + if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_COMPILE) + { + strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz); + return 1; + } + } + return 0; +} +int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) +{ + int i=0; + for(i=0; i< MAX_TABLE_NUM; i++) + { + if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP) + { + strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz); + return 1; + } + } + return 0; +} +const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id) +{ + if(table_id>MAX_TABLE_NUM) + { + return NULL; + } + if(table_mgr->p_table_info[table_id]) + { + return table_mgr->p_table_info[table_id]->table_name[0]; + } + return NULL; +} +enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id) +{ + if(table_id>MAX_TABLE_NUM) + { + return TABLE_TYPE_INVALID; + } + if(table_mgr->p_table_info[table_id]) + { + return table_mgr->p_table_info[table_id]->table_type; + } + return TABLE_TYPE_INVALID; + +} +struct Maat_table_desc * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id) +{ + if(table_id>MAX_TABLE_NUM) + { + return NULL; + } + + return table_mgr->p_table_info[table_id]; +} + +struct Maat_table_desc * Maat_table_get_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id) +{ + + struct Maat_table_desc **p_table_info=table_mgr->p_table_info; + size_t n_table=MAX_TABLE_NUM; + + struct Maat_table_desc *p_table=NULL, *p_real_table=NULL; + if((unsigned int) table_id>n_table) + { + return NULL; + } + if(p_table_info[table_id]==NULL) + { + return NULL; + } + p_table=p_table_info[table_id]; + if(p_table==NULL) + { + return NULL; + } + if(p_table->table_type==TABLE_TYPE_VIRTUAL) + { + p_real_table=p_table_info[p_table->virtual_table.real_table_id]; + *virutal_table_id=table_id; + } + else + { + p_real_table=p_table; + if(virutal_table_id) *virutal_table_id=0; + } + if(p_real_table->table_type!=expect_type) + { + if((expect_type==TABLE_TYPE_EXPR && p_table->table_type!=TABLE_TYPE_EXPR_PLUS)|| + (expect_type==TABLE_TYPE_IP && p_table->table_type!=TABLE_TYPE_IP_PLUS)) + { + return NULL; + } + } + return p_real_table; +} +int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name) +{ + int table_id=-1,ret=0; + ret=map_str2int(table_mgr->map_tablename2id, table_name, &table_id); + if(ret>0) + { + return table_id; + } + else + { + return -1; + } +} +int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, + int table_id, + Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para + Maat_update_callback_t *update,//table line ,u_para + Maat_finish_callback_t *finish,//u_para + void* u_para) +{ + int idx=0; + struct Maat_table_desc *p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + struct plugin_table_desc *plugin_desc=&(p_table->plugin); + if(p_table==NULL) + { + return -1; + } + + idx=plugin_desc->cb_plug_cnt; + if(idx==MAX_PLUGIN_PER_TABLE) + { + return -1; + } + plugin_desc->cb_plug_cnt++; + plugin_desc->cb_plug[idx].start=start; + plugin_desc->cb_plug[idx].update=update; + plugin_desc->cb_plug[idx].finish=finish; + plugin_desc->cb_plug[idx].u_para=u_para; + return 1; +} + +struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx) +{ + int table_id=-1; + struct Maat_table_desc *p_table=NULL; + + table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name); + if(table_id<0) + { + return NULL; + } + p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL); + if(!p_table) + { + return NULL; + } + if(idxcompile.ex_data_num) + { + return p_table->compile.ex_desc+idx; + } + return NULL; + +} +int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name, + Maat_rule_EX_new_func_t *new_func, + Maat_rule_EX_free_func_t* free_func, + Maat_rule_EX_dup_func_t* dup_func, + long argl, void *argp) +{ + int table_id=-1; + struct Maat_table_desc *p_table=NULL; + table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name); + if(table_id<0) + { + return -1; + } + p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL); + if(!p_table) + { + return -1; + } + int idx=-1; + + struct compile_table_desc* compile_desc=&(p_table->compile); + if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM) + { + return -1; + } + idx=compile_desc->ex_data_num; + compile_desc->ex_desc[idx].idx=idx; + compile_desc->ex_desc[idx].table_id=table_id; + compile_desc->ex_desc[idx].argl=argl; + compile_desc->ex_desc[idx].argp=argp; + compile_desc->ex_desc[idx].new_func=new_func; + compile_desc->ex_desc[idx].free_func=free_func; + compile_desc->ex_desc[idx].dup_func=dup_func; + + compile_desc->ex_data_num++; + + return idx; +} +int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp) + +{ + struct Maat_table_desc *table_desc=NULL;; + table_desc=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + struct plugin_table_desc* plugin_desc=&(table_desc->plugin); + + if(plugin_desc->have_exdata + || plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0) + { + return -1; + } + plugin_desc->ex_desc.new_func=new_func; + plugin_desc->ex_desc.free_func=free_func; + plugin_desc->ex_desc.dup_func=dup_func; + plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used. + plugin_desc->ex_desc.argl=argl; + plugin_desc->ex_desc.argp=argp; + plugin_desc->have_exdata=1; + return 0; +} +void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type) +{ + table_mgr->active_plugin_table_num=0; + int i=0, j=0; + struct Maat_table_desc* p_table=NULL; + struct plugin_table_desc* plugin_desc=NULL; + + for(i=0; ip_table_info[i]; + plugin_desc=&(p_table->plugin); + if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->cb_plug_cnt==0) { continue; } + + table_mgr->active_plugin_table_num++; - switch(p_table_info[i]->table_type) + for(j=0;jcb_plug_cnt;j++) { - case TABLE_TYPE_GROUP: - strncpy(group_tn, p_table_info[i]->table_name[0], n_gtn); - break; - case TABLE_TYPE_COMPILE: - strncpy(compile_tn, p_table_info[i]->table_name[0], n_ctn); - break; - case TABLE_TYPE_VIRTUAL: - ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id)); - if(ret<0) + if(plugin_desc->cb_plug[j].start!=NULL) + { + plugin_desc->cb_plug[j].start(update_type, plugin_desc->cb_plug[j].u_para); + } + } + } + +} +void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr) +{ + int i=0, j=0; + struct Maat_table_desc* p_table=NULL; + struct plugin_table_desc* plugin_desc=NULL; + + int call_plugin_table_cnt=0; + for(i=0;ip_table_info[i]; + if(p_table==NULL) + { + continue; + } + switch(p_table->table_type) + { + case TABLE_TYPE_PLUGIN: + plugin_desc=&(p_table->plugin); + call_plugin_table_cnt++; + if(call_plugin_table_cnt==table_mgr->active_plugin_table_num) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Undefined real table %s, virtual table %s of table id %d.", - p_table_info[i]->virtual_table.real_table_name, - p_table_info[i]->table_name[j], - p_table_info[i]->table_id); - goto failed; + table_mgr->is_last_plugin_table_updating=1; } + for(j=0;jcb_plug_cnt;j++) + { + if(plugin_desc->cb_plug[j].finish!=NULL) + { + plugin_desc->cb_plug[j].finish(plugin_desc->cb_plug[j].u_para); + } + } + table_mgr->is_last_plugin_table_updating=0; break; default: break; - } - - - for(j=0; jconj_cnt; j++) - { - ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Duplicate table %s of table id %d", - p_table_info[i]->table_name[j], - p_table_info[i]->table_id); - continue; - } - } - - } - return map_tablename2id; -failed: - map_destroy(map_tablename2id); - return NULL; -} -void Maat_table_clear(struct Maat_table_desc** p_table_info, size_t n_table) -{ - size_t i=0; - for(i=0;iactive_plugin_table_num=0; return; } +int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr) +{ + return table_mgr->is_last_plugin_table_updating; +} +struct Maat_table_desc* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name) +{ + struct Maat_table_desc * p_table=NULL; + int table_id=0; + table_id=Maat_table_get_id_by_name(table_mgr, table_name); + if(table_id<0) + { + return NULL; + } + p_table=table_mgr->p_table_info[table_id]; + return p_table; +} +void Maat_table_set_updating_name(struct Maat_table_desc* p_table, const char* table_name) +{ + int i=0; + for(i=0; iconj_cnt; i++) + { + if(0==strcmp(p_table->table_name[i], table_name)) + { + p_table->updating_name=i; + } + } + assert(i<=p_table->conj_cnt); +} + diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp new file mode 100644 index 0000000..5b9dbc0 --- /dev/null +++ b/src/entry/Maat_table_runtime.cpp @@ -0,0 +1,427 @@ +#include "Maat_table_runtime.h" +#include "Maat_rule.h" +#include "Maat_utils.h" + +#include +#include +#include + +struct wrap_plugin_EX_data +{ + MAAT_RULE_EX_DATA exdata; + const struct Maat_table_desc* ref_plugin_table; +}; +void wrap_plugin_EX_data_free(void *data) +{ + struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data; + const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc); + ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp); + wrap_data->ref_plugin_table=NULL; + free(wrap_data); + return; +} +MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index) +{ + MESA_htable_handle key2ex_hash=NULL; + unsigned int slot_size=1; + while(estimate_size!=0) + { + estimate_size=estimate_size>>1; + slot_size*=2; + } + if(slot_size==1) + { + slot_size=4096; + } + + MESA_htable_create_args_t hargs; + memset(&hargs,0,sizeof(hargs)); + hargs.thread_safe=8; + hargs.hash_slot_size = slot_size; + hargs.max_elem_num = 0; + hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; + hargs.expire_time = 0; + hargs.key_comp = NULL; + hargs.key2index = NULL; //Not supported yet. + hargs.recursive = 1; + hargs.data_free = wrap_plugin_EX_data_free; + hargs.data_expire_with_condition = NULL; + key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(key2ex_hash, 0); + return key2ex_hash; +} + +int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, + MESA_htable_handle key2ex_hash, void *logger) +{ + size_t key_offset=0, key_len=0; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + int ret=0; + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data del error: cannot find column %d of %s", + plugin_desc->key_column, line); + return -1; + } + ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data del error: no such key %.*s of %s", + key_len, line+key_offset, line); + return -1; + } + return 0; +} + +int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, + MESA_htable_handle key2ex_hash, void *logger) +{ + char* key=NULL; + size_t key_offset=0, key_len=0; + MAAT_RULE_EX_DATA exdata=NULL; + struct wrap_plugin_EX_data* wrap_data=NULL; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + int ret=0; + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data add error: cannot find column %d of %s", + plugin_desc->key_column, line); + return -1; + } + key=ALLOC(char, key_len+1); + memcpy(key, line+key_offset, key_len); + plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata, + plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); + wrap_data=ALLOC(struct wrap_plugin_EX_data, 1); + wrap_data->exdata=exdata; + wrap_data->ref_plugin_table=plugin_table; + ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data); + free(key); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data add error: duplicated key %.*s of %s", + key_len, line+key_offset, line); + wrap_plugin_EX_data_free(wrap_data); + return -1; + } + return 0; +} + +struct Maat_table_runtime_manager +{ + struct Maat_table_runtime** table_rt; + size_t n_table_rt; +}; +static GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest, + short cfds_lvl, void* tag) +{ + GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1); + int digest_len=0; + rule->id=id; + rule->operation=op; + if(digest!=NULL) + { + digest_len=strlen(digest); + rule->sfh=(char*)calloc(sizeof(char),digest_len+1); + memcpy(rule->sfh,digest,digest_len); + + } + rule->sfh_length=digest_len; + rule->cfds_lvl=cfds_lvl; + rule->tag=tag; + return rule; +} +static void destroy_digest_rule(GIE_digest_t*rule) +{ + if(rule->sfh!=NULL) + { + free(rule->sfh); + rule->sfh=NULL; + } + free(rule); + rule=NULL; + return; +} + +static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num) +{ + + struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1); + table_rt->table_type=table_desc->table_type; + switch(table_desc->table_type) + { + case TABLE_TYPE_DIGEST: + case TABLE_TYPE_SIMILARITY: + table_rt->similar.update_q=MESA_lqueue_create(0,0); + break; + case TABLE_TYPE_PLUGIN: + table_rt->plugin.cache_lines=dynamic_array_create(1, 1024); + if(table_desc->plugin.have_exdata) + { + table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, + table_desc->plugin.ex_desc.key2index_func); + } + break; + default: + break; + } + + table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num); + table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); + table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num); + table_rt->stream_num=alignment_int64_array_alloc(max_thread_num); + table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num); + return table_rt; +} +static void table_runtime_free(struct Maat_table_runtime* p) +{ + long q_cnt=0,data_size=0; + int i=0; + UNUSED int q_ret=0; + + GIE_digest_t* digest_rule=NULL; + if(p==NULL) + { + return; + } + switch(p->table_type) + { + case TABLE_TYPE_DIGEST: + case TABLE_TYPE_SIMILARITY: + if(p->similar.gie_handle!=NULL) + { + GIE_destory(p->similar.gie_handle); + } + if(p->similar.update_q!=NULL) + { + q_cnt=MESA_lqueue_get_count(p->similar.update_q); + for(i=0;isimilar.update_q,&digest_rule,&data_size); + assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); + destroy_digest_rule(digest_rule); + } + MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL); + } + break; + case TABLE_TYPE_PLUGIN: + dynamic_array_destroy(p->plugin.cache_lines, free); + p->plugin.cache_lines=NULL; + if(p->plugin.key2ex_hash!=NULL) + { + MESA_htable_destroy(p->plugin.key2ex_hash, NULL); + } + default: + break; + } + + alignment_int64_array_free(p->scan_cnt); + alignment_int64_array_free(p->scan_cpu_time); + alignment_int64_array_free(p->input_bytes); + alignment_int64_array_free(p->stream_num); + alignment_int64_array_free(p->hit_cnt); + free(p); + return; +} + +struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num) +{ + const struct Maat_table_desc* table_desc=NULL; + struct Maat_table_runtime* table_rt=NULL; + struct Maat_table_runtime_manager* table_rt_mgr=ALLOC(struct Maat_table_runtime_manager, 1); + size_t i=0; + table_rt_mgr->n_table_rt=Maat_table_manager_get_size(table_manager); + table_rt_mgr->table_rt=ALLOC(struct Maat_table_runtime*, table_rt_mgr->n_table_rt); + for(i=0; in_table_rt; i++) + { + table_desc=Maat_table_get_by_id_raw(table_manager, i); + if(!table_desc) + { + continue; + } + table_rt=table_runtime_new(table_desc, max_thread_num); + table_rt_mgr->table_rt[i]=table_rt; + } + return table_rt_mgr; +} +void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr) +{ + size_t i=0; + for(i=0; in_table_rt; i++) + { + table_runtime_free(table_rt_mgr->table_rt[i]); + table_rt_mgr->table_rt[i]=NULL; + } + free(table_rt_mgr->table_rt); + table_rt_mgr->table_rt=NULL; + free(table_rt_mgr); +} +struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id) +{ + assert(table_id<(int)table_rt_mgr->n_table_rt); + return table_rt_mgr->table_rt[table_id]; +} +long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt) +{ + struct plugin_runtime* plugin_rt=&(table_rt->plugin); + return plugin_rt->cache_line_num; +} +const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line) +{ + const char* line=NULL; + struct plugin_runtime* plugin_rt=&(table_rt->plugin); + + line=(const char*)dynamic_array_read(plugin_rt->cache_lines, Nth_line); + return line; +} +MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table, + struct dynamic_array_t* lines, size_t line_cnt, void* logger) +{ + MESA_htable_handle key2ex_hash=NULL; + size_t i=0; + const char* line=NULL; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + + key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func); + + for(i=0; i< line_cnt; i++) + { + line=(const char*)dynamic_array_read(lines, i); + plugin_EX_data_new(plugin_table, line, key2ex_hash, logger); + } + return key2ex_hash; +} +int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, void* logger) +{ + assert(table_rt->plugin.key2ex_hash==NULL); + if(table_rt->plugin.key2ex_hash) + { + return -1; + } + table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines, + table_rt->plugin.cache_line_num, logger); + return 0; +} +MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, const char* key) +{ + struct plugin_table_desc* plugin_desc=NULL; + struct wrap_plugin_EX_data* wrap_data=NULL; + MAAT_RULE_EX_DATA exdata=NULL; + + plugin_desc=&(table_desc->plugin); + if(!plugin_desc->have_exdata) + { + assert(0); + return NULL; + } + wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash, + (const unsigned char*)key, strlen(key)); + if(wrap_data!=NULL) + { + plugin_desc->ex_desc.dup_func(table_desc->table_id, &(exdata), &(wrap_data->exdata), + plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); + } + return exdata; + +} + +void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag) +{ + GIE_digest_t* digest_rule=NULL; + char *dup_digest=_maat_strdup(digest); + + if(table_rt->table_type==TABLE_TYPE_SIMILARITY) + { + dup_digest=str_unescape(dup_digest); + } + digest_rule=create_digest_rule(expr_id, GIE_INSERT_OPT, + dup_digest, + confidence_degree, + tag); + MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*)); + return; +} +void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id) +{ + GIE_digest_t* digest_rule=NULL; + digest_rule=create_digest_rule(expr_id, GIE_DELETE_OPT //del digest + ,NULL + ,0 + ,NULL); + MESA_lqueue_join_tail(table_rt->similar.update_q,&digest_rule, sizeof(void*)); + return; +} +int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt) +{ + long i=0,data_size=0; + int ret=0; + GIE_digest_t* digest_rule=NULL; + GIE_digest_t** update_array=NULL; + UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; + + + GIE_create_para_t para; + para.gram_value=7; + para.position_accuracy=10; + + const long q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q); + if(q_cnt==0) + { + return 0; + } + if(table_rt->similar.gie_handle==NULL) + { + if(table_rt->table_type==TABLE_TYPE_SIMILARITY) + { + para.ED_reexamine=1; + para.format=GIE_INPUT_FORMAT_PLAIN; + } + else + { + para.ED_reexamine=0; + para.format=GIE_INPUT_FORMAT_SFH; + } + table_rt->similar.gie_handle=GIE_create(¶); + } + + update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt); + for(i=0;isimilar.update_q, &digest_rule, &data_size); + assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); + update_array[i]=digest_rule; + digest_rule=NULL; + } + ret=GIE_update(table_rt->similar.gie_handle, update_array, (int)q_cnt); + for(i=0;ioperation==GIE_INSERT_OPT) + { + table_rt->origin_rule_num++; + } + else + { + table_rt->origin_rule_num--; + } + destroy_digest_rule(update_array[i]); + update_array[i]=NULL; + } + free(update_array); + update_array=NULL; + + if(ret!=(int)q_cnt) + { + return -1; + } + return q_cnt; +} + + diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 39f7461..892fe09 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -3,7 +3,7 @@ #include #include #include - +#include #include "Maat_utils.h" pid_t gettid() { @@ -252,3 +252,9 @@ const char* charset_get_name(enum MAAT_CHARSET charset) { return CHARSET_STRING[charset]; } +int lqueue_destroy_cb(void *data, long data_len, void *arg) +{ + assert(0); + return 0; +} + diff --git a/src/inc_internal/Maat_limits.h b/src/inc_internal/Maat_limits.h index feef2a8..6d025f6 100644 --- a/src/inc_internal/Maat_limits.h +++ b/src/inc_internal/Maat_limits.h @@ -1,4 +1,3 @@ -#define MAX_TABLE_NUM 256 #define MAX_TABLE_LINE_SIZE (1024*16) #define MAX_EXPR_KEYLEN 1024 #define MAX_DISTRICT_LEN 64 diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 35633dc..64e9d47 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -4,6 +4,7 @@ #include "Maat_command.h" #include "Maat_limits.h" #include "Maat_table.h" +#include "Maat_table_runtime.h" #include #include @@ -218,54 +219,7 @@ struct _stream_para_t pthread_mutex_t fuzzy_mutex; unsigned char query_point[8]; }; -struct similar_runtime -{ - GIE_handle_t* gie_handle; - MESA_lqueue_head update_q; -}; -struct plugin_runtime -{ - dynamic_array_t *cache_lines; - long long cache_line_num; - long long acc_line_num; - long long cache_size; - MESA_htable_handle key2ex_hash; -}; -struct expr_runtime -{ - long long expr_rule_cnt; //expr_type=0,1,3 - long long regex_rule_cnt; //expr_type=2 -}; -struct ip_runtime -{ - long long ipv4_rule_cnt; - long long ipv6_rule_cnt; - -}; -struct group_runtime -{ - long long not_flag_group; -}; -struct Maat_table_runtime -{ - enum MAAT_TABLE_TYPE table_type; - long origin_rule_num; - union - { - struct similar_runtime similar; //for digest and similarity - struct plugin_runtime plugin; - struct expr_runtime expr; - struct ip_runtime ip; - struct group_runtime group; - void * other; - }; - mcore_long_t scan_cnt; - mcore_long_t scan_cpu_time; //nano - mcore_long_t input_bytes; - mcore_long_t stream_num; - mcore_long_t hit_cnt; -}; struct rule_tag { char* tag_name; @@ -280,9 +234,10 @@ struct Maat_scanner size_t gie_update_q_size; size_t to_update_group_cnt; size_t to_update_compile_cnt; - - struct Maat_table_runtime* table_rt[MAX_TABLE_NUM]; + struct Maat_table_runtime_manager* table_rt_mgr; + size_t max_table_num; + MESA_htable_handle region_hash; //key: region_id, value: struct region_group_relation* MESA_htable_handle exprid_hash; //key: expr_id, value: int array_idx of Maat_group_inner->regions; MESA_htable_handle group_hash; //key: group_id, value: struct Maat_group_inner* @@ -344,7 +299,8 @@ struct _Maat_feather_t struct Maat_scanner *scanner; struct Maat_scanner *update_tmp_scanner; MESA_lqueue_head garbage_q; - int table_cnt; + struct Maat_table_manager* table_mgr; + int DEFERRED_LOAD_ON; int REDIS_MODE_ON; enum data_source input_mode; @@ -360,8 +316,7 @@ struct _Maat_feather_t int cumulative_update_off; int stat_on; int perf_on; - struct Maat_table_desc *p_table_info[MAX_TABLE_NUM]; - MESA_htable_handle map_tablename2id; + void* logger; long long maat_version; long long last_full_version; @@ -394,8 +349,6 @@ struct _Maat_feather_t int foreign_cont_linger; //internal states long long new_version; - int active_plugin_table_num; - int is_last_plugin_table_updating; //for scanner independent stat>>>> int backgroud_update_enabled; diff --git a/src/inc_internal/Maat_table.h b/src/inc_internal/Maat_table.h index e60d887..27112d2 100644 --- a/src/inc_internal/Maat_table.h +++ b/src/inc_internal/Maat_table.h @@ -20,6 +20,7 @@ enum USER_REGION_ENCODE enum MAAT_TABLE_TYPE { + TABLE_TYPE_INVALID=-1, TABLE_TYPE_EXPR=0, TABLE_TYPE_IP, TABLE_TYPE_IP_PLUS, @@ -50,6 +51,21 @@ struct compile_table_desc struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; }; +struct expr_table_desc +{ + enum MAAT_CHARSET src_charset; + enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; + int src_charset_in_dst; + int do_charset_merge; + int cross_cache_size; + int quick_expr_switch;//obsolete since 20190401 + long long iconv_err_cnt; +}; +struct virtual_table_desc +{ + int real_table_id; + char real_table_name[MAX_TABLE_NAME_LEN]; +}; struct plugin_table_callback_desc { Maat_start_callback_t *start; @@ -79,26 +95,7 @@ struct plugin_table_desc struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; struct plugin_table_ex_data_desc ex_desc; }; -struct expr_table_desc -{ - enum MAAT_CHARSET src_charset; - enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; - int src_charset_in_dst; - int do_charset_merge; - int cross_cache_size; - int quick_expr_switch;//obsolete since 20190401 - long long iconv_err_cnt; -}; -struct ip_table_desc -{ - int ipv4_rule_cnt; - int ipv6_rule_cnt; -}; -struct virtual_table_desc -{ - int real_table_id; - char real_table_name[MAX_TABLE_NAME_LEN]; -}; + struct Maat_table_desc { int table_id; @@ -110,17 +107,52 @@ struct Maat_table_desc { struct compile_table_desc compile; struct expr_table_desc expr; - struct ip_table_desc ip; struct plugin_table_desc plugin; struct virtual_table_desc virtual_table;; - void* others;//group, interval and digest don't have sperate description info. + void* others;//group, ip, interval and digest don't have sperate description info. }; //for stat>>>>>>>> unsigned long long udpate_err_cnt; unsigned long long unmatch_tag_cnt; int stat_line_id; }; -int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_table, const char* table_info_path, void* logger); -MESA_htable_handle Maat_table_build(struct Maat_table_desc** p_table_info, size_t n_table, char* compile_tn, size_t n_ctn, char* group_tn, size_t n_gtn, void* logger); -void Maat_table_clear(struct Maat_table_desc** p_table_info, size_t n_table); +struct Maat_table_manager; +struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger); +void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr); +size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr); +size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr); + +struct Maat_table_desc * Maat_table_get_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id); +struct Maat_table_desc * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id); + +int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name); +int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, + int table_id, + Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para + Maat_update_callback_t *update,//table line ,u_para + Maat_finish_callback_t *finish,//u_para + void* u_para); +int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); +int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); +const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id); +enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id); + +int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name, + Maat_rule_EX_new_func_t *new_func, + Maat_rule_EX_free_func_t* free_func, + Maat_rule_EX_dup_func_t* dup_func, + long argl, void *argp); +struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx); +int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp); +void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type); +void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr); + +int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr); +struct Maat_table_desc* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name); +void Maat_table_set_updating_name(struct Maat_table_desc* p_table, const char* table_name); diff --git a/src/inc_internal/Maat_table_runtime.h b/src/inc_internal/Maat_table_runtime.h new file mode 100644 index 0000000..caeae5d --- /dev/null +++ b/src/inc_internal/Maat_table_runtime.h @@ -0,0 +1,69 @@ +#include "Maat_table.h" +#include "gram_index_engine.h" +#include "alignment_int64.h" +#include "dynamic_array.h" +#include +#include + +struct similar_runtime +{ + GIE_handle_t* gie_handle; + MESA_lqueue_head update_q; +}; + +struct plugin_runtime +{ + dynamic_array_t *cache_lines; + long long cache_line_num; + long long acc_line_num; + long long cache_size; + MESA_htable_handle key2ex_hash; +}; +struct expr_runtime +{ + long long expr_rule_cnt; //expr_type=0,1,3 + long long regex_rule_cnt; //expr_type=2 +}; +struct ip_runtime +{ + long long ipv4_rule_cnt; + long long ipv6_rule_cnt; + +}; +struct group_runtime +{ + long long not_flag_group; +}; +struct Maat_table_runtime +{ + enum MAAT_TABLE_TYPE table_type; + long origin_rule_num; + union + { + struct similar_runtime similar; //for digest and similarity + struct plugin_runtime plugin; + struct expr_runtime expr; + struct ip_runtime ip; + struct group_runtime group; + void * other; + }; + mcore_long_t scan_cnt; + mcore_long_t scan_cpu_time; //nano + mcore_long_t input_bytes; + mcore_long_t stream_num; + mcore_long_t hit_cnt; +}; +struct Maat_table_runtime_manager; +struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num); +void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr); +struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id); +long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt); +const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line); + +int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, void* logger); +MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, const char* key); +void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag); +void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id); +int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt); + + diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index 167048d..358b826 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -72,5 +72,6 @@ char* md5_file(const char* filename, char* md5string); int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len); const char** charset_get_all_name(void); const char* charset_get_name(enum MAAT_CHARSET charset); +int lqueue_destroy_cb(void *data, long data_len, void *arg); From 484637060d2811d435c01f76cd514177ea991f53 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Sun, 28 Jul 2019 15:00:24 +0600 Subject: [PATCH 05/29] =?UTF-8?q?=E8=99=9A=E6=8B=9F=E8=A1=A8=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95=E9=80=9A=E8=BF=87?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/maat_json.json | 38 +++++++++++++++++++++++++++++++++++--- test/table_info.conf | 6 +++++- test/test_maatframe.cpp | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/test/maat_json.json b/test/maat_json.json index ee03144..47da2f5 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -298,7 +298,7 @@ "is_valid": "yes", "groups": [ { - "group_name": "Untitled", + "group_name": "TakeMeHome", "regions": [ { "table_name": "KEYWORDS_TABLE", @@ -1141,8 +1141,40 @@ ], "not_flag" : 0 } - ] - } + ] + }, + { + "compile_id": 160, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "Virtual", + "is_valid": "yes", + "groups": [ + { + "group_name":"TakeMeHome", + "virtual_table":"HTTP_RESPONSE_KEYWORDS", + "not_flag" : 0 + }, + { + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "https://blog.csdn.net/littlefang/article/details/8213058", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + } ], "plugin_table": [ { diff --git a/test/table_info.conf b/test/table_info.conf index 5119ef6..e960a64 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -13,6 +13,9 @@ #For plugin table. The first column's id is 1. 0 as not speicified. #id name type column_define # +#For virtual Table +#id name type real_table_name +# #For expr/expr_plus Table #id name type src_charset dst_charset do_merge cross_cache quick_mode 0 COMPILE compile escape -- @@ -35,4 +38,5 @@ 16 APP_PAYLOAD expr_plus UTF8 UTF8 yes 0 quickoff 17 TROJAN_PAYLOAD expr UTF8 UTF8 yes 0 quickoff 18 MAIL_ADDR expr UTF8 UTF8 yes 0 quickoff -19 IP_PLUS_CONFIG ip_plus -- \ No newline at end of file +19 IP_PLUS_CONFIG ip_plus -- +20 HTTP_RESPONSE_KEYWORDS virtual KEYWORDS_TABLE -- \ No newline at end of file diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 64f062e..15b3edc 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -1462,6 +1462,40 @@ TEST(ScanResult, LongerServiceDefine) free(buff); return; } +TEST(VirtualTable, Test1) +{ +#define TestVirtualTable + int ret=0, table_id=0; + const char* http_content="Batman\\:Take me Home.Superman/:Fine,stay with me."; + const char* http_url="https://blog.csdn.net/littlefang/article/details/8213058"; + + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + + table_id=Maat_table_register(g_feather, "HTTP_URL"); + ASSERT_GT(table_id, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_url, strlen(http_url), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, -2); + + + table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_KEYWORDS"); + ASSERT_GT(table_id, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_content, strlen(http_content), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 160); + + Maat_clean_status(&mid); + return; +} + class MaatFileTest : public testing::Test { From 718bbc59ddb5840e13ca2353a8abee36dba9775b Mon Sep 17 00:00:00 2001 From: zhengchao Date: Sun, 28 Jul 2019 15:03:33 +0600 Subject: [PATCH 06/29] =?UTF-8?q?Maat=20JSON=E6=94=AF=E6=8C=81=E8=99=9A?= =?UTF-8?q?=E6=8B=9F=E8=A1=A8=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 4 ++-- src/entry/Maat_rule.cpp | 2 +- src/entry/json2iris.cpp | 18 ++++++++++++++---- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index abcc421..5cf746e 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -103,7 +103,7 @@ size_t pickup_hit_region_from_compile(struct bool_expr *compile_hit, group_id=(unsigned long long)dynamic_array_read(hitted_id, i); for(j=0; jitem_num; j++) { - if(group_id==compile_hit->items[j].item_id) + if(group_id==(compile_hit->items[j].item_id&0x00000000ffffffff)) { region_pos[k]=i; k++; @@ -204,7 +204,7 @@ int region_compile(_Maat_feather_t*feather, struct _INNER_scan_status_t *_mid, c ret=insert_set_id(&(_mid->all_hit_group_array), &(_mid->all_hit_group_array_sz), _mid->all_hit_group_cnt, - group_rule->top_groups[j]); + (unsigned long long)region_hit_wraper->virtual_table_id<<32|group_rule->top_groups[j]); _mid->all_hit_group_cnt+=ret; } } diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 7c12f23..2c46dbb 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -2177,7 +2177,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru table->udpate_err_cnt++; return; } - if(strlen(virtual_table_name)>0&&!strcasecmp(virtual_table_name, "null")) + if(strlen(virtual_table_name)>0&&strcasecmp(virtual_table_name, "null")) { db_group_rule.virtual_table_id=Maat_table_get_id_by_name(table_mgr, virtual_table_name); if(db_group_rule.virtual_table_id<0) diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 2a5aae8..e6b175b 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -888,7 +888,7 @@ int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void * set_file_rulenum(table_info->table_path,table_info->line_count,logger); return compile_id; } -int write_group_line(int group_id, int parent_id, int group_not_flag, int parent_type, struct iris_description_t *p_iris, void * logger) +int write_group_line(int group_id, int parent_id, int group_not_flag, int parent_type, const char* virtual_table, struct iris_description_t *p_iris, void * logger) { FILE*fp=NULL; int ret=0; @@ -908,7 +908,7 @@ int write_group_line(int group_id, int parent_id, int group_not_flag, int parent "fopen %s error %s.",p_iris->group_table->table_path,strerror(errno)); return -1; } - fprintf(fp,"%d\t%d\t1\t%d\t%d\n",group_id, parent_id, group_not_flag, parent_type); + fprintf(fp,"%d\t%d\t1\t%d\t%d\t%s\n",group_id, parent_id, group_not_flag, parent_type, virtual_table); fclose(fp); p_iris->group_table->line_count++; ret=set_file_rulenum(p_iris->group_table->table_path,p_iris->group_table->line_count,logger); @@ -943,7 +943,7 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac int group_not_flag=0; cJSON *region_json=NULL, *item=NULL; cJSON *sub_groups=NULL, *region_rule=NULL; - const char* group_name=NULL; + const char* group_name=NULL, *virtual_table=NULL; struct group_info_t *group_info=NULL; struct group_info_t untitled_group; @@ -956,6 +956,16 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac { group_name=item->valuestring; } + + item=cJSON_GetObjectItem(group_json, "virtual_table"); + if(item==NULL||item->type!=cJSON_String) + { + virtual_table="null"; + } + else + { + virtual_table=item->valuestring; + } item=cJSON_GetObjectItem(group_json,"not_flag"); if(item==NULL||item->type!=cJSON_Number) { @@ -985,7 +995,7 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info); } } - ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, p_iris, logger); + ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, virtual_table, p_iris, logger); if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, From 3473cf6dd48497873a5c1febf5bec25347d49fb1 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Sun, 28 Jul 2019 19:13:04 +0600 Subject: [PATCH 07/29] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=80=A7=E8=83=BD?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B=EF=BC=8C=E5=A4=84=E7=90=86?= =?UTF-8?q?=E6=97=A0=E6=B3=95=E8=8E=B7=E5=8F=96redis=20time=E7=9A=84?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/CMakeLists.txt | 2 +- src/entry/Maat_command.cpp | 31 +++- src/entry/Maat_rule.cpp | 40 +++-- src/inc_internal/Maat_rule_internal.h | 3 + test/CMakeLists.txt | 3 + test/perf_test_maatframe.cpp | 205 ++++++++++++++++++++++++++ test/test_maatframe.cpp | 101 ++----------- tools/maat_redis_tool.cpp | 4 + 8 files changed, 278 insertions(+), 111 deletions(-) create mode 100644 test/perf_test_maatframe.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1ef76dd..2e79378 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.5) set(MAAT_FRAME_MAJOR_VERSION 2) -set(MAAT_FRAME_MINOR_VERSION 7) +set(MAAT_FRAME_MINOR_VERSION 8) set(MAAT_FRAME_PATCH_VERSION 0) set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION}) diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 16d206b..5b7bc7f 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -143,9 +143,11 @@ long long redis_server_time(redisContext* ctx) long long server_time=0; redisReply* data_reply=NULL; data_reply=_wrap_redisCommand(ctx,"TIME"); - assert(data_reply->type==REDIS_REPLY_ARRAY); - server_time=atoll(data_reply->element[0]->str); - freeReplyObject(data_reply); + if(data_reply->type==REDIS_REPLY_ARRAY) + { + server_time=atoll(data_reply->element[0]->str); + freeReplyObject(data_reply); + } return server_time; } enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p) @@ -1434,7 +1436,10 @@ void check_maat_expiration(redisContext *ctx, void *logger) long long server_time=0; server_time=redis_server_time(ctx); - + if(!server_time) + { + return; + } data_reply=_wrap_redisCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_expire_sset,server_time); if(data_reply->type!=REDIS_REPLY_ARRAY||data_reply->elements==0) { @@ -1473,7 +1478,10 @@ void cleanup_update_status(redisContext *ctx, void *logger) long long server_time=0, version_upper_bound=0,version_lower_bound=0,version_num=0,entry_num=0; server_time=redis_server_time(ctx); - + if(!server_time) + { + return; + } reply=_wrap_redisCommand(ctx,"MULTI"); freeReplyObject(reply); redisAppendCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_version_sset,server_time-MAAT_REDIS_SYNC_TIME); @@ -2103,6 +2111,10 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru return -1; } server_time=redis_server_time(write_ctx); + if(!server_time) + { + return -1; + } s_rule=(struct serial_rule_t *)calloc(sizeof(struct serial_rule_t),line_num); for(i=0;inext; } _feather->server_time=redis_server_time(write_ctx); - + if(!_feather->server_time) + { + goto error_out; + } if(_feather->AUTO_NUMBERING_ON==1) { data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_region_id_var, new_region_num); @@ -2612,6 +2627,10 @@ static int _Maat_command_set_one_line(struct _Maat_feather_t* _feather, enum MAA { redisContext* write_ctx=get_redis_ctx_for_write(_feather); _feather->server_time=redis_server_time(write_ctx); + if(!_feather->server_time) + { + return -1; + } struct serial_rule_t s_rule; set_serial_rule(&s_rule, op, id, 0, table_name, line, 0); int transaction_success=0; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 2c46dbb..2859656 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -33,7 +33,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_7_20190629=1; +int MAAT_FRAME_VERSION_2_8_20190728=1; int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type) @@ -3169,26 +3169,39 @@ void vector_print(igraph_vector_t *v) { } printf("\n"); } - +static size_t effective_vertices_count(igraph_vector_t *vids) +{ + size_t i=0; + int tmp_vid=0; + for(i=0; i<(size_t)igraph_vector_size(vids); i++) + { + tmp_vid=(int) VECTOR(*vids)[i]; + if(tmp_vid<0) + { + break; + } + } + return i; +} void walk_group_hash(const uchar * key, uint size, void * data, void * user) { struct Maat_group_inner* group_rule=(struct Maat_group_inner*)data; struct Maat_group_inner* parent_group=NULL; struct Maat_scanner* scanner=(struct Maat_scanner*)user; int tmp_vid=0; - igraph_vector_t vids; - igraph_vector_init(&vids, 0); + igraph_vector_t *vids=&(scanner->dfs_vids); igraph_dfs(&(scanner->group_graph), group_rule->vertex_id, IGRAPH_OUT, - 0, &vids, NULL, NULL, NULL, NULL, NULL, NULL); + 0, vids, NULL, NULL, NULL, NULL, NULL, NULL); - long int i=0; - long long* temp_group_ids=ALLOC(long long, igraph_vector_size(&vids)); - size_t top_group_cnt=0; - for(i=0; ivertex_id2group, tmp_vid); if(parent_group->has_compile_neighbors)//including itself { - temp_group_ids[top_group_cnt]=parent_group->group_id; + temp_group_ids[top_group_cnt]=parent_group->group_id; top_group_cnt++; } } + pthread_mutex_lock(&(group_rule->mutex)); free(group_rule->top_groups); group_rule->top_group_cnt=top_group_cnt; @@ -3211,7 +3225,6 @@ void walk_group_hash(const uchar * key, uint size, void * data, void * user) scanner->most_popular_sub_group=group_rule->group_id; } pthread_mutex_unlock(&(group_rule->mutex)); - igraph_vector_destroy(&vids); free(temp_group_ids); temp_group_ids=NULL; return; @@ -3219,7 +3232,10 @@ void walk_group_hash(const uchar * key, uint size, void * data, void * user) void find_group_paths(struct Maat_scanner* scanner) { + scanner->group_graph_vcount=igraph_vcount(&scanner->group_graph); + igraph_vector_init(&(scanner->dfs_vids), scanner->group_graph_vcount); MESA_htable_iterate(scanner->group_hash, walk_group_hash, scanner); + igraph_vector_destroy(&scanner->dfs_vids); return; } void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, int scan_thread_num, void* logger) diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 64e9d47..6d5be4b 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -249,6 +249,9 @@ struct Maat_scanner MESA_htable_handle vertex_id2group; igraph_t group_graph; + igraph_integer_t group_graph_vcount; + igraph_vector_t dfs_vids; + int grp_vertex_id_generator; int most_popular_sub_group; unsigned long long max_presented_top_group_cnt; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 02e8590..db47175 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -7,6 +7,9 @@ target_link_libraries(test_igraph igraph-static) add_executable(test_maatframe test_maatframe.cpp) target_link_libraries(test_maatframe maat_frame_shared gtest) +add_executable(perf_test_maatframe perf_test_maatframe.cpp) +target_link_libraries(perf_test_maatframe maat_frame_shared gtest) + configure_file(table_info.conf table_info.conf COPYONLY) configure_file(t2_tableinfo.conf t2_tableinfo.conf COPYONLY) configure_file(maat_json.json maat_json.json COPYONLY) diff --git a/test/perf_test_maatframe.cpp b/test/perf_test_maatframe.cpp new file mode 100644 index 0000000..2a627cf --- /dev/null +++ b/test/perf_test_maatframe.cpp @@ -0,0 +1,205 @@ +#include "Maat_rule.h" +#include "stream_fuzzy_hash.h" +#include "Maat_command.h" +#include +#include +int test_add_expr_command_copy(Maat_feather_t feather,const char* region_table,int config_id, int timeout,int label_id, const char* keywords) +{ + struct Maat_cmd_t* cmd=NULL; + struct Maat_rule_t rule; + char huge_serv_def[1024*2]; + memset(huge_serv_def,'s',sizeof(huge_serv_def)); + struct Maat_region_t region; + int group_num=1,ret=0; + memset(&rule,0,sizeof(rule)); + rule.config_id=config_id; + strcpy(rule.service_defined,"maat_command"); + //MUST acqire by function, because Maat_cmd_t has some hidden members. + cmd=Maat_create_cmd(&rule, group_num); + cmd->expire_after=timeout; + cmd->label_id=label_id; + memset(®ion,0,sizeof(region)); + region.region_type=REGION_EXPR; + region.table_name=region_table; + region.expr_rule.district=NULL; + region.expr_rule.keywords=keywords; + region.expr_rule.expr_type=EXPR_TYPE_AND; + region.expr_rule.match_method=MATCH_METHOD_SUB; + region.expr_rule.hex_bin=UNCASE_PLAIN; + Maat_cmd_set_opt(cmd, MAAT_RULE_SERV_DEFINE, huge_serv_def, sizeof(huge_serv_def)); + Maat_add_region2cmd(cmd, 0, ®ion); + //use pipeline model. + ret=Maat_cmd_append(feather, cmd, MAAT_OP_ADD); + if(ret<0) + { + printf("Add Maat command %d failed.\n",rule.config_id); + Maat_free_cmd(cmd); + return 0; + } + //cmd has been saved in feather, so free cmd before commit is allowed. + Maat_free_cmd(cmd); + return 0; + +} +void wait_for_cmd_effective_copy(Maat_feather_t feather, long long version_before) +{ + long long version_after=version_before; + int is_updating=1; + long long wating_us=0, sleep_us=1000*100; + while(is_updating||version_before==version_after) + { + Maat_read_state(feather,MAAT_STATE_IN_UPDATING, &is_updating, sizeof(is_updating)); + Maat_read_state(feather,MAAT_STATE_VERSION, &version_after, sizeof(version_after)); + + usleep(sleep_us);//waiting for commands go into effect + wating_us+=sleep_us; + } +// printf("wait for %lld ms\n", wating_us/1000); +} + +class MaatCMDPerfTest : public testing::Test +{ + +protected: + + + static void SetUpTestCase() + { + const char* test_maat_redis_ip="127.0.0.1"; + unsigned short test_maat_redis_port=6379; + int g_iThreadNum=4; + const char* table_info_path="./table_info.conf"; + int scan_interval_ms=500; + int effective_interval_ms=0; + + logger=MESA_create_runtime_log_handle("test_maat_redis.log",0); + + _shared_feather=Maat_feather(g_iThreadNum, table_info_path, logger); + Maat_set_feather_opt(_shared_feather,MAAT_OPT_INSTANCE_NAME,"perf", strlen("perf")+1); + Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_IP, test_maat_redis_ip, strlen(test_maat_redis_ip)+1); + Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_PORT, &test_maat_redis_port, sizeof(test_maat_redis_port)); + Maat_set_feather_opt(_shared_feather, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms)); + //Set a short intevral for testing. + Maat_set_feather_opt(_shared_feather, MAAT_OPT_EFFECT_INVERVAL_MS,&effective_interval_ms, sizeof(effective_interval_ms)); + + const char* foregin_dir="./foreign_files/"; + Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_DIR, foregin_dir, strlen(foregin_dir)+1); + int linger_timeout=2; + Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_LINGER, &linger_timeout, sizeof(linger_timeout)); + + Maat_cmd_flushDB(_shared_feather); + Maat_initiate_feather(_shared_feather); + } + static void TearDownTestCase() + { + Maat_burn_feather(_shared_feather); + MESA_destroy_runtime_log_handle(logger); + + } + // Some expensive resource shared by all tests. + static Maat_feather_t _shared_feather; + static void *logger; +}; +Maat_feather_t MaatCMDPerfTest::_shared_feather; + +void* MaatCMDPerfTest::logger; + +//Following tests must be coded/tested at last, for they stalled the maat update thread and interrupt other tests. +TEST_F(MaatCMDPerfTest, SetExpr100K) +{ + const int CMD_EXPR_NUM=100*1000; + const char* table_name="HTTP_URL"; + + const char* keywords1="Hiredis"; + const char* keywords2="C Client"; + char escape_buff1[256],escape_buff2[256]; + char keywords[256]; + + int label_id=5210, config_id=0,ret=0, output_id_cnt=0; + Maat_feather_t feather=MaatCMDPerfTest::_shared_feather; + long long version_before=0; + ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before)); + + Maat_str_escape(escape_buff1, sizeof(escape_buff1),keywords1); + Maat_str_escape(escape_buff2, sizeof(escape_buff2),keywords2); + snprintf(keywords,sizeof(keywords),"%s&%s",escape_buff1,escape_buff2); + + config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", CMD_EXPR_NUM); + int i=0; + for(i=0; i=0); + wait_for_cmd_effective_copy(feather, version_before); + struct Maat_cmd_t* cmd=NULL; + struct Maat_rule_t rule; + memset(&rule,0,sizeof(rule)); + int *output_ids=(int*)malloc(sizeof(int)*CMD_EXPR_NUM); + output_id_cnt=Maat_cmd_select(feather,label_id, output_ids, CMD_EXPR_NUM); + EXPECT_EQ(output_id_cnt, CMD_EXPR_NUM); + for(i=0; i=0); - wait_for_cmd_effective(feather, version_before); - struct Maat_cmd_t* cmd=NULL; - struct Maat_rule_t rule; - memset(&rule,0,sizeof(rule)); - int *output_ids=(int*)malloc(sizeof(int)*CMD_EXPR_NUM); - output_id_cnt=Maat_cmd_select(feather,label_id, output_ids, CMD_EXPR_NUM); - EXPECT_EQ(output_id_cnt, CMD_EXPR_NUM); - for(i=0; i0) { absolute_expire_time=server_time+timeout; From eadab7ecba7123c7af74edbc006b088749e3c2b4 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Sun, 28 Jul 2019 20:42:17 +0600 Subject: [PATCH 08/29] =?UTF-8?q?=E6=B7=B1=E5=BA=A6=E9=81=8D=E5=8E=86group?= =?UTF-8?q?=20graph=E6=97=B6=EF=BC=8C=E8=8B=A5=E5=88=86=E7=BB=84=E4=B8=8D?= =?UTF-8?q?=E8=A2=AB=E5=85=B6=E5=AE=83=E5=88=86=E7=BB=84=E5=BC=95=E7=94=A8?= =?UTF-8?q?=EF=BC=8C=E6=89=A7=E8=A1=8C=E5=BF=AB=E9=80=9F=E5=A4=84=E7=90=86?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=EF=BC=8C=E6=8F=90=E9=AB=98=E5=8A=A0=E8=BD=BD?= =?UTF-8?q?=E6=80=A7=E8=83=BD=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_rule.cpp | 50 ++++++++++++++++----------- src/inc_internal/Maat_rule_internal.h | 2 +- test/perf_test_maatframe.cpp | 4 +-- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 2859656..b14a545 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -1990,7 +1990,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro } else { - group_rule->has_compile_neighbors=1; + group_rule->ref_by_compile_cnt++; compile_rule=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id); if(compile_rule==NULL) { @@ -2092,6 +2092,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro { destroy_compile_group_relation(relation, scanner); } + group_rule->ref_by_compile_cnt--; } destroy_group_rule(group_rule, DESTROY_GROUP_BY_PARENT, scanner); scanner->to_update_group_cnt++; @@ -3189,31 +3190,38 @@ void walk_group_hash(const uchar * key, uint size, void * data, void * user) struct Maat_group_inner* parent_group=NULL; struct Maat_scanner* scanner=(struct Maat_scanner*)user; int tmp_vid=0; - igraph_vector_t *vids=&(scanner->dfs_vids); - - igraph_dfs(&(scanner->group_graph), group_rule->vertex_id, IGRAPH_OUT, - 0, vids, NULL, NULL, NULL, NULL, NULL, NULL); - - size_t i=0, top_group_cnt=0; - size_t parent_group_cnt=effective_vertices_count(vids); - long long* temp_group_ids=ALLOC(long long, parent_group_cnt); - - for(i=0; i<(size_t)igraph_vector_size(vids); i++) + long long* temp_group_ids=NULL; + if(group_rule->ref_by_compile_cnt==group_rule->ref_by_parent_cnt) { - tmp_vid=(int) VECTOR(*vids)[i]; - if(tmp_vid<0) + //fast path, group is only referenced by compile rules. + top_group_cnt=1; + temp_group_ids=ALLOC(long long, top_group_cnt); + temp_group_ids[0]=group_rule->group_id; + } + else + { + igraph_vector_t *vids=&(scanner->dfs_vids); + igraph_dfs(&(scanner->group_graph), group_rule->vertex_id, IGRAPH_OUT, + 0, vids, NULL, NULL, NULL, NULL, NULL, NULL); + + temp_group_ids=ALLOC(long long, effective_vertices_count(vids)); + + for(i=0; i<(size_t)igraph_vector_size(vids); i++) { - break; - } - parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->vertex_id2group, tmp_vid); - if(parent_group->has_compile_neighbors)//including itself - { - temp_group_ids[top_group_cnt]=parent_group->group_id; - top_group_cnt++; + tmp_vid=(int) VECTOR(*vids)[i]; + if(tmp_vid<0) + { + break; + } + parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->vertex_id2group, tmp_vid); + if(parent_group->ref_by_compile_cnt>0)//including itself + { + temp_group_ids[top_group_cnt]=parent_group->group_id; + top_group_cnt++; + } } } - pthread_mutex_lock(&(group_rule->mutex)); free(group_rule->top_groups); group_rule->top_group_cnt=top_group_cnt; diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 6d5be4b..0e58d93 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -152,7 +152,7 @@ struct Maat_group_inner int ref_by_parent_cnt; int ref_by_children_cnt; char* group_name; - int has_compile_neighbors; + int ref_by_compile_cnt; int vertex_id; size_t top_group_cnt; long long* top_groups; diff --git a/test/perf_test_maatframe.cpp b/test/perf_test_maatframe.cpp index 2a627cf..806dd2f 100644 --- a/test/perf_test_maatframe.cpp +++ b/test/perf_test_maatframe.cpp @@ -105,9 +105,9 @@ Maat_feather_t MaatCMDPerfTest::_shared_feather; void* MaatCMDPerfTest::logger; //Following tests must be coded/tested at last, for they stalled the maat update thread and interrupt other tests. -TEST_F(MaatCMDPerfTest, SetExpr100K) +TEST_F(MaatCMDPerfTest, SetExpr200K) { - const int CMD_EXPR_NUM=100*1000; + const int CMD_EXPR_NUM=200*1000; const char* table_name="HTTP_URL"; const char* keywords1="Hiredis"; From c58c7d7f3aa5d295d87c0d0991b41eda8d1733ec Mon Sep 17 00:00:00 2001 From: zhengchao Date: Thu, 1 Aug 2019 14:41:28 +0600 Subject: [PATCH 09/29] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E5=A4=84=E7=90=86region=20compile=E6=97=B6=E5=AF=B9bool=20matc?= =?UTF-8?q?her=E5=88=A4=E7=A9=BA=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 10 ++++++++-- src/entry/Maat_command.cpp | 2 +- src/entry/Maat_rule.cpp | 1 + 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 5cf746e..3a96137 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -209,10 +209,16 @@ int region_compile(_Maat_feather_t*feather, struct _INNER_scan_status_t *_mid, c } } - scan_ret=bool_matcher_match(bm, thread_num, + if(bm) + { + scan_ret=bool_matcher_match(bm, thread_num, _mid->all_hit_group_array, _mid->all_hit_group_cnt, (void **)relation_array, MAX_SCANNER_HIT_NUM); - + } + else + { + scan_ret=0; + } if(scan_ret>1) { qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation**), diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 5b7bc7f..ebe9417 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -247,7 +247,7 @@ int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) void serialize_group(const struct Maat_group_t* p_group, enum MAAT_OPERATION op, char* buff, size_t sz) { if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD; - snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d", p_group->group_id, + snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d\tnull", p_group->group_id, p_group->parent_id, op, p_group->not_flag, diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index b14a545..4fb6d79 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -619,6 +619,7 @@ void make_group_set(struct Maat_compile_group_relation* relation, struct bool_ex { continue; } + //high 32 bit is virtual table id, low 32 bit is group id. a_set->items[j].item_id=(unsigned long long)relation->virtual_table_id[j]<<32|group->group_id; a_set->items[j].not_flag=relation->not_flag[j]; if(a_set->items[j].not_flag) From fdf41bfc1a0f2e4bee0d6d8f40ab382c4a4206d6 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Thu, 1 Aug 2019 14:42:03 +0600 Subject: [PATCH 10/29] =?UTF-8?q?maat=5Fdebug=5Ftool=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E6=89=AB=E6=8F=8FIPv4=E5=9C=B0=E5=9D=80=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/maat_debug_tool.cpp | 136 +++++++++++++++++++++++++++----------- 1 file changed, 98 insertions(+), 38 deletions(-) diff --git a/tools/maat_debug_tool.cpp b/tools/maat_debug_tool.cpp index 5b8d3be..5697bab 100644 --- a/tools/maat_debug_tool.cpp +++ b/tools/maat_debug_tool.cpp @@ -2,16 +2,42 @@ #include "Maat_command.h" #include +#include #include /* for printf */ #include /* for exit */ #include #include +#include +void debug_maat_result_print(const char* table_name, int scan_ret, struct Maat_rule_t* result) +{ + printf("Scan table %s ", table_name); + if(scan_ret==-1) + { + printf("error.\n"); + } + else if(scan_ret==-2) + { + printf("hits group, but not compile.\n"); + } + else if(scan_ret==0) + { + printf("not hit."); + } + else + { + printf("hits "); + for(int i=0; i0) { - printf("Read %s failed, invalid maat json.\n", arg_value[ARG_INPUT_JSON]); + ret=Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, arg_value[ARG_INPUT_JSON], strlen(arg_value[ARG_INPUT_JSON])+1); + if(ret!=0) + { + printf("Read %s failed, invalid maat json.\n", arg_value[ARG_INPUT_JSON]); + ret=-1; + goto clean_up; + } + } + else if(strlen(arg_value[ARG_INPUT_FULL_INDEX])) + { + ret=Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, arg_value[ARG_INPUT_FULL_INDEX], strlen(arg_value[ARG_INPUT_FULL_INDEX])+1); + if(ret!=0) + { + printf("Set %s failed, invalid maat json.\n", arg_value[ARG_INPUT_FULL_INDEX]); + ret=-1; + goto clean_up; + } + } + else + { + printf("Error: One of --%s and --%s should be specified.\n", long_options[ARG_INPUT_JSON].name, long_options[ARG_INPUT_FULL_INDEX].name); ret=-1; goto clean_up; } @@ -165,6 +221,10 @@ int main(int argc, char ** argv) debug_maat_str_scan(feather, arg_value[ARG_TABLE_NAME], arg_value[ARG_SCAN_DISTRICT], file_buff, file_size); free(file_buff); } + if(strlen(arg_value[ARG_SCAN_IPv4])>0) + { + debug_maat_ip_scan(feather, arg_value[ARG_TABLE_NAME], arg_value[ARG_SCAN_IPv4]); + } clean_up: Maat_burn_feather(feather); MESA_destroy_runtime_log_handle(g_logger); From 912556ea33c41046603a17159de52cd4fbf26aba Mon Sep 17 00:00:00 2001 From: zhengchao Date: Fri, 16 Aug 2019 14:32:26 +0800 Subject: [PATCH 11/29] =?UTF-8?q?=E4=BD=BF=E7=94=A8cJSON=5FArrayForEach?= =?UTF-8?q?=E6=9B=BF=E4=BB=A3cJSON=5FGetArrayItem=EF=BC=8C=E6=8F=90?= =?UTF-8?q?=E9=AB=98maat=20json=E8=A7=A3=E6=9E=90=E9=80=9F=E5=BA=A6?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 6 +++++- src/entry/Maat_rule.cpp | 2 +- src/entry/json2iris.cpp | 25 ++++++++----------------- test/test_igraph.cpp | 2 +- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 3a96137..c0a63f3 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -631,6 +631,10 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo break; case MAAT_OPT_JSON_FILE_PATH: assert(_feather->input_mode==SOURCE_NONE); + MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , + "Maat initial with JSON file %s, formating..", + (const char*)value); + ret=json2iris((const char*)value, _feather->compile_tn,_feather->group_tn, NULL, @@ -647,7 +651,7 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo md5_file(_feather->json_ctx.json_file, _feather->json_ctx.effective_json_md5); MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Maat initial with JSON file %s md5: %s, generate index file %s OK.", + "JSON file %s md5: %s, generate index file %s OK.", _feather->json_ctx.json_file, _feather->json_ctx.effective_json_md5, _feather->json_ctx.iris_file); diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 4fb6d79..9c0991e 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -33,7 +33,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_8_20190728=1; +int MAAT_FRAME_VERSION_2_8_20190816=1; int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type) diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index e6b175b..79da6c5 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -937,8 +937,6 @@ int write_index_file(struct iris_description_t *p_iris,void* logger) int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger) { const char* _str_parent_type[2]={"compile", "group"}; - int i=0; - int sub_group_cnt=0, region_cnt=0; int ret=0; int group_not_flag=0; cJSON *region_json=NULL, *item=NULL; @@ -1005,10 +1003,8 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac region_json=cJSON_GetObjectItem(group_json,"regions"); if(region_json!=NULL) { - region_cnt=cJSON_GetArraySize(region_json); - for(i=0; igroup_id, p_iris, logger); if(ret<0) { @@ -1022,10 +1018,9 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac if(sub_groups!=NULL) { //recursively - sub_group_cnt=cJSON_GetArraySize(sub_groups); - for(i=0; igroup_id, PARENT_TYPE_GROUP, tracking_compile_id, p_iris, logger); if(ret<0) { @@ -1037,18 +1032,16 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac } int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) { - int i=0,j=0; - int compile_id=-1, compile_cnt=0, group_cnt=0, plug_table_cnt=0; + int i=0; + int compile_id=-1, compile_cnt=0, group_cnt=0; int ret=0; cJSON *c_rules=NULL, *g_rules=NULL, *plug_tables=NULL; cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL; plug_tables=cJSON_GetObjectItem(json,"plugin_table"); if(NULL!=plug_tables) { - plug_table_cnt=cJSON_GetArraySize(plug_tables); - for(i=0;i Date: Fri, 16 Aug 2019 14:58:41 +0800 Subject: [PATCH 12/29] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=AF=B9group=E8=A1=A8?= =?UTF-8?q?=E5=90=8D=E7=9A=84=E6=A0=A1=E9=AA=8C=EF=BC=8C=E5=88=86=E7=BB=84?= =?UTF-8?q?=E8=A1=A8=E4=B8=AD=E6=97=A0vitrtual=20table=E7=9A=84=E6=83=85?= =?UTF-8?q?=E5=86=B5=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_rule.cpp | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 9c0991e..18ba8d3 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -35,7 +35,24 @@ int MAAT_FRAME_VERSION_2_8_20190816=1; - +int is_valid_table_name(const char* str) +{ + size_t i=0, integer_cnt=0; + for(i=0; i='0'&&str[i]<='9') + { + integer_cnt++; + } + } + if(strlen(str)==0 || + integer_cnt==strlen(str) || + 0==strcasecmp(str, "null")) + { + return 0; + } + return 1; +} int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type) { switch(expr_type) @@ -2179,7 +2196,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru table->udpate_err_cnt++; return; } - if(strlen(virtual_table_name)>0&&strcasecmp(virtual_table_name, "null")) + if(is_valid_table_name(virtual_table_name)) { db_group_rule.virtual_table_id=Maat_table_get_id_by_name(table_mgr, virtual_table_name); if(db_group_rule.virtual_table_id<0) From 9e0ae6f10d2538ae757fa4ec117fb37906ee1064 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 27 Aug 2019 09:24:42 +0800 Subject: [PATCH 13/29] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E5=A4=84=E7=90=86=EF=BC=8C=E4=BF=AE=E5=A4=8D#19?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 6 +++++- src/entry/Maat_command.cpp | 6 ++---- src/entry/Maat_rule.cpp | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index c0a63f3..b1c339a 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -982,7 +982,11 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, pthread_mutex_unlock(&(_feather->background_update_mutex)); return -1; } - + if(!_feather->scanner) + { + pthread_mutex_unlock(&(_feather->background_update_mutex)); + return 1; + } const char* line=NULL; struct Maat_table_runtime* table_rt=NULL; table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index ebe9417..353667b 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -2374,7 +2374,7 @@ int Maat_cmd_commit(Maat_feather_t feather) { _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - int ret=0,i=0; + int ret=-1, i=0; int new_region_num=0,new_group_num=0; int serial_rule_num=0,serial_rule_idx=0; UNUSED int transaction_success=1; @@ -2386,7 +2386,7 @@ int Maat_cmd_commit(Maat_feather_t feather) struct serial_rule_t* s_rule=NULL; if(_feather->input_mode!=SOURCE_REDIS) { - return -1; + return ret; } if(_feather->cmd_q_cnt==0) { @@ -2414,7 +2414,6 @@ int Maat_cmd_commit(Maat_feather_t feather) if(data_reply->type!=REDIS_REPLY_INTEGER) { freeReplyObject(data_reply); - ret=-1; goto error_out; } _feather->base_rgn_seq=data_reply->integer-new_region_num; @@ -2424,7 +2423,6 @@ int Maat_cmd_commit(Maat_feather_t feather) if(data_reply->type!=REDIS_REPLY_INTEGER) { freeReplyObject(data_reply); - ret=-1; goto error_out; } _feather->base_grp_seq=data_reply->integer-new_group_num; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 18ba8d3..d0eb3e8 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -33,7 +33,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_8_20190816=1; +int MAAT_FRAME_VERSION_2_8_20190827=1; int is_valid_table_name(const char* str) { From abdfa9a18377c948df92254c70421d2fcb17346d Mon Sep 17 00:00:00 2001 From: zhengchao Date: Fri, 30 Aug 2019 14:19:07 +0800 Subject: [PATCH 14/29] =?UTF-8?q?=E5=9F=9F=E9=85=8D=E7=BD=AE=E5=91=BD?= =?UTF-8?q?=E4=B8=AD=E7=BC=93=E5=AD=98=E7=94=B164=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E5=88=B04096=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 8 +++++--- src/entry/Maat_rule.cpp | 3 +++ src/inc_internal/Maat_limits.h | 2 +- src/inc_internal/Maat_rule_internal.h | 1 + 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index b1c339a..d3ac7e9 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -1878,7 +1878,6 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); int do_query=0; - GIE_result_t region_result[MAX_SCANNER_HIT_NUM]; int hit_region_cnt=0,compile_ret=0; _compile_result_t compile_result[rule_num];//dynamic array if(data==NULL||data_len<=0) @@ -1889,6 +1888,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { return 0; } + GIE_result_t* region_result=sp->feather->scanner->gie_rslt_buff+MAX_SCANNER_HIT_NUM*sp->thread_num; + struct Maat_table_runtime *table_rt=Maat_table_runtime_get(sp->feather->scanner->table_rt_mgr, sp->p_real_table->table_id); GIE_handle_t* GIE_handle=table_rt->similar.gie_handle; unsigned long long digest_len=0; @@ -2065,7 +2066,6 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { int hit_region_cnt=0,compile_ret=0; struct _OUTER_scan_status_t* _mid=NULL; - GIE_result_t region_result[MAX_SCANNER_HIT_NUM]; _compile_result_t compile_result[rule_num]; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct Maat_scanner* my_scanner=NULL; @@ -2087,6 +2087,8 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { return 0; } + GIE_result_t* region_result=my_scanner->gie_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); if(table_rt->origin_rule_num==0) { @@ -2096,7 +2098,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id INC_SCANNER_REF(my_scanner,thread_num); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); - hit_region_cnt=GIE_query(gie_handle, data, data_len,region_result, MAX_SCANNER_HIT_NUM); + hit_region_cnt=GIE_query(gie_handle, data, data_len, region_result, MAX_SCANNER_HIT_NUM); if(hit_region_cnt<0) { DEC_SCANNER_REF(my_scanner, thread_num); diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index d0eb3e8..57830e7 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -1028,6 +1028,7 @@ struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *f scanner->tomb_ref=feather->garbage_q; scanner->logger_ref=feather->logger; scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num); + scanner->gie_rslt_buff=ALLOC(GIE_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num); scanner->table_rt_mgr=Maat_table_runtime_manager_create(feather->table_mgr, feather->scan_thread_num); scanner->max_table_num=Maat_table_manager_get_size(feather->table_mgr); return scanner; @@ -1067,6 +1068,8 @@ void destroy_maat_scanner(struct Maat_scanner*scanner) MESA_lqueue_destroy(scanner->region_update_q, lqueue_destroy_cb, NULL); free(scanner->region_rslt_buff); scanner->region_rslt_buff=NULL; + free(scanner->gie_rslt_buff); + scanner->gie_rslt_buff=NULL; alignment_int64_array_free(scanner->ref_cnt); scanner->ref_cnt=NULL; for(i=0;igarbage_q From 6002ff094f083fac4bc7cca590d7f3ed527023f5 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Thu, 19 Sep 2019 18:23:55 +0800 Subject: [PATCH 15/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dbug=EF=BC=9A=E5=BD=93?= =?UTF-8?q?=E5=8C=85=E5=90=AB=E5=A4=96=E9=94=AE=E7=9A=84=E4=B8=80=E4=B8=AA?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E5=85=88=E6=97=A0=E6=95=88=E5=86=8D=E6=9C=89?= =?UTF-8?q?=E6=95=88=E6=97=B6=EF=BC=8C=E5=8D=B3=E9=85=8D=E7=BD=AEID?= =?UTF-8?q?=E9=87=8D=E7=94=A8=EF=BC=8C=E5=A4=96=E9=94=AE=E7=94=9F=E6=88=90?= =?UTF-8?q?=E7=9A=84=E6=9C=AC=E5=9C=B0=E6=96=87=E4=BB=B6=E4=BC=9A=E8=A2=AB?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E7=9A=84=E5=88=A0=E9=99=A4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/Maat_rule.h | 3 +- src/entry/Maat_api.cpp | 4 -- src/entry/Maat_command.cpp | 65 +++++++++++---------------- src/entry/Maat_rule.cpp | 2 +- src/inc_internal/Maat_rule_internal.h | 1 - test/perf_test_maatframe.cpp | 4 +- test/test_maatframe.cpp | 2 - 7 files changed, 30 insertions(+), 51 deletions(-) diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 71eb650..752fb5e 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -158,8 +158,7 @@ enum MAAT_INIT_OPT //This option also disables background update. MAAT_OPT_ENABLE_UPDATE, //VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled. DEFAULT: Backgroud update is enabled. Runtime setting is allowed. MAAT_OPT_ACCEPT_TAGS, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]} - MAAT_OPT_FOREIGN_CONT_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files - MAAT_OPT_FOREIGN_CONT_LINGER //VALUE is interger *, SIZE=sizeof(int). Greater than 0: delete after VALUE seconds; 0: delete foreign content right after the notification callbacks; Less than 0: NEVER delete. Default: 0. + MAAT_OPT_FOREIGN_CONT_DIR //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files }; //return -1 if failed, return 0 on success; int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size); diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index d3ac7e9..c97311e 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -543,7 +543,6 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* feather->base_rgn_seq=0; feather->AUTO_NUMBERING_ON=1; feather->backgroud_update_enabled=1; - feather->foreign_cont_linger=0; snprintf(feather->foreign_cont_dir, sizeof(feather->foreign_cont_dir), "%s_files", table_info_path); pthread_mutex_init(&(feather->background_update_mutex),NULL); snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path); @@ -758,9 +757,6 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo _feather->foreign_cont_dir[strlen(_feather->foreign_cont_dir)-1]='\0'; } break; - case MAAT_OPT_FOREIGN_CONT_LINGER: - _feather->foreign_cont_linger=*(int*)value; - break; default: return -1; } diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 353667b..3496ef8 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -1732,22 +1732,38 @@ void _get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int for(i=0;iop==MAAT_OP_DEL||p->n_foreign==0) + if(p->n_foreign==0) { continue; } - for(j=0; jn_foreign; j++) + if(p->op==MAAT_OP_DEL) { - if(p->f_keys[j].is_existed==1) + for(j=0; jn_foreign; j++) + { + if(p->f_keys[j].is_existed==1) + { + continue; + } + snprintf(redis_cmd,sizeof(redis_cmd),"GET %s", p->f_keys[j].key); + ret=redisAppendCommand(ctx, redis_cmd); + track[key_num].rule_idx=i; + track[key_num].foreign_idx=j; + key_num++; + assert(ret==REDIS_OK); } - snprintf(redis_cmd,sizeof(redis_cmd),"GET %s", p->f_keys[j].key); - ret=redisAppendCommand(ctx, redis_cmd); - track[key_num].rule_idx=i; - track[key_num].foreign_idx=j; - key_num++; - assert(ret==REDIS_OK); } } for(i=0;iforeign_cont_linger==0) - { - ret=system_cmd_rm(rule_list[i].f_keys[j].filename); - if(ret==-1) - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, - "Foreign content file %s remove failed.", - rule_list[i].f_keys[j].filename); - } - } - else if(feather->foreign_cont_linger>0) - { - garbage_bagging_with_timeout(GARBAGE_FOREIGN_FILE, rule_list[i].f_keys[j].filename, feather->foreign_cont_linger, feather->garbage_q); - rule_list[i].f_keys[j].filename=NULL;//transfer owner to garbage collection. - } - else - { - //Less than 0, don't delete. - } - } - - } } finish(u_para); diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 57830e7..2e0f512 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -33,7 +33,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_8_20190827=1; +int MAAT_FRAME_VERSION_2_8_20190919=1; int is_valid_table_name(const char* str) { diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index c45d457..ecef3d2 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -350,7 +350,6 @@ struct _Maat_feather_t int n_tags; char foreign_cont_dir[MAX_TABLE_NAME_LEN]; - int foreign_cont_linger; //internal states long long new_version; diff --git a/test/perf_test_maatframe.cpp b/test/perf_test_maatframe.cpp index 806dd2f..f139e39 100644 --- a/test/perf_test_maatframe.cpp +++ b/test/perf_test_maatframe.cpp @@ -84,9 +84,7 @@ protected: const char* foregin_dir="./foreign_files/"; Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_DIR, foregin_dir, strlen(foregin_dir)+1); - int linger_timeout=2; - Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_LINGER, &linger_timeout, sizeof(linger_timeout)); - + Maat_cmd_flushDB(_shared_feather); Maat_initiate_feather(_shared_feather); } diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 6bfe6d1..dae84dd 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -1630,8 +1630,6 @@ protected: const char* foregin_dir="./foreign_files/"; Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_DIR,foregin_dir, strlen(foregin_dir)+1); - linger_timeout=2; - Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_LINGER,&linger_timeout, sizeof(linger_timeout)); Maat_cmd_flushDB(_shared_feather); Maat_initiate_feather(_shared_feather); From fb9f5587ef9dac404ab98a57199d01500e70ac7a Mon Sep 17 00:00:00 2001 From: zhengchao Date: Thu, 19 Sep 2019 20:09:01 +0800 Subject: [PATCH 16/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dbug=EF=BC=9A=E4=B8=8A?= =?UTF-8?q?=E6=AC=A1=E6=8F=90=E4=BA=A4=E6=9C=AA=E5=BD=BB=E5=BA=95=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E9=85=8D=E7=BD=AEID=E9=87=8D=E7=94=A8=E6=97=B6?= =?UTF-8?q?=E5=A4=96=E9=94=AE=E6=96=87=E4=BB=B6=E6=97=A0=E6=95=88=E7=9A=84?= =?UTF-8?q?bug=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_command.cpp | 14 +++++--------- src/inc_internal/Maat_rule_internal.h | 1 - 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 3496ef8..51d407d 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -1607,10 +1607,9 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p) } void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n_foreign, const char* dir, void* logger) { - int ret=0, i=0; + int i=0; const char* p_foreign=NULL; int foreign_key_size=0; - struct stat file_info; p_rule->n_foreign=n_foreign; p_rule->f_keys=ALLOC(struct foreign_key, n_foreign); for(i=0; if_keys[i].key=(char*)calloc(sizeof(char),foreign_key_size+1); memcpy(p_rule->f_keys[i].key, p_foreign, foreign_key_size); - p_rule->f_keys[i].filename=get_foreign_cont_filename(p_rule->table_name, p_rule->rule_id, p_rule->f_keys[i].key, dir); - ret=stat(p_rule->f_keys[i].filename, &file_info); - if(ret==0) - { - p_rule->f_keys[i].is_existed=1; - } } if(i!=n_foreign) { @@ -1729,6 +1722,8 @@ void _get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int redisReply* reply=NULL; struct serial_rule_t*p=NULL; FILE* fp=NULL; + struct stat file_info; + for(i=0;in_foreign; j++) { - if(p->f_keys[j].is_existed==1) + ret=stat(p->f_keys[i].filename, &file_info); + if(ret==0) { continue; } diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index ecef3d2..8ccdb81 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -396,7 +396,6 @@ struct _maat_garbage_t struct foreign_key { int column; - int is_existed; char* key; char* filename; }; From 2f10504d643926f43eb38397ed5333f5c3ab5171 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Thu, 21 Nov 2019 18:48:22 +0800 Subject: [PATCH 17/29] =?UTF-8?q?1=E3=80=81=E4=BF=AE=E5=A4=8Dbug=EF=BC=9Av?= =?UTF-8?q?irtual=20table=E6=89=AB=E6=8F=8F=E6=97=B6=EF=BC=8C=E6=9C=AA?= =?UTF-8?q?=E8=83=BD=E6=AD=A3=E7=A1=AE=E8=AF=86=E5=88=AB=E4=B8=8A=E4=B8=80?= =?UTF-8?q?=E6=AC=A1=E5=91=BD=E4=B8=AD=E7=9A=84compile=E7=9A=84bug?= =?UTF-8?q?=EF=BC=8C=E5=AF=BC=E8=87=B4compile=E5=A4=9A=E6=AC=A1=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=E3=80=822=E3=80=81=E4=BF=AE=E5=A4=8Dbug=EF=BC=9A?= =?UTF-8?q?=E5=9C=A8=E8=BF=9B=E8=A1=8Cvirtual=20table+=20Expr=20plus?= =?UTF-8?q?=E7=9A=84=E6=89=AB=E6=8F=8F=E6=97=B6=EF=BC=8C=E7=94=B1=E4=BA=8E?= =?UTF-8?q?=E7=AC=94=E8=AF=AF=EF=BC=8C=E9=94=99=E8=AF=AF=E7=9A=84=E8=AE=A4?= =?UTF-8?q?=E4=B8=BAtable=20id=E4=B8=8D=E5=90=88=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 6 +++--- src/entry/Maat_rule.cpp | 2 +- src/entry/Maat_table.cpp | 4 ++-- src/inc_internal/Maat_utils.h | 2 +- test/test_maatframe.cpp | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index c97311e..ba3999b 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -103,7 +103,7 @@ size_t pickup_hit_region_from_compile(struct bool_expr *compile_hit, group_id=(unsigned long long)dynamic_array_read(hitted_id, i); for(j=0; jitem_num; j++) { - if(group_id==(compile_hit->items[j].item_id&0x00000000ffffffff)) + if(group_id==compile_hit->items[j].item_id) { region_pos[k]=i; k++; @@ -198,13 +198,13 @@ int region_compile(_Maat_feather_t*feather, struct _INNER_scan_status_t *_mid, c { if(_mid->cur_hit_group_cntcur_hit_groups, _mid->cur_hit_group_cnt, (void*)group_rule->top_groups[j]); + dynamic_array_write(_mid->cur_hit_groups, _mid->cur_hit_group_cnt, (void*)TO_RELATION_ID(region_hit_wraper->virtual_table_id,group_rule->top_groups[j])); _mid->cur_hit_group_cnt++; } ret=insert_set_id(&(_mid->all_hit_group_array), &(_mid->all_hit_group_array_sz), _mid->all_hit_group_cnt, - (unsigned long long)region_hit_wraper->virtual_table_id<<32|group_rule->top_groups[j]); + TO_RELATION_ID(region_hit_wraper->virtual_table_id, group_rule->top_groups[j])); _mid->all_hit_group_cnt+=ret; } } diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 2e0f512..c3be724 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -33,7 +33,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_8_20190919=1; +int MAAT_FRAME_VERSION_2_8_20191121=1; int is_valid_table_name(const char* str) { diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table.cpp index 5cd8a64..32b2c39 100644 --- a/src/entry/Maat_table.cpp +++ b/src/entry/Maat_table.cpp @@ -551,8 +551,8 @@ struct Maat_table_desc * Maat_table_get_by_id(struct Maat_table_manager* table_m } if(p_real_table->table_type!=expect_type) { - if((expect_type==TABLE_TYPE_EXPR && p_table->table_type!=TABLE_TYPE_EXPR_PLUS)|| - (expect_type==TABLE_TYPE_IP && p_table->table_type!=TABLE_TYPE_IP_PLUS)) + if((expect_type==TABLE_TYPE_EXPR && p_real_table->table_type!=TABLE_TYPE_EXPR_PLUS)|| + (expect_type==TABLE_TYPE_IP && p_real_table->table_type!=TABLE_TYPE_IP_PLUS)) { return NULL; } diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index 358b826..ab233a2 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -46,7 +46,7 @@ #define UNUSED __attribute__((unused)) const char* module_name_str(const char*name); #define maat_module (module_name_str("MAAT_Frame")) - +#define TO_RELATION_ID(vid, gid) ((unsigned long long)vid<<32|gid) char* _maat_strdup(const char* s); char* str_unescape(char* s); inline void ipv6_ntoh(unsigned int *v6_addr) diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index dae84dd..0c117aa 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -1491,7 +1491,7 @@ TEST(VirtualTable, Test1) EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 160); - Maat_clean_status(&mid); + const char* should_not_hit="2018-10-05 is a keywords of table KEYWORDS_TABLE. Should not hit."; mid=NULL; From 5f06db95586dc4a9e3a24792c4bde1ae567db4a9 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 26 Nov 2019 14:40:03 +0800 Subject: [PATCH 18/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dbug=EF=BC=9A=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E5=88=86=E7=BB=84=E4=BB=A5=E8=99=9A=E6=8B=9F=E8=A1=A8?= =?UTF-8?q?=E5=A4=9A=E6=AC=A1=E5=87=BA=E7=8E=B0=E5=9C=A8=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E5=86=85=EF=BC=8C=E9=94=99=E8=AF=AF=E7=9A=84?= =?UTF-8?q?=E5=88=A4=E6=96=AD=E4=B8=BA=E5=88=86=E7=BB=84=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_rule.cpp | 61 +++++++++++------------ test/maat_json.json | 107 ++++++++++++++++++++++++++++++++++++++-- test/table_info.conf | 6 ++- test/test_maatframe.cpp | 87 ++++++++++++++++++++++++++++++-- 4 files changed, 221 insertions(+), 40 deletions(-) diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index c3be724..0f88fa3 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -33,7 +33,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_8_20191121=1; +int MAAT_FRAME_VERSION_2_8_20191126=1; int is_valid_table_name(const char* str) { @@ -1445,7 +1445,7 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa } else { - if(p->group_id==a_rule_group->group_id)//duplicate group + if(p->group_id==a_rule_group->group_id && relation->virtual_table_id[i]==virual_table_id)//duplicate group { ret=-1; goto error_out; @@ -2021,7 +2021,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "update error,add group: %s %d to compile rule %d error, compile rule is full or duplicate group.", + "update error, add group: %s %d to compile rule %d error, compile rule is full or duplicate group.", table->table_name[table->updating_name], db_group_rule->group_id, db_group_rule->parent_id); @@ -2232,7 +2232,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru ret=add_group_rule(table,&db_group_rule, scanner, logger); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + MESA_handle_runtime_log(logger,RLOG_LV_FATAL, maat_module, "duplicate config of group table %s group_id %d compile_id %d.", table->table_name[0], db_group_rule.group_id, db_group_rule.parent_id); @@ -2271,8 +2271,8 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc ,&(maat_str_rule->is_valid)); if(ret!=7) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config: invalid format of expr table %s:%s",table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config: invalid format of expr table %s:%s", table->table_name[table->updating_name], table_line); free(maat_str_rule); maat_str_rule=NULL; table->udpate_err_cnt++; @@ -2290,8 +2290,8 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc ,&(maat_str_rule->is_valid)); if(ret!=8) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config: invalid format of expr_plus table %s:%s",table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config: invalid format of expr_plus table %s:%s", table->table_name[table->updating_name], table_line); free(maat_str_rule); maat_str_rule=NULL; table->udpate_err_cnt++; @@ -2317,30 +2317,30 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc maat_str_rule->is_case_sensitive=TRUE; break; default: - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config %d:update error,invalid hexbin value of expr table %s:%s" - ,maat_str_rule->region_id - ,table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config %d:update error,invalid hexbin value of expr table %s:%s", + maat_str_rule->region_id, + table->table_name[table->updating_name], table_line); table->udpate_err_cnt++; goto error_out; } if(!is_valid_match_method(maat_str_rule->match_method)) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config %d:update error,invalid match method=%d in expr table %s:%s" - ,maat_str_rule->region_id - ,maat_str_rule->match_method - ,table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config %d:update error,invalid match method=%d in expr table %s:%s", + maat_str_rule->region_id, + maat_str_rule->match_method, + table->table_name[table->updating_name],table_line); table->udpate_err_cnt++; goto error_out; } if(!is_valid_expr_type(maat_str_rule->expr_type)) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config %d:update error,invalid expr type=%d in expr table %s:%s" - ,maat_str_rule->region_id - ,maat_str_rule->expr_type - ,table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config %d:update error,invalid expr type=%d in expr table %s:%s", + maat_str_rule->region_id, + maat_str_rule->expr_type, + table->table_name[table->updating_name], table_line); table->udpate_err_cnt++; goto error_out; } @@ -2365,9 +2365,8 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc { rule_type=RULETYPE_STR; } - ret=del_region_rule(table - ,maat_str_rule->region_id,maat_str_rule->group_id,rule_type - ,scanner, logger); + ret=del_region_rule(table, maat_str_rule->region_id, maat_str_rule->group_id, rule_type, + scanner, logger); if(ret<0) { table->udpate_err_cnt++; @@ -2382,18 +2381,18 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc if(maat_str_rule->expr_type==EXPR_TYPE_AND &&maat_str_rule->match_method!=MATCH_METHOD_SUB) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "table %s region cfg %d is EXPR_TYPE_AND,but match method is not MATCH_METHOD_SUB,force fixed.", - table->table_name[table->updating_name],maat_str_rule->region_id); + table->table_name[table->updating_name], maat_str_rule->region_id); maat_str_rule->match_method=MATCH_METHOD_SUB; } - ret=add_expr_rule(table, maat_str_rule,scanner, logger); + ret=add_expr_rule(table, maat_str_rule, scanner, logger); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "duplicate config of expr table %s region_id=%d" - ,table->table_name[table->updating_name],maat_str_rule->region_id); + MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module, + "duplicate config of expr table %s region_id=%d", + table->table_name[table->updating_name], maat_str_rule->region_id); table->udpate_err_cnt++; } else diff --git a/test/maat_json.json b/test/maat_json.json index 47da2f5..df4c290 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -196,7 +196,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_REGION", + "table_name": "HTTP_SIGNATURE", "table_type": "expr_plus", "table_content": { "district": "HTTP\\bURL", @@ -1027,7 +1027,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_REGION", + "table_name": "HTTP_SIGNATURE", "table_type": "expr_plus", "table_content": { "district": "Content-Type", @@ -1150,7 +1150,7 @@ "do_blacklist": 0, "do_log": 0, "effective_rage": 0, - "user_region": "Virtual", + "user_region": "VirtualWithPhysical", "is_valid": "yes", "groups": [ { @@ -1174,6 +1174,107 @@ ] } ] + }, + { + "compile_id": 161, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "virtual_table_test_temp", + "is_valid": "yes", + "groups": [ + { + "group_name":"vt_grp_http_sig1", + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "User-Agent", + "keywords": "Chrome/78.0.3904.108", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "group_name":"vt_grp_http_sig2", + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "Cookie", + "keywords": "uid=12345678", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + }, + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "Cookie", + "keywords": "sessionid=888888", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 162, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "VirtualWithVirtual", + "is_valid": "yes", + "groups": [ + { + "group_name":"vt_grp_http_sig1", + "virtual_table":"HTTP_REQUEST_HEADER", + "not_flag":0 + }, + { + "group_name":"vt_grp_http_sig2", + "virtual_table":"HTTP_RESPONSE_HEADER", + "not_flag":0 + } + ] + }, + { + "compile_id": 163, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "OneGroupInTwoVirtual", + "is_valid": "yes", + "groups": [ + { + "group_name":"vt_grp_http_sig2", + "virtual_table":"HTTP_REQUEST_HEADER", + "not_flag":0 + }, + { + "group_name":"vt_grp_http_sig2", + "virtual_table":"HTTP_RESPONSE_HEADER", + "not_flag":0 + } + ] } ], "plugin_table": [ diff --git a/test/table_info.conf b/test/table_info.conf index e960a64..013dfeb 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -27,7 +27,7 @@ 5 CONTENT_SIZE intval -- 6 QD_ENTRY_INFO plugin 4 -- 7 FILE_DIGEST digest -- -8 HTTP_REGION expr_plus GBK GBK yes 0 +8 HTTP_SIGNATURE expr_plus GBK GBK yes 0 9 SIM_URL similar -- 10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff 11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} -- @@ -39,4 +39,6 @@ 17 TROJAN_PAYLOAD expr UTF8 UTF8 yes 0 quickoff 18 MAIL_ADDR expr UTF8 UTF8 yes 0 quickoff 19 IP_PLUS_CONFIG ip_plus -- -20 HTTP_RESPONSE_KEYWORDS virtual KEYWORDS_TABLE -- \ No newline at end of file +20 HTTP_RESPONSE_KEYWORDS virtual KEYWORDS_TABLE -- +21 HTTP_REQUEST_HEADER virtual HTTP_SIGNATURE -- +22 HTTP_RESPONSE_HEADER virtual HTTP_SIGNATURE -- \ No newline at end of file diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 0c117aa..5682666 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -357,7 +357,7 @@ TEST(StringScan, ExprPlus) int found_pos[4]; const char* region_name="HTTP URL"; const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=abckkk,1234567"; - table_id=Maat_table_register(g_feather, "HTTP_REGION"); + table_id=Maat_table_register(g_feather, "HTTP_SIGNATURE"); ASSERT_GT(table_id, 0); scan_status_t mid=NULL; ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, scan_data, strlen(scan_data), @@ -437,7 +437,7 @@ TEST(StringScan, ExprPlusWithHex) const char* scan_data="text/html; charset=UTF-8"; const char* region_name="Content-Type"; int found_pos[4]; - table_id=Maat_table_register(g_feather, "HTTP_REGION"); + table_id=Maat_table_register(g_feather, "HTTP_SIGNATURE"); ASSERT_GT(table_id, 0); scan_status_t mid=NULL; ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, region_name, strlen(region_name)); @@ -1462,9 +1462,9 @@ TEST(ScanResult, LongerServiceDefine) free(buff); return; } -TEST(VirtualTable, Test1) +TEST(VirtualTable, VirtualWithPhysical) { -#define TestVirtualTable +#define TestVirtualTable1 int ret=0, table_id=0; const char* http_content="Batman\\:Take me Home.Superman/:Fine,stay with me."; const char* http_url="https://blog.csdn.net/littlefang/article/details/8213058"; @@ -1502,6 +1502,85 @@ TEST(VirtualTable, Test1) return; } +TEST(VirtualTable, VirtualWithVirtual) +{ +#define TestVirtualTable2 + int ret=0, table_id=0; + const char* http_req_hdr_ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"; + const char* http_resp_hdr_cookie="uid=12345678;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; + + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + + table_id=Maat_table_register(g_feather, "HTTP_REQUEST_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "User-Agent", strlen("User-Agent")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_req_hdr_ua, strlen(http_req_hdr_ua), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, -2); + + + table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 162); + + Maat_clean_status(&mid); + + return; +} +TEST(VirtualTable, OneGroupInTwoVirtual) +{ +#define TestVirtualTable3 + int ret=0, table_id=0; + const char* http_resp_hdr_cookie="sessionid=888888;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; + + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + + table_id=Maat_table_register(g_feather, "HTTP_REQUEST_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, -2); + + + table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 163); + + Maat_clean_status(&mid); + + return; +} class MaatFileTest : public testing::Test From ce2c64cb231d80e178936eb5500c15c1d67820aa Mon Sep 17 00:00:00 2001 From: zhengchao Date: Wed, 27 Nov 2019 11:32:10 +0800 Subject: [PATCH 19/29] =?UTF-8?q?=E5=B0=86=E6=9C=AA=E7=9F=A5=E8=99=9A?= =?UTF-8?q?=E6=8B=9F=E6=9C=BA=E8=A1=A8=E7=9A=84=E5=91=8A=E8=AD=A6=E7=BA=A7?= =?UTF-8?q?=E5=88=AB=E9=99=8D=E4=BD=8E=E4=B8=BADebug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_rule.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 0f88fa3..64ea8c6 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -2193,7 +2193,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru } if(db_group_rule.parent_type==PARENT_TYPE_GROUP && db_group_rule.not_flag) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module , "update error, invalid format of group table %s:%s not operation is forbidden for non-compile parent.", table->table_name[table->updating_name], table_line); table->udpate_err_cnt++; @@ -2204,8 +2204,10 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru db_group_rule.virtual_table_id=Maat_table_get_id_by_name(table_mgr, virtual_table_name); if(db_group_rule.virtual_table_id<0) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "update error, unknown virutal table name: %s of group table %s:%s.", + //This happens when one data source (e.g. redis) is consumed by multiple Maat instance. + //Maat ignores unrealated groups. + MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, maat_module, + "group table load abandon, unknown virtual table name: %s of group table %s:%s.", virtual_table_name, table->table_name[table->updating_name], table_line); table->udpate_err_cnt++; From 08bf1e9228aa5e3e86464538c5417dcafa0f1311 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Fri, 29 Nov 2019 11:33:20 +0800 Subject: [PATCH 20/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dbug=EF=BC=9A=E5=88=A0?= =?UTF-8?q?=E9=99=A4group=E6=97=B6=EF=BC=8C=E6=9C=AA=E6=A3=80=E6=9F=A5virt?= =?UTF-8?q?ual=20table=20id=E3=80=82make=5Fgroup=5Fset=E4=B8=AD=EF=BC=8C?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E9=94=99=E8=AF=AF=E7=9A=84=E4=B8=8B=E6=A0=87?= =?UTF-8?q?=EF=BC=8C=E5=AF=BC=E8=87=B4=E5=88=A0=E9=99=A4=E5=88=86=E7=BB=84?= =?UTF-8?q?=E5=90=8E=E4=B8=8D=E5=91=BD=E4=B8=AD=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_rule.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 64ea8c6..9c08b90 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -33,7 +33,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_8_20191126=1; +int MAAT_FRAME_VERSION_2_8_20191129=1; int is_valid_table_name(const char* str) { @@ -637,8 +637,8 @@ void make_group_set(struct Maat_compile_group_relation* relation, struct bool_ex continue; } //high 32 bit is virtual table id, low 32 bit is group id. - a_set->items[j].item_id=(unsigned long long)relation->virtual_table_id[j]<<32|group->group_id; - a_set->items[j].not_flag=relation->not_flag[j]; + a_set->items[j].item_id=TO_RELATION_ID(relation->virtual_table_id[i], group->group_id); + a_set->items[j].not_flag=relation->not_flag[i]; if(a_set->items[j].not_flag) { *has_not=1; @@ -1481,7 +1481,7 @@ error_out: return ret; } -struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relation*relation, int group_id) +struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relation*relation, int group_id, int virual_table_id) { int i=0; struct Maat_group_inner* group_rule=NULL; @@ -1493,14 +1493,15 @@ struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relati { continue; } - if(group_rule->group_id==group_id) + if(group_rule->group_id==group_id && relation->virtual_table_id[i]==virual_table_id) { dynamic_array_write(relation->groups,i,NULL); if(relation->not_flag[i]==1) { relation->not_group_cnt--; relation->not_flag[i]=0; - } + } + relation->virtual_table_id[i]=0; relation->group_cnt--; break; } @@ -2099,7 +2100,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro db_group_rule->parent_id); return 0; } - group_rule=del_group_from_compile(relation, db_group_rule->group_id); + group_rule=del_group_from_compile(relation, db_group_rule->group_id, db_group_rule->virtual_table_id); if(group_rule==NULL) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, From ead6efa2773256dba647d88a586c5396e0ccc2b9 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Fri, 29 Nov 2019 13:43:38 +0800 Subject: [PATCH 21/29] =?UTF-8?q?=E5=A2=9E=E5=8A=A0Virtual=20Table?= =?UTF-8?q?=E5=88=86=E7=BB=84=E5=88=A0=E9=99=A4=E7=9A=84=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=94=A8=E4=BE=8B=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/Maat_command.h | 1 + src/entry/Maat_command.cpp | 100 ++++++++++++++++--------------- test/test_maatframe.cpp | 117 +++++++++++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 47 deletions(-) diff --git a/inc/Maat_command.h b/inc/Maat_command.h index 816dd63..5e0c6e1 100644 --- a/inc/Maat_command.h +++ b/inc/Maat_command.h @@ -109,6 +109,7 @@ struct Maat_region_t struct Maat_group_t { const char* table_name; + const char* virtual_table_name; int group_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number. int parent_id; int not_flag; diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 51d407d..6d061fd 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -247,11 +247,17 @@ int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) void serialize_group(const struct Maat_group_t* p_group, enum MAAT_OPERATION op, char* buff, size_t sz) { if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD; - snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d\tnull", p_group->group_id, + const char* vt_name="null"; + if(p_group->virtual_table_name!=NULL) + { + vt_name=p_group->virtual_table_name; + } + snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d\t%s", p_group->group_id, p_group->parent_id, op, p_group->not_flag, - p_group->parent_type); + p_group->parent_type, + vt_name); return; } void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int group_num, enum MAAT_OPERATION op, char* buff, size_t sz) @@ -276,65 +282,65 @@ void serialize_region(const struct Maat_region_t* p, int group_id, char* buff, s switch(p->region_type) { case REGION_IP: - ret=snprintf(buff,sz,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1" - ,p->region_id - ,group_id - ,p->ip_rule.addr_type - ,p->ip_rule.src_ip - ,p->ip_rule.mask_src_ip - ,p->ip_rule.src_port - ,p->ip_rule.mask_src_port - ,p->ip_rule.dst_ip - ,p->ip_rule.mask_dst_ip - ,p->ip_rule.dst_port - ,p->ip_rule.mask_dst_port - ,p->ip_rule.protocol - ,p->ip_rule.direction); + ret=snprintf(buff,sz,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1", + p->region_id, + group_id, + p->ip_rule.addr_type, + p->ip_rule.src_ip, + p->ip_rule.mask_src_ip, + p->ip_rule.src_port, + p->ip_rule.mask_src_port, + p->ip_rule.dst_ip, + p->ip_rule.mask_dst_ip, + p->ip_rule.dst_port, + p->ip_rule.mask_dst_port, + p->ip_rule.protocol, + p->ip_rule.direction); break; case REGION_EXPR: if(p->expr_rule.district==NULL) { - ret=snprintf(buff,sz,"%d\t%d\t%s\t%d\t%d\t%d\t1" - ,p->region_id - ,group_id - ,p->expr_rule.keywords - ,p->expr_rule.expr_type - ,p->expr_rule.match_method - ,p->expr_rule.hex_bin); + ret=snprintf(buff,sz,"%d\t%d\t%s\t%d\t%d\t%d\t1", + p->region_id, + group_id, + p->expr_rule.keywords, + p->expr_rule.expr_type, + p->expr_rule.match_method, + p->expr_rule.hex_bin); } else //expr_plus { - ret=snprintf(buff,sz,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1" - ,p->region_id - ,group_id - ,p->expr_rule.keywords - ,p->expr_rule.district - ,p->expr_rule.expr_type - ,p->expr_rule.match_method - ,p->expr_rule.hex_bin); + ret=snprintf(buff,sz,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1", + p->region_id, + group_id, + p->expr_rule.district, + p->expr_rule.keywords, + p->expr_rule.expr_type, + p->expr_rule.match_method, + p->expr_rule.hex_bin); } break; case REGION_INTERVAL: - ret=snprintf(buff,sz,"%d\t%d\t%u\t%u\t1" - ,p->region_id - ,group_id - ,p->interval_rule.low_boundary - ,p->interval_rule.up_boundary); + ret=snprintf(buff,sz,"%d\t%d\t%u\t%u\t1", + p->region_id, + group_id, + p->interval_rule.low_boundary, + p->interval_rule.up_boundary); break; case REGION_DIGEST: - ret=snprintf(buff,sz,"%d\t%d\t%llu\t%s\t%hd\t1" - ,p->region_id - ,group_id - ,p->digest_rule.orgin_len - ,p->digest_rule.digest_string - ,p->digest_rule.confidence_degree); + ret=snprintf(buff,sz,"%d\t%d\t%llu\t%s\t%hd\t1", + p->region_id, + group_id, + p->digest_rule.orgin_len, + p->digest_rule.digest_string, + p->digest_rule.confidence_degree); break; case REGION_SIMILARITY: - ret=snprintf(buff,sz,"%d\t%d\t%s\t%hd\t1" - ,p->region_id - ,group_id - ,p->similarity_rule.target - ,p->similarity_rule.threshold); + ret=snprintf(buff,sz,"%d\t%d\t%s\t%hd\t1", + p->region_id, + group_id, + p->similarity_rule.target, + p->similarity_rule.threshold); break; default: assert(0); diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 5682666..3cffc1b 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -2292,6 +2292,123 @@ TEST_F(MaatCmdTest, RefGroup) EXPECT_EQ(result[0].config_id, compile1.config_id); Maat_clean_status(&mid); +} +#define MaatCmdTest_VirtualTable +TEST_F(MaatCmdTest, VirtualTable) +{ + Maat_feather_t feather=MaatCmdTest::_shared_feather; + const char* group_table_name="GROUP"; + const char* compile_table_name="COMPILE"; + const char* region_table_name="HTTP_SIGNATURE"; + + struct Maat_rule_t compile1; + struct Maat_group_t group1, group2; + struct Maat_region_t region1, region2; + + memset(&compile1, 0, sizeof(compile1)); + compile1.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1); + Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 2); + + + //group1->compile1 + memset(&group1, 0, sizeof(group1)); + group1.group_id=Maat_cmd_get_new_group_id(feather); + group1.table_name=group_table_name; + group1.virtual_table_name="HTTP_REQUEST_HEADER"; + group1.parent_id=compile1.config_id; + group1.parent_type=PARENT_TYPE_COMPILE; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1); + + + /*region1->group1->compile1 + */ + memset(®ion1, 0, sizeof(region1)); + region1.region_id=Maat_cmd_get_new_region_id(feather); + region1.region_type=REGION_EXPR; + region1.table_name=region_table_name; + region1.expr_rule.district="User-Agent"; + region1.expr_rule.keywords="AppleWebKit"; + region1.expr_rule.expr_type=EXPR_TYPE_STRING; + Maat_command_raw_set_region(feather, MAAT_OP_ADD, ®ion1, group1.group_id); + + + //group2->compile1 + memset(&group2, 0, sizeof(group2)); + group2.group_id=Maat_cmd_get_new_group_id(feather); + group2.table_name=group_table_name; + group2.virtual_table_name="HTTP_RESPONSE_HEADER"; + group2.parent_id=compile1.config_id; + group2.parent_type=PARENT_TYPE_COMPILE; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2); + + //region2->group2 + memset(®ion2, 0, sizeof(region2)); + region2.region_id=Maat_cmd_get_new_region_id(feather); + region2.region_type=REGION_EXPR; + region2.table_name=region_table_name; + region2.expr_rule.district="Cookie"; + region2.expr_rule.keywords="uid=12345678;"; + region2.expr_rule.expr_type=EXPR_TYPE_STRING; + Maat_command_raw_set_region(feather, MAAT_OP_ADD, ®ion2, group2.group_id); + + sleep(1); + + int ret=0, table_id=0; + const char* http_req_hdr_ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"; + const char* http_resp_hdr_cookie="uid=12345678;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; + + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + + table_id=Maat_table_register(feather, "HTTP_REQUEST_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(feather, &mid, MAAT_SET_SCAN_DISTRICT, "User-Agent", strlen("User-Agent")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, http_req_hdr_ua, strlen(http_req_hdr_ua), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, -2); + + + table_id=Maat_table_register(feather, "HTTP_RESPONSE_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, compile1.config_id); + + Maat_clean_status(&mid); + + //Delete group1 + Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1); + Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile1, compile_table_name, NULL, 2); + Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1); + + sleep(1); + + table_id=Maat_table_register(feather, "HTTP_RESPONSE_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, compile1.config_id); + Maat_clean_status(&mid); + return; + } TEST_F(MaatCmdTest, SetLines) From e054471f7ae322e4a5378af8e77873c59e814c07 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Wed, 8 Jan 2020 14:55:58 +0800 Subject: [PATCH 22/29] =?UTF-8?q?=E6=89=AB=E6=8F=8F=E6=94=AF=E6=8C=81windo?= =?UTF-8?q?ws-1251=E7=BC=96=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/Maat_rule.h | 3 ++- src/entry/Maat_rule.cpp | 1 + src/entry/Maat_utils.cpp | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 752fb5e..e9444d6 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -30,7 +30,8 @@ enum MAAT_CHARSET CHARSET_UNICODE_NCR_DEC, //SGML Numeric character reference,decimal base, e.g. "ا" CHARSET_UNICODE_NCR_HEX, //SGML Numeric character reference,hexdecimal base, e.g. "ا" CHARSET_URL_ENCODE_GB2312, //URL encode with GB2312, e.g. the chinese word "china" was encoded to %D6%D0%B9%FA - CHARSET_URL_ENCODE_UTF8 //11, URL encode with UTF8,e.g. the chinese word "china" was encoded to %E4%B8%AD%E5%9B%BD + CHARSET_URL_ENCODE_UTF8, //11, URL encode with UTF8,e.g. the chinese word "china" was encoded to %E4%B8%AD%E5%9B%BD + CHARSET_WINDOWS1251 }; enum MAAT_ACTION { diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 9c08b90..90373d3 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -235,6 +235,7 @@ int universal_charset_convert(struct Maat_scanner* scanner,enum MAAT_CHARSET fro case CHARSET_BIG5: case CHARSET_UNICODE: case CHARSET_UTF8: + case CHARSET_WINDOWS1251: ret=iconv_convert(scanner,from,to,src,srclen,dst,dstlen); return ret; break; diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 892fe09..3d40912 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -242,7 +242,7 @@ char* md5_file(const char* filename, char* md5string) return md5string; } const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", - "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; + "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8", "windows-1251", ""}; const char** charset_get_all_name(void) { From 64ca71e2955a146f6c5cf88842b43668ccec185c Mon Sep 17 00:00:00 2001 From: liuxueli Date: Thu, 9 Jan 2020 18:14:56 +0800 Subject: [PATCH 23/29] =?UTF-8?q?=E6=B7=BB=E5=8A=A0windows1251=E7=BC=96?= =?UTF-8?q?=E7=A0=81=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/maat_json.json | 28 +++++++++++++++- test/table_info.conf | 2 +- test/test_maatframe.cpp | 49 ++++++++++++++++++++++++++++ test/testdata/charsetWindows1251.txt | 48 +++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 test/testdata/charsetWindows1251.txt diff --git a/test/maat_json.json b/test/maat_json.json index df4c290..1c690f9 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -1275,7 +1275,33 @@ "not_flag":0 } ] - } + }, + { + "compile_id": 164, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "CharsetWindows1251", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "string", + "table_content": { + "keywords": ">ЗÐО\\b«Севергазвтоматика\\bÐйС»<", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + } ], "plugin_table": [ { diff --git a/test/table_info.conf b/test/table_info.conf index 013dfeb..b698c40 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -22,7 +22,7 @@ 1 GROUP group -- 2 HTTP_URL expr UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff 2 HTTP_HOST expr UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff -3 KEYWORDS_TABLE expr UTF8 GBK/BIG5/UNICODE/UTF8/unicode_ascii_esc/unicode_ascii_aligned/unicode_ncr_dec/unicode_ncr_hex yes 0 +3 KEYWORDS_TABLE expr UTF8 GBK/BIG5/UNICODE/UTF8/unicode_ascii_esc/unicode_ascii_aligned/unicode_ncr_dec/unicode_ncr_hex/windows-1251 yes 0 4 IP_CONFIG ip -- 5 CONTENT_SIZE intval -- 6 QD_ENTRY_INFO plugin 4 -- diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 3cffc1b..469a524 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -377,6 +377,7 @@ TEST(StringScan, ExprPlus) return; } + TEST(StringScan, ExprPlusWithOffset) { int ret=0, table_id=0; @@ -653,6 +654,54 @@ TEST(NOTLogic, OneRegion) Maat_clean_status(&mid); } + +TEST(StringScan, CharsetWindows1251) +{ + int table_id=0,ret=0; + int read_size=0,pass_flag=0; + struct Maat_rule_t result[4]; + scan_status_t mid=NULL; + //const char* fn="./testdata/mesa_logo.jpg"; + const char* table_name="KEYWORDS_TABLE"; + const char* fn="./testdata/charsetWindows1251.txt"; + FILE* fp=fopen(fn,"r"); + ASSERT_FALSE(fp==NULL); + + char scan_data[4096]={0}; + table_id=Maat_table_register(g_feather,table_name); + ASSERT_GT(table_id, 0); + + + struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10); + stream_para_t sp=Maat_stream_scan_string_start(g_feather,table_id,0); + int detail_ret=0; + + ASSERT_FALSE(sp==NULL); + + while(0==feof(fp)) + { + read_size=fread(scan_data,1,sizeof(scan_data),fp); + ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data,read_size + ,result,4,hit_detail,10 + ,&detail_ret,&mid); + if(ret>0) + { + pass_flag=1; + break; + } + } + EXPECT_EQ(pass_flag, 1); + EXPECT_EQ(result[0].config_id, 164); + + Maat_stream_scan_string_end(&sp); + free(hit_detail); + fclose(fp); + Maat_clean_status(&mid); + return; + +} + + TEST(NOTLogic, ScanNotAtLast) { const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-144."; diff --git a/test/testdata/charsetWindows1251.txt b/test/testdata/charsetWindows1251.txt new file mode 100644 index 0000000..da978a6 --- /dev/null +++ b/test/testdata/charsetWindows1251.txt @@ -0,0 +1,48 @@ +ñÿ àâòîðèçàöèÿ!','JS_CORE_WINDOW_AUTH':'Âîéòè','JS_CORE_IMAGE_FULL':'Ïîëíûé ðàçìåð'}); + + + + + + + + + + + + + + +ÇÀÎ «Ñåâåðãàçâòîìàòèêà ÀéÑ» + + + + + +
+ +
+ + + + + +
+ +Çàêðûòü + +
+ + + + + + +
+ Ëîãèí:
+ decrypt_key)&&strlen(feather->decrypt_algo)) + { + ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&json_buff, err_str, err_str_sz); + } + if(json_buff==NULL)//decryption failed or no decryption. + { + ret=load_file_to_memory(maat_json_fn, &json_buff); + } + ret=json2iris(json_buff, + maat_json_fn, + feather->compile_tn, feather->group_tn, + NULL, + feather->json_ctx.iris_file, + sizeof(feather->json_ctx.iris_file), + feather->logger); + free(json_buff); + json_buff=NULL; + if(ret<0) + { + return -1; + } + strncpy(feather->json_ctx.json_file, maat_json_fn, sizeof(feather->json_ctx.json_file)); + + ret=stat(maat_json_fn, &fstat_buf); + feather->json_ctx.last_md5_time=fstat_buf.st_ctim; + + md5_file(feather->json_ctx.json_file, feather->json_ctx.effective_json_md5); + MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO, maat_module, + "JSON file %s md5: %s, generate index file %s OK.", + feather->json_ctx.json_file, + feather->json_ctx.effective_json_md5, + feather->json_ctx.iris_file); + feather->input_mode=SOURCE_JSON_FILE; + + return 0; +} Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger) { if(max_thread_num<=0) @@ -543,6 +589,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* feather->base_rgn_seq=0; feather->AUTO_NUMBERING_ON=1; feather->backgroud_update_enabled=1; + snprintf(feather->decrypt_algo, sizeof(feather->decrypt_algo), "aes-256-cbc"); snprintf(feather->foreign_cont_dir, sizeof(feather->foreign_cont_dir), "%s_files", table_info_path); pthread_mutex_init(&(feather->background_update_mutex),NULL); snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path); @@ -554,8 +601,8 @@ failed: int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size) { _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct stat attrib; int intval=0,ret=-1; + char err_str[MAX_TABLE_NAME_LEN]; switch(type) { case MAAT_OPT_ENABLE_UPDATE: @@ -630,31 +677,15 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo break; case MAAT_OPT_JSON_FILE_PATH: assert(_feather->input_mode==SOURCE_NONE); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Maat initial with JSON file %s, formating..", - (const char*)value); - - ret=json2iris((const char*)value, - _feather->compile_tn,_feather->group_tn, - NULL, - _feather->json_ctx.iris_file, - sizeof(_feather->json_ctx.iris_file), - _feather->logger); + ret=load_maat_json_file(_feather, (const char *)value, err_str, sizeof(err_str)); if(ret<0) { + MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, + "Load maat json file %s failed: %s.", + (const char*)value, err_str); return -1; } - memcpy(_feather->json_ctx.json_file, value, size); - stat(_feather->json_ctx.json_file, &attrib); - _feather->json_ctx.last_md5_time=attrib.st_ctime; - md5_file(_feather->json_ctx.json_file, _feather->json_ctx.effective_json_md5); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "JSON file %s md5: %s, generate index file %s OK.", - _feather->json_ctx.json_file, - _feather->json_ctx.effective_json_md5, - _feather->json_ctx.iris_file); - _feather->input_mode=SOURCE_JSON_FILE; break; case MAAT_OPT_STAT_ON: _feather->stat_on=1; @@ -667,10 +698,10 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo { return -1; } - memcpy(_feather->stat_file,(const char*)value,size); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Maat performance statistic output to %s." - ,(const char*)value); + memcpy(_feather->stat_file, (const char*)value, size); + MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module, + "Maat performance statistic output to %s.", + (const char*)value); _feather->stat_on=1; break; case MAAT_OPT_SCAN_DETAIL: @@ -678,9 +709,9 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo _feather->rule_scan_type=intval; break; case MAAT_OPT_INSTANCE_NAME: - snprintf(_feather->instance_name - ,sizeof(_feather->instance_name) - ,"%s", + snprintf(_feather->instance_name, + sizeof(_feather->instance_name), + "%s", (const char*)value); break; case MAAT_OPT_DECRYPT_KEY: diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 6d061fd..a51b4de 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -1824,7 +1824,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para void (*finish)(void*),//u_para void* u_para, - const unsigned char* dec_key, + const char* dec_key, _Maat_feather_t* feather) { int table_id=0, i=0, rule_num=0, empty_value_num=0, valid_column=-1; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 90373d3..3d93ad8 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -33,7 +33,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_8_20191129=1; +int MAAT_FRAME_VERSION_2_8_20200113=1; int is_valid_table_name(const char* str) { @@ -3500,7 +3500,8 @@ void *thread_rule_monitor(void *arg) int scan_dir_cnt=0; int ret=0; char md5_tmp[MD5_DIGEST_LENGTH*2+1]={0}; - char tmp_dir[MAX_TABLE_NAME_LEN]={0}; + char err_str[MAX_TABLE_NAME_LEN]={0}; + struct stat attrib; size_t total_wait_rule_cnt=0; @@ -3557,31 +3558,24 @@ void *thread_rule_monitor(void *arg) break; case SOURCE_JSON_FILE: memset(md5_tmp, 0, sizeof(md5_tmp)); - memset(tmp_dir, 0, sizeof(tmp_dir)); stat(feather->json_ctx.json_file, &attrib); - if(attrib.st_ctime!=feather->json_ctx.last_md5_time) + if(memcmp(&attrib.st_ctim, &(feather->json_ctx.last_md5_time), sizeof(attrib.st_ctim))) { - feather->json_ctx.last_md5_time=attrib.st_ctime; + feather->json_ctx.last_md5_time=attrib.st_ctim; md5_file(feather->json_ctx.json_file, md5_tmp); if(0!=strcmp(md5_tmp,feather->json_ctx.effective_json_md5)) { - ret=json2iris(feather->json_ctx.json_file, - feather->compile_tn, feather->group_tn, - NULL, - tmp_dir, - sizeof(tmp_dir), - feather->logger); + ret=load_maat_json_file(feather, feather->json_ctx.json_file, err_str, sizeof(err_str)); if(ret<0) { MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module , - "Maat re-initiate with JSON file %s failed, md5: %s", + "Maat re-initiate with JSON file %s (md5=%s)failed: %s", feather->json_ctx.json_file, - md5_tmp); + md5_tmp, + err_str); } else { - strcpy(feather->json_ctx.effective_json_md5, md5_tmp); - strcpy(feather->json_ctx.iris_file, tmp_dir); config_monitor_traverse(0, feather->json_ctx.iris_file, maat_start_cb, diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 3d40912..f4c16dd 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -1,9 +1,11 @@ #include -#include #include #include -#include #include +#include +#include +#include + #include "Maat_utils.h" pid_t gettid() { @@ -257,4 +259,125 @@ int lqueue_destroy_cb(void *data, long data_len, void *arg) assert(0); return 0; } +#define DECRYPT_BLOCK_SIZE (16*1024) +int decrypt_open(const char* filename, const char* key, const char* algorithm, unsigned char**pp_out, char* err_str, size_t err_str_sz) +{ + unsigned char inbuf[DECRYPT_BLOCK_SIZE]; + int inlen, out_blk_len=0; + int out_buff_len=0,buff_offset=0; + EVP_CIPHER_CTX *ctx; + + unsigned char cipher_key[EVP_MAX_KEY_LENGTH]; + unsigned char cipher_iv[EVP_MAX_IV_LENGTH]; + memset(cipher_key,0,sizeof(cipher_key)); + memset(cipher_iv,0,sizeof(cipher_iv)); + + const EVP_CIPHER *cipher; + const EVP_MD *dgst=NULL; + const unsigned char *salt=NULL; + int ret=0; + + FILE*in=fopen(filename, "r"); + if(in==NULL) + { + return -1; + } + + OpenSSL_add_all_algorithms(); + cipher=EVP_get_cipherbyname(algorithm); + if(cipher==NULL) + { + snprintf(err_str, err_str_sz, "Cipher %s is not supported.",algorithm); + return 0; + } + dgst=EVP_get_digestbyname("md5"); + if(dgst==NULL) + { + snprintf(err_str, err_str_sz, "Get MD5 object failed."); + return 0; + } + ret=EVP_BytesToKey(cipher, dgst, salt, (unsigned char*)key, strlen((const char*)key), 1, cipher_key, cipher_iv); + if(ret==0) + { + snprintf(err_str, err_str_sz, "Key and IV generatioin failed."); + return 0; + } + /* Don't set key or IV right away; we want to check lengths */ + ctx = EVP_CIPHER_CTX_new(); + EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL,0); + OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) % 16==0); + OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16); + + /* Now we can set key and IV */ + EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, 0); + out_buff_len=DECRYPT_BLOCK_SIZE; + *pp_out=(unsigned char*)malloc(out_buff_len*sizeof(unsigned char)); + for (;;) + { + inlen = fread(inbuf, 1, sizeof(inbuf), in); + if (inlen <= 0) + break; + + if(out_buff_len-buff_offsetcfg_path,"r"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error,open %s failed.",index->cfg_path); - return -1; - } + char* table_file_buff=NULL; + int file_sz=0, file_offset=0; + if(strlen(index->encryp_algorithm)>0) { if(key==NULL||strlen((const char*)key)==0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, no key to decrypt %s.",index->cfg_path); - fclose(fp); return -1; } - decrypt_len=decrypt_open(fp, key,index->encryp_algorithm, &decrypt_buff,logger); - if(decrypt_len==0) + file_sz=decrypt_open(index->cfg_path, key, index->encryp_algorithm, (unsigned char**)&table_file_buff, error_string, sizeof(error_string)); + if(file_sz==0) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, %s decrypt failed.",index->cfg_path); - fclose(fp); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, %s decrypt failed: %s", + index->cfg_path, error_string); return -1; } - read_nxt_line_from_buff(decrypt_buff, decrypt_len, &decrypt_offset, line, sizeof(line)); - sscanf(line,"%d\n",&cfg_num); - do_decrypt=1; } else { - fscanf(fp,"%d\n",&cfg_num); + file_sz=load_file_to_memory(index->cfg_path, &table_file_buff); + if(file_sz==0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, %s decrypt failed: %s", + index->cfg_path, error_string); + return -1; + } + } + read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line)); + sscanf(line, "%d\n", &cfg_num); + if(cfg_num!=index->cfg_num) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor ,"file %s config num not matched",index->cfg_path); - fclose(fp); + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor, "file %s config num not matched", index->cfg_path); return -1; } for(i=0;ivaluestring; } - ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,redis_write_ctx,&iris_cfg,logger); + ret=set_iris_descriptor(json_filename, json, compile_tn, group_tn, redis_write_ctx, &iris_cfg, logger); if(ret<0) { goto error_out; @@ -1163,31 +1132,24 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,r ret=create_tmp_dir(&iris_cfg); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, "create tmp folder %s error",iris_cfg.tmp_iris_dir); goto error_out; } - ret=write_iris(json,&iris_cfg,logger); + ret=write_iris(json ,&iris_cfg, logger); if(ret<0) { goto error_out; } - memcpy(iris_dir_buf,iris_cfg.tmp_iris_index_dir,MIN(strlen(iris_cfg.tmp_iris_index_dir)+1,(unsigned int)buf_len)); + memcpy(iris_dir_buf,iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir)+1, (unsigned int)buf_len)); cJSON_Delete(json); - fclose(json_fp); - free(json_buff); clear_iris_descriptor(&iris_cfg); return 0; error_out: cJSON_Delete(json); - if(json_fp!=NULL) - { - fclose(json_fp); - } - free(json_buff); clear_iris_descriptor(&iris_cfg); return -1; } diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 8ccdb81..3aed79e 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -21,7 +21,6 @@ #include #include -extern const char *maat_module; #define mr_region_id_var "SEQUENCE_REGION" @@ -287,7 +286,7 @@ struct source_json_ctx char json_file[MAX_TABLE_NAME_LEN]; char iris_file[MAX_TABLE_NAME_LEN]; char effective_json_md5[MD5_DIGEST_LENGTH*2+1]; - time_t last_md5_time; + struct timespec last_md5_time; }; struct source_redis_ctx { @@ -333,7 +332,8 @@ struct _Maat_feather_t char compile_tn[MAX_TABLE_NAME_LEN]; char group_tn[MAX_TABLE_NAME_LEN]; pthread_mutex_t background_update_mutex; - unsigned char decrypt_key[MAX_TABLE_NAME_LEN]; + char decrypt_key[MAX_TABLE_NAME_LEN]; + char decrypt_algo[MAX_TABLE_NAME_LEN]; pthread_t cfg_mon_t; int AUTO_NUMBERING_ON; @@ -450,14 +450,14 @@ void empty_serial_rules(struct serial_rule_t* rule); int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,unsigned int serial_rule_num, long long server_time, void* logger); long long redis_server_time(redisContext* ctx); redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db, void* logger); -char* md5_file(const char* filename, char* md5string); +int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char* err_str, size_t err_str_sz); void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx, void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para void (*finish)(void*),//u_para void* u_para, - const unsigned char* dec_key, + const char* dec_key, _Maat_feather_t* feather); diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index ab233a2..6f7352f 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -2,6 +2,7 @@ #include "Maat_rule.h" #include #include +#include #include #include //fstat #include //fstat @@ -73,5 +74,7 @@ int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len const char** charset_get_all_name(void); const char* charset_get_name(enum MAAT_CHARSET charset); int lqueue_destroy_cb(void *data, long data_len, void *arg); +int decrypt_open(const char* filename, const char* key, const char* algorithm, unsigned char**pp_out, char* err_str, size_t err_str_sz); +int load_file_to_memory(const char* file_name, char**pp_out); diff --git a/src/inc_internal/config_monitor.h b/src/inc_internal/config_monitor.h index faabf50..6167ce4 100644 --- a/src/inc_internal/config_monitor.h +++ b/src/inc_internal/config_monitor.h @@ -9,7 +9,7 @@ void config_monitor_traverse(long long version,const char*idx_dir, int (*update)(const char*, const char*, void*),//table name ,line ,u_para void (*finish)(void*),//u_para void* u_para, - const unsigned char* dec_key, + const char* dec_key, void* logger); #endif diff --git a/src/inc_internal/json2iris.h b/src/inc_internal/json2iris.h index f61f6e5..ab8e5c6 100644 --- a/src/inc_internal/json2iris.h +++ b/src/inc_internal/json2iris.h @@ -1,6 +1,6 @@ #ifndef H_MAAT_JSON2IRIS_H_INCLUDE #define H_MAAT_JSON2IRIS_H_INCLUDE -int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger); -int set_file_rulenum(const char* path,int rulenum,void* logger); +int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, void* logger); +int set_file_rulenum(const char* path, int rulenum, void* logger); #endif diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 3cffc1b..1a59791 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -54,7 +54,7 @@ void wait_for_cmd_effective(Maat_feather_t feather, long long version_before) // printf("wait for %lld ms\n", wating_us/1000); } -void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old) +void scan_with_old_or_new_cfg(Maat_feather_t feather, int is_old) { const char* hit_old_data="Hello world! I'm eve."; const char* hit_new_data="Maat was borned in MESA."; @@ -72,7 +72,7 @@ void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old) ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, hit_old_data, strlen(hit_old_data), &result,NULL, 1, &mid, 0); - if(hit_old) + if(is_old) { EXPECT_EQ(ret, 1); EXPECT_TRUE(result.config_id==1); @@ -87,7 +87,7 @@ void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old) ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, hit_new_data, strlen(hit_new_data), &result,NULL, 1, &mid, 0); - if(!hit_old) + if(!is_old) { EXPECT_EQ(ret, 1); EXPECT_TRUE(result.config_id==2); @@ -141,7 +141,7 @@ TEST_F(JSONUpdate, NewCfg) sleep(2); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1); system_cmd_cp(new_json, watched_json); - sleep(2); + sleep(5); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 0); } diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp index 58ba87e..584ab3e 100644 --- a/tools/maat_redis_tool.cpp +++ b/tools/maat_redis_tool.cpp @@ -1,4 +1,5 @@ #include "Maat_rule.h" +#include "Maat_utils.h" #include "Maat_command.h" #include "Maat_rule_internal.h" #include "cJSON.h" @@ -325,8 +326,13 @@ int main(int argc, char * argv[]) read_rule_from_redis(ctx,desired_version,dump_dir, NULL); } else if(model==WORK_MODE_JSON) - { - ret=json2iris(json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL); + { + ret=load_file_to_memory(json_file, &json_buff); + if(ret<0) + { + printf("open %s failed.\n", json_file); + } + ret=json2iris(json_buff, json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL); if(ret<0) { printf("Invalid json format.\n"); From 1df85b7825dd88c21345ff348dbfb22a65652dda Mon Sep 17 00:00:00 2001 From: zhengchao Date: Wed, 22 Jan 2020 18:25:01 +0800 Subject: [PATCH 25/29] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=B8=B2=E6=9F=93iris?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E7=9A=84=E4=BB=A3=E7=A0=81=EF=BC=8C=E9=80=90?= =?UTF-8?q?=E6=9D=A1=E5=86=99=E5=85=A5=E5=86=85=E5=AD=98=E5=90=8E=EF=BC=8C?= =?UTF-8?q?=E5=86=8D=E4=B8=80=E6=AC=A1=E6=80=A7=E5=86=99=E5=85=A5=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/Maat_rule.h | 3 +- src/entry/Maat_utils.cpp | 11 ++ src/entry/json2iris.cpp | 247 ++++++++++++++-------------------- src/inc_internal/Maat_table.h | 2 +- src/inc_internal/Maat_utils.h | 2 + test/test_maatframe.cpp | 4 +- 6 files changed, 119 insertions(+), 150 deletions(-) diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index e9444d6..3a93a6e 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -31,7 +31,8 @@ enum MAAT_CHARSET CHARSET_UNICODE_NCR_HEX, //SGML Numeric character reference,hexdecimal base, e.g. "ا" CHARSET_URL_ENCODE_GB2312, //URL encode with GB2312, e.g. the chinese word "china" was encoded to %D6%D0%B9%FA CHARSET_URL_ENCODE_UTF8, //11, URL encode with UTF8,e.g. the chinese word "china" was encoded to %E4%B8%AD%E5%9B%BD - CHARSET_WINDOWS1251 + CHARSET_WINDOWS1251, + __CHARSET_MAX }; enum MAAT_ACTION { diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index f4c16dd..58da1a8 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -166,6 +166,17 @@ char* str_unescape(char* s) s[j]='\0'; return s; } +size_t memcat(void**dest, size_t offset, size_t *n_dest, const void* src, size_t n_src) +{ + if(*n_destiris_table_map, (const unsigned char*)table_name,strlen(table_name)); if(table_info==NULL) { - table_info=(struct iris_table_t*)calloc(sizeof(struct iris_table_t),1); + table_info=ALLOC(struct iris_table_t, 1); table_info->line_count=0; - memcpy(table_info->table_name,table_name,MIN(sizeof(table_info->table_name)-1, strlen(table_name))); - snprintf(table_info->table_path,sizeof(table_info->table_path),"%s/%s.local",p_iris->tmp_iris_dir,table_info->table_name); - MESA_htable_add(p_iris->iris_table_map,(const unsigned char*)table_info->table_name,strlen(table_info->table_name),table_info); + table_info->table_type=table_type; + memcpy(table_info->table_name, table_name, MIN(sizeof(table_info->table_name)-1, strlen(table_name))); + snprintf(table_info->table_path,sizeof(table_info->table_path), "%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name); + MESA_htable_add(p_iris->iris_table_map, (const unsigned char*)table_info->table_name, strlen(table_info->table_name), table_info); } return table_info; } - +void free_iris_table_info(void* p) +{ + struct iris_table_t* table=(struct iris_table_t*)p; + free(table->buff); + table->buff=NULL; + free(table); +} static int get_group_seq(struct iris_description_t* iris_cfg) { redisReply* data_reply=NULL; @@ -131,7 +142,8 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn, iris_cfg->group_name_map=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(iris_cfg->group_name_map, 0); - + + hargs.data_free = free_iris_table_info; iris_cfg->iris_table_map=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(iris_cfg->iris_table_map, 0); @@ -172,8 +184,8 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn, map_register(iris_cfg->str2int_map, "hexbin",1); map_register(iris_cfg->str2int_map, "case plain",2); - iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn); - iris_cfg->group_table=query_table_info(iris_cfg, group_tn); + iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn, TABLE_TYPE_COMPILE); + iris_cfg->group_table=query_table_info(iris_cfg, group_tn, TABLE_TYPE_GROUP); return 0; } @@ -231,14 +243,13 @@ int set_file_rulenum(const char* path,int rulenum,void* logger) fclose(fp); return 0; } -int direct_write_rule(cJSON* json,MESA_htable_handle str2int,struct traslate_command_t*cmd,int cmd_cnt,const char* path,void* logger) +int direct_write_rule(cJSON* json, MESA_htable_handle str2int, struct traslate_command_t*cmd, int cmd_cnt, struct iris_table_t* table, void* logger) { int i=0,ret=-1; cJSON* item=NULL; cJSON dummy; char *p=NULL; int int_value=0; - FILE* fp=NULL; for(i=0;ivaluestring)+1,1); - memcpy(cmd[i].json_value,item->valuestring,strlen(item->valuestring)); + cmd[i].json_value=ALLOC(char, strlen(item->valuestring)+1); + memcpy(cmd[i].json_value, item->valuestring, strlen(item->valuestring)); break; default://impossible ,already checked assert(0); @@ -289,30 +302,16 @@ int direct_write_rule(cJSON* json,MESA_htable_handle str2int,struct traslate_com } } } - fp=fopen(path,"a"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "fopen %s error %s.",path,strerror(errno)); - goto error_out; - } for(i=0;iwrite_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, cmd[i].json_value, strlen(cmd[i].json_value)); + table->write_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, "\t", 1); } - fprintf(fp,"\n"); - fclose(fp); - - for(i=0;iwrite_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, "\n", 1); + table->line_count++; + ret=0; -error_out: +error_out: for(i=0;istr2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) +int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -516,11 +516,11 @@ int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, co json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,enum MAAT_TABLE_TYPE table_type,void * logger) +int write_expr_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -534,7 +534,7 @@ int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const c json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - if(table_type==TABLE_TYPE_EXPR_PLUS) + if(table->table_type==TABLE_TYPE_EXPR_PLUS) { json_cmd[cmd_cnt].json_string="district"; json_cmd[cmd_cnt].json_type=cJSON_String; @@ -564,10 +564,10 @@ int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const c json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_intval_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +int write_intval_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -593,10 +593,10 @@ int write_intval_line(cJSON *region_json,struct iris_description_t *p_iris,const json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); } -int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) +int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -626,10 +626,10 @@ int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, con json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) +int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -655,28 +655,27 @@ int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, co json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_plugin_line(cJSON* plug_table_json,int sequence,iris_description_t* p_iris,void* logger) +int write_plugin_line(cJSON* plug_table_json, int sequence, iris_description_t* p_iris, void* logger) { - cJSON* item=NULL,*table_content=NULL,*each_line=NULL; + cJSON* item=NULL,*table_content=NULL, *each_line=NULL; struct iris_table_t* table_info=NULL; - const char* table_name=NULL,*line_content=NULL; - int ret=0,i=0,line_cnt=0; - FILE*fp=NULL; + const char* table_name=NULL, *line_content=NULL; + int i=0, line_cnt=0; - item=cJSON_GetObjectItem(plug_table_json,"table_name"); + item=cJSON_GetObjectItem(plug_table_json, "table_name"); if(item==NULL||item->type!=cJSON_String) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "The %d plugin_table's table_name not defined or format error.",sequence); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, + "The %d plugin_table's table_name not defined or format error.", sequence); return -1; } - table_name= item->valuestring; - table_info=query_table_info(p_iris, table_name); - table_content=cJSON_GetObjectItem(plug_table_json,"table_content"); + table_name=item->valuestring; + table_info=query_table_info(p_iris, table_name, TABLE_TYPE_PLUGIN); + table_content=cJSON_GetObjectItem(plug_table_json, "table_content"); if(table_content==NULL||table_content->type!=cJSON_Array) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, @@ -685,21 +684,6 @@ int write_plugin_line(cJSON* plug_table_json,int sequence,iris_description_t* p_ return -1; } line_cnt=cJSON_GetArraySize(table_content); - if(table_info->line_count==0) - { - ret=set_file_rulenum(table_info->table_path,0,logger); - if(ret<0) - { - return -1; - } - } - fp=fopen(table_info->table_path,"a"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "fopen %s error %s.",table_info->table_path,strerror(errno)); - return -1; - } for(i=0;ivaluestring; - fprintf(fp,"%s\n",line_content); + table_info->write_pos+=memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), line_content, strlen(line_content)); + table_info->write_pos+=memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), "\n", 1); table_info->line_count++; } - fclose(fp); - set_file_rulenum(table_info->table_path,table_info->line_count,logger); return 0; } -int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_description_t* p_iris,void* logger) +int write_region_rule(cJSON* region_json, int compile_id, int group_id, iris_description_t* p_iris, void* logger) { cJSON* item=NULL,*table_content=NULL; int ret=0; @@ -727,15 +710,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_EXPR; struct iris_table_t* table_info=NULL; - item=cJSON_GetObjectItem(region_json,"table_name"); - if(item==NULL||item->type!=cJSON_String) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d's region table_name not defined or format error.",compile_id); - return -1; - } - table_name=item->valuestring; - table_info=query_table_info( p_iris, table_name); + item=cJSON_GetObjectItem(region_json,"table_type"); if(item==NULL||item->type!=cJSON_String) { @@ -745,7 +720,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri return -1; } table_type_str=item->valuestring; - ret=map_str2int(p_iris->str2int_map,table_type_str,(int*)&(table_type)); + ret=map_str2int(p_iris->str2int_map, table_type_str, (int*)&(table_type)); if(ret!=1) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, @@ -753,6 +728,16 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri ,compile_id,table_name,table_type_str); return -1; } + item=cJSON_GetObjectItem(region_json,"table_name"); + if(item==NULL||item->type!=cJSON_String) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d's region table_name not defined or format error.",compile_id); + return -1; + } + table_name=item->valuestring; + table_info=query_table_info(p_iris, table_name, table_type); + table_content=cJSON_GetObjectItem(region_json,"table_content"); if(table_content==NULL||table_content->type!=cJSON_Object) { @@ -761,14 +746,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri ,compile_id,table_name); return -1; } - if(table_info->line_count==0) - { - ret=set_file_rulenum(table_info->table_path,0,logger); - if(ret<0) - { - return -1; - } - } + region_id=get_region_seq(p_iris); cJSON_AddNumberToObject(table_content, "region_id", region_id); cJSON_AddNumberToObject(table_content, "group_id", group_id); @@ -778,32 +756,27 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - ret=write_expr_line(table_content, p_iris, table_info->table_path,table_type, logger); + ret=write_expr_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_IP: - ret=write_ip_line(table_content, p_iris, table_info->table_path, logger); + ret=write_ip_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_IP_PLUS: - write_ip_plus_line(table_content, p_iris, table_info->table_path, logger); + write_ip_plus_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_INTERVAL: - ret=write_intval_line(table_content, p_iris, table_info->table_path, logger); + ret=write_intval_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_DIGEST: - ret=write_digest_line(table_content, p_iris, table_info->table_path, logger); + ret=write_digest_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_SIMILARITY: - write_similar_line(table_content, p_iris,table_info->table_path, logger); + ret=write_similar_line(table_content, p_iris, table_info, logger); break; default: assert(0); break; } - if(ret>=0) - { - table_info->line_count++; - set_file_rulenum(table_info->table_path,table_info->line_count,logger); - } return ret; } @@ -860,18 +833,10 @@ int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void * } else { - table_info=query_table_info(p_iris,item->valuestring); + table_info=query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE); } - if(table_info->line_count==0) - { - ret=set_file_rulenum(table_info->table_path, 0,logger); - if(ret<0) - { - return -1; - } - } - ret=direct_write_rule(compile, p_iris->str2int_map,compile_cmd,cmd_cnt, table_info->table_path,logger); + ret=direct_write_rule(compile, p_iris->str2int_map,compile_cmd,cmd_cnt, table_info, logger); if(ret<0) { return -1; @@ -884,41 +849,29 @@ int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void * return -1; } compile_id=item->valueint; - table_info->line_count++; - set_file_rulenum(table_info->table_path,table_info->line_count,logger); return compile_id; } int write_group_line(int group_id, int parent_id, int group_not_flag, int parent_type, const char* virtual_table, struct iris_description_t *p_iris, void * logger) { - FILE*fp=NULL; - int ret=0; - - if(p_iris->group_table->line_count==0) - { - ret=set_file_rulenum(p_iris->group_table->table_path,0,logger); - if(ret<0) - { - return -1; - } - } - fp=fopen(p_iris->group_table->table_path,"a"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "fopen %s error %s.",p_iris->group_table->table_path,strerror(errno)); - return -1; - } - fprintf(fp,"%d\t%d\t1\t%d\t%d\t%s\n",group_id, parent_id, group_not_flag, parent_type, virtual_table); - fclose(fp); - p_iris->group_table->line_count++; - ret=set_file_rulenum(p_iris->group_table->table_path,p_iris->group_table->line_count,logger); + char buff[1024*4]; + struct iris_table_t* table=p_iris->group_table; + snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%d\t%s\n", group_id, parent_id, group_not_flag, parent_type, virtual_table); + table->write_pos+=memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); + table->line_count++; return 0; } void table_idx_write_cb(const uchar * key, uint size, void * data, void * user) { - struct iris_table_t* p_table=(struct iris_table_t*)data; - FILE* fp=(FILE*)user; - fprintf(fp,"%s\t%d\t%s\n",p_table->table_name,p_table->line_count,p_table->table_path); + FILE* fp=NULL; + struct iris_table_t* table=(struct iris_table_t*)data; + fp=fopen(table->table_path, "w"); + fprintf(fp,"%d\n", table->line_count); + fwrite(table->buff, table->write_pos, 1, fp); + fclose(fp); + + fp=(FILE*)user; + fprintf(fp,"%s\t%d\t%s\n", table->table_name, table->line_count, table->table_path); + } int write_index_file(struct iris_description_t *p_iris,void* logger) { @@ -990,10 +943,10 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac { group_info=ALLOC(struct group_info_t, 1); group_info->group_id=get_group_seq(p_iris); - MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info); + MESA_htable_add(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name), group_info); } } - ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, virtual_table, p_iris, logger); + ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, virtual_table, p_iris, logger); if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, @@ -1092,7 +1045,7 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) } } } - ret=write_index_file(p_iris,logger); + ret=write_index_file(p_iris, logger); if(ret<0) { return -1; diff --git a/src/inc_internal/Maat_table.h b/src/inc_internal/Maat_table.h index 27112d2..d056b2d 100644 --- a/src/inc_internal/Maat_table.h +++ b/src/inc_internal/Maat_table.h @@ -6,7 +6,7 @@ #define MAX_COMPILE_EX_DATA_NUM 2 #define MAX_FOREIGN_CLMN_NUM 8 #define MAX_PLUGIN_PER_TABLE 32 -#define MAX_CHARSET_NUM 16 +#define MAX_CHARSET_NUM __CHARSET_MAX #define MAX_CONJUNCTION_TABLE_NUM 8 #define MAX_TABLE_NAME_LEN 256 diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index 6f7352f..faf5a67 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -64,6 +64,8 @@ char* str_tolower(char* string); char *strtok_r_esc(char *s, const char delim, char **save_ptr); char *str_unescape_and(char*s); char* str_unescape(char* s); +size_t memcat(void**dest, size_t offset, size_t *n_dest, const void* src, size_t n_src); + pid_t gettid(void); int system_cmd_mkdir(const char* path); int system_cmd_rm(const char* src_file); diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 1a59791..afe4316 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -112,10 +112,12 @@ class JSONUpdate : public testing::Test protected: static void SetUpTestCase() { + const char* decrypt_key="himaat!"; system_cmd_cp(old_json, watched_json); _shared_feather_j=Maat_feather(g_iThreadNum, table_info_path, g_logger); + // Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_DECRYPT_KEY, decrypt_key, strlen(decrypt_key)+1); Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_JSON_FILE_PATH, watched_json, strlen(watched_json)+1); - Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms)); + Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms)); Maat_initiate_feather(_shared_feather_j); From cc40446df726d15086dce22f3f5caabf3e72a76f Mon Sep 17 00:00:00 2001 From: zhengchao Date: Wed, 22 Jan 2020 20:49:45 +0800 Subject: [PATCH 26/29] =?UTF-8?q?maat=20json=E6=96=87=E4=BB=B6=E6=94=AF?= =?UTF-8?q?=E6=8C=81aes-256-cbc=E5=8A=A0=E5=AF=86=EF=BC=8C=E5=AF=86?= =?UTF-8?q?=E7=A0=81=E9=80=9A=E8=BF=87MAAT=5FOPT=5FDECRYPT=5FKEY=E9=80=89?= =?UTF-8?q?=E9=A1=B9=E6=8C=87=E5=AE=9A=EF=BC=8C=E5=8F=AA=E5=9C=A8=E5=86=85?= =?UTF-8?q?=E5=AD=98=E4=B8=AD=E8=A7=A3=E5=AF=86=EF=BC=8Ciris=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E7=9A=84=E4=B8=AD=E9=97=B4=E7=8A=B6=E6=80=81=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E4=B9=9F=E8=A2=AB=E5=8A=A0=E5=AF=86=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 8 ++- src/entry/Maat_utils.cpp | 109 +++++++++++++++++----------------- src/entry/config_monitor.cpp | 18 +++--- src/entry/json2iris.cpp | 79 ++++++++++++++++-------- src/inc_internal/Maat_utils.h | 12 ++-- src/inc_internal/json2iris.h | 2 +- test/test_maatframe.cpp | 14 +++-- tools/maat_redis_tool.cpp | 6 +- 8 files changed, 144 insertions(+), 104 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 75b6082..7f0a637 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -513,18 +513,18 @@ int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char int ret=0; struct stat fstat_buf; char* json_buff=NULL; - + size_t buff_sz=0; MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module , "Maat initial with JSON file %s, formating..", maat_json_fn); if(strlen(feather->decrypt_key)&&strlen(feather->decrypt_algo)) { - ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&json_buff, err_str, err_str_sz); + ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&json_buff, &buff_sz, err_str, err_str_sz); } if(json_buff==NULL)//decryption failed or no decryption. { - ret=load_file_to_memory(maat_json_fn, &json_buff); + ret=load_file_to_memory(maat_json_fn, (unsigned char**)&json_buff, &buff_sz); } ret=json2iris(json_buff, maat_json_fn, @@ -532,6 +532,8 @@ int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char NULL, feather->json_ctx.iris_file, sizeof(feather->json_ctx.iris_file), + strlen(feather->decrypt_key)?feather->decrypt_key:NULL, + strlen(feather->decrypt_algo)?feather->decrypt_algo:NULL, feather->logger); free(json_buff); json_buff=NULL; diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 58da1a8..8462ba7 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -173,7 +173,7 @@ size_t memcat(void**dest, size_t offset, size_t *n_dest, const void* src, size_t *n_dest=(offset+n_src)*2; *dest=realloc(*dest, sizeof(char)*(*n_dest)); } - memcpy(*dest+offset, src, n_src); + memcpy((char*)*dest+offset, src, n_src); return n_src; } @@ -212,16 +212,21 @@ int system_cmd_mkdir(const char* path) int system_cmd_mv(const char* src_file,const char*dst_file) { char cmd[MAX_SYSTEM_CMD_LEN] = { 0 }; - snprintf(cmd,sizeof(cmd), "mv %s %s", src_file, dst_file); + snprintf(cmd, sizeof(cmd), "mv %s %s", src_file, dst_file); return system(cmd); } int system_cmd_cp(const char* src_file,const char*dst_file) { char cmd[MAX_SYSTEM_CMD_LEN] = { 0 }; - snprintf(cmd,sizeof(cmd), "cp -f %s %s", src_file, dst_file); + snprintf(cmd, sizeof(cmd), "cp -f %s %s", src_file, dst_file); + return system(cmd); +} +int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password) +{ + char cmd[MAX_SYSTEM_CMD_LEN] = { 0 }; + snprintf(cmd,sizeof(cmd), "openssl enc -e -aes-256-cbc -k %s -p -nosalt -in %s -out %s", password, src_file, dst_file); return system(cmd); } - int system_cmd_rm(const char* src_file) { char cmd[MAX_SYSTEM_CMD_LEN] = { 0 }; @@ -270,13 +275,11 @@ int lqueue_destroy_cb(void *data, long data_len, void *arg) assert(0); return 0; } -#define DECRYPT_BLOCK_SIZE (16*1024) -int decrypt_open(const char* filename, const char* key, const char* algorithm, unsigned char**pp_out, char* err_str, size_t err_str_sz) -{ - unsigned char inbuf[DECRYPT_BLOCK_SIZE]; - int inlen, out_blk_len=0; - int out_buff_len=0,buff_offset=0; +int crypt_memory(const unsigned char* inbuf, size_t inlen, unsigned char** pp_out, size_t *out_sz, const char* key, const char* algorithm, int do_encrypt, char* err_str, size_t err_str_sz) +{ + int ret=0, out_blk_len=0; + int out_buff_len=0, out_buff_offset=0; EVP_CIPHER_CTX *ctx; unsigned char cipher_key[EVP_MAX_KEY_LENGTH]; @@ -287,19 +290,12 @@ int decrypt_open(const char* filename, const char* key, const char* algorithm, u const EVP_CIPHER *cipher; const EVP_MD *dgst=NULL; const unsigned char *salt=NULL; - int ret=0; - - FILE*in=fopen(filename, "r"); - if(in==NULL) - { - return -1; - } OpenSSL_add_all_algorithms(); cipher=EVP_get_cipherbyname(algorithm); if(cipher==NULL) { - snprintf(err_str, err_str_sz, "Cipher %s is not supported.",algorithm); + snprintf(err_str, err_str_sz, "Cipher %s is not supported.", algorithm); return 0; } dgst=EVP_get_digestbyname("md5"); @@ -316,52 +312,39 @@ int decrypt_open(const char* filename, const char* key, const char* algorithm, u } /* Don't set key or IV right away; we want to check lengths */ ctx = EVP_CIPHER_CTX_new(); - EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL,0); + EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, do_encrypt); OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) % 16==0); OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16); /* Now we can set key and IV */ - EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, 0); - out_buff_len=DECRYPT_BLOCK_SIZE; + //It should be set to 1 for encryption, 0 for decryption and -1 to leave the value unchanged (the actual value of 'enc' being supplied in a previous call). + EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, -1); + out_buff_len=inlen+EVP_CIPHER_block_size(cipher)-1; *pp_out=(unsigned char*)malloc(out_buff_len*sizeof(unsigned char)); - for (;;) + if (!EVP_CipherUpdate(ctx, *pp_out+out_buff_offset, &out_blk_len, inbuf, inlen)) { - inlen = fread(inbuf, 1, sizeof(inbuf), in); - if (inlen <= 0) - break; - - if(out_buff_len-buff_offsetencryp_algorithm)>0) { @@ -294,20 +294,20 @@ int cm_read_table_file(struct cm_table_info_t* index, MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, no key to decrypt %s.",index->cfg_path); return -1; } - file_sz=decrypt_open(index->cfg_path, key, index->encryp_algorithm, (unsigned char**)&table_file_buff, error_string, sizeof(error_string)); - if(file_sz==0) + ret=decrypt_open(index->cfg_path, key, index->encryp_algorithm, (unsigned char**)&table_file_buff, &file_sz, error_string, sizeof(error_string)); + if(ret<0) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, %s decrypt failed: %s", + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, decrypt %s failed: %s", index->cfg_path, error_string); return -1; } } else { - file_sz=load_file_to_memory(index->cfg_path, &table_file_buff); - if(file_sz==0) + ret=load_file_to_memory(index->cfg_path, (unsigned char**)&table_file_buff, &file_sz); + if(ret<0) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, %s decrypt failed: %s", + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, open %s failed: %s", index->cfg_path, error_string); return -1; } diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 60ff5e4..31943f3 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -51,6 +51,9 @@ struct iris_description_t MESA_htable_handle iris_table_map; MESA_htable_handle str2int_map; redisContext *redis_write_ctx; + char* encrypt_key; + char* encrypt_algo; + FILE* idx_fp; }; struct traslate_command_t { @@ -119,7 +122,7 @@ static int get_region_seq(struct iris_description_t* iris_cfg) return sequence; } -int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,const char* group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger) +int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_key, const char* encrypt_algo, const char*compile_tn,const char* group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger) { memset(iris_cfg,0,sizeof(struct iris_description_t)); snprintf(iris_cfg->tmp_iris_dir,sizeof(iris_cfg->tmp_iris_dir),"%s_iris_tmp",json_file); @@ -186,6 +189,12 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn, iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn, TABLE_TYPE_COMPILE); iris_cfg->group_table=query_table_info(iris_cfg, group_tn, TABLE_TYPE_GROUP); + + if(encrypt_key && encrypt_algo) + { + iris_cfg->encrypt_key=_maat_strdup(encrypt_key); + iris_cfg->encrypt_algo=_maat_strdup(encrypt_algo); + } return 0; } @@ -200,6 +209,8 @@ void clear_iris_descriptor(struct iris_description_t *iris_cfg) MESA_htable_destroy(iris_cfg->iris_table_map, NULL); } map_destroy(iris_cfg->str2int_map); + free(iris_cfg->encrypt_algo); + free(iris_cfg->encrypt_key); return; } int create_tmp_dir(struct iris_description_t *p) @@ -862,29 +873,50 @@ int write_group_line(int group_id, int parent_id, int group_not_flag, int parent } void table_idx_write_cb(const uchar * key, uint size, void * data, void * user) { - FILE* fp=NULL; + struct iris_description_t *p_iris=(struct iris_description_t *)user; struct iris_table_t* table=(struct iris_table_t*)data; - fp=fopen(table->table_path, "w"); - fprintf(fp,"%d\n", table->line_count); - fwrite(table->buff, table->write_pos, 1, fp); - fclose(fp); - - fp=(FILE*)user; - fprintf(fp,"%s\t%d\t%s\n", table->table_name, table->line_count, table->table_path); + FILE* table_fp=NULL; + char line_cnt_str[32], err_str[256]; + snprintf(line_cnt_str, sizeof(line_cnt_str), "%010d\n", table->line_count); + int ret=0; + size_t table_file_sz=strlen(line_cnt_str)+table->write_pos; + unsigned char* buff=ALLOC(unsigned char, table_file_sz); + unsigned char* encrypt_buff=NULL; + size_t encrypt_buff_sz=0; + memcpy(buff, line_cnt_str, strlen(line_cnt_str)); + memcpy(buff+strlen(line_cnt_str), table->buff, table->write_pos); + table_fp=fopen(table->table_path, "w"); + if(p_iris->encrypt_key) + { + ret=crypt_memory(buff, table_file_sz, &encrypt_buff, &encrypt_buff_sz, p_iris->encrypt_key, p_iris->encrypt_algo, 1, err_str, sizeof(err_str)); + assert(ret==0); + fwrite(encrypt_buff, encrypt_buff_sz, 1, table_fp); + fprintf(p_iris->idx_fp,"%s\t%d\t%s\t%s\n", table->table_name, table->line_count, table->table_path, p_iris->encrypt_algo); + } + else + { + fwrite(buff, table_file_sz, 1, table_fp); + fprintf(p_iris->idx_fp,"%s\t%d\t%s\n", table->table_name, table->line_count, table->table_path); + } + fclose(table_fp); + free(buff); + buff=NULL; + + } int write_index_file(struct iris_description_t *p_iris,void* logger) { - FILE*fp=NULL; - fp=fopen(p_iris->index_path,"w"); - if(fp==NULL) + p_iris->idx_fp=fopen(p_iris->index_path,"w"); + if(p_iris->idx_fp==NULL) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "index file %s fopen error %s.",p_iris->index_path,strerror(errno)); + "index file %s fopen error %s.",p_iris->index_path, strerror(errno)); return -1; } - MESA_htable_iterate(p_iris->iris_table_map, table_idx_write_cb, fp); - fclose(fp); + MESA_htable_iterate(p_iris->iris_table_map, table_idx_write_cb, p_iris); + fclose(p_iris->idx_fp); + p_iris->idx_fp=NULL; return 0; } int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger) @@ -1053,31 +1085,30 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) return 0; } // redis_write_ctx is used by maat_redis_tool to write json to redis. -int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, void* logger) +int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger) { cJSON *json=NULL, *tmp_obj=NULL; int ret=-1; struct iris_description_t iris_cfg; - memset(&iris_cfg,0,sizeof(iris_cfg)); - + memset(&iris_cfg, 0, sizeof(iris_cfg)); json=cJSON_Parse(json_buff); if (!json) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,"Error before: %-200.200s",cJSON_GetErrorPtr()); goto error_out; } - tmp_obj=cJSON_GetObjectItem(json,"compile_table"); + tmp_obj=cJSON_GetObjectItem(json, "compile_table"); if(tmp_obj) { compile_tn=tmp_obj->valuestring; } - tmp_obj=cJSON_GetObjectItem(json,"group_table"); + tmp_obj=cJSON_GetObjectItem(json, "group_table"); if(tmp_obj) { group_tn=tmp_obj->valuestring; } - ret=set_iris_descriptor(json_filename, json, compile_tn, group_tn, redis_write_ctx, &iris_cfg, logger); + ret=set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, compile_tn, group_tn, redis_write_ctx, &iris_cfg, logger); if(ret<0) { goto error_out; @@ -1086,15 +1117,15 @@ int json2iris(const char* json_buff, const char* json_filename, const char*compi if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, - "create tmp folder %s error",iris_cfg.tmp_iris_dir); + "create tmp folder %s error", iris_cfg.tmp_iris_dir); goto error_out; } - ret=write_iris(json ,&iris_cfg, logger); + ret=write_iris(json, &iris_cfg, logger); if(ret<0) { goto error_out; } - memcpy(iris_dir_buf,iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir)+1, (unsigned int)buf_len)); + memcpy(iris_dir_buf, iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir)+1, (unsigned int)buf_len)); cJSON_Delete(json); clear_iris_descriptor(&iris_cfg); diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index faf5a67..ac4054b 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -69,14 +69,18 @@ size_t memcat(void**dest, size_t offset, size_t *n_dest, const void* src, size_t pid_t gettid(void); int system_cmd_mkdir(const char* path); int system_cmd_rm(const char* src_file); -int system_cmd_mv(const char* src_file,const char*dst_file); -int system_cmd_cp(const char* src_file,const char*dst_file); +int system_cmd_mv(const char* src_file, const char*dst_file); +int system_cmd_cp(const char* src_file, const char*dst_file); +int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password); + char* md5_file(const char* filename, char* md5string); int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len); const char** charset_get_all_name(void); const char* charset_get_name(enum MAAT_CHARSET charset); int lqueue_destroy_cb(void *data, long data_len, void *arg); -int decrypt_open(const char* filename, const char* key, const char* algorithm, unsigned char**pp_out, char* err_str, size_t err_str_sz); -int load_file_to_memory(const char* file_name, char**pp_out); +int decrypt_open(const char* file_name, const char* key, const char* algorithm, unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz); +int load_file_to_memory(const char* file_name, unsigned char**pp_out, size_t *out_sz); +//do_encrypt: 1 for encryption, 0 for decryption. +int crypt_memory(const unsigned char* inbuf, size_t inlen, unsigned char** pp_out, size_t *out_sz, const char* key, const char* algorithm, int do_encrypt, char* err_str, size_t err_str_sz); diff --git a/src/inc_internal/json2iris.h b/src/inc_internal/json2iris.h index ab8e5c6..247a191 100644 --- a/src/inc_internal/json2iris.h +++ b/src/inc_internal/json2iris.h @@ -1,6 +1,6 @@ #ifndef H_MAAT_JSON2IRIS_H_INCLUDE #define H_MAAT_JSON2IRIS_H_INCLUDE -int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, void* logger); +int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger); int set_file_rulenum(const char* path, int rulenum, void* logger); #endif diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index afe4316..bc4b7d0 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -32,6 +32,7 @@ extern int my_scandir(const char *dir, struct dirent ***namelist, int(*compar)(const void *, const void *)); extern char* md5_file(const char* filename, char* md5string); extern int system_cmd_cp(const char* src_file,const char*dst_file); +extern int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password); Maat_feather_t g_feather=NULL; void *g_logger=NULL; int g_iThreadNum=4; @@ -105,6 +106,7 @@ const char* watched_json="./json_update/maat.json"; const char* old_json="./json_update/old.json"; const char* new_json="./json_update/new.json"; const char* corrupted_json="./json_update/corrupted.json"; +const char* json_decrypt_key="himaat!"; class JSONUpdate : public testing::Test { @@ -112,10 +114,11 @@ class JSONUpdate : public testing::Test protected: static void SetUpTestCase() { - const char* decrypt_key="himaat!"; - system_cmd_cp(old_json, watched_json); + + system_cmd_encrypt(old_json, watched_json, json_decrypt_key); + _shared_feather_j=Maat_feather(g_iThreadNum, table_info_path, g_logger); - // Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_DECRYPT_KEY, decrypt_key, strlen(decrypt_key)+1); + Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_DECRYPT_KEY, json_decrypt_key, strlen(json_decrypt_key)+1); Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_JSON_FILE_PATH, watched_json, strlen(watched_json)+1); Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms)); @@ -139,10 +142,11 @@ TEST_F(JSONUpdate, OldCfg) } TEST_F(JSONUpdate, NewCfg) { - system_cmd_cp(corrupted_json, watched_json); + system_cmd_encrypt(corrupted_json, watched_json, json_decrypt_key); sleep(2); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1); - system_cmd_cp(new_json, watched_json); + + system_cmd_encrypt(new_json, watched_json, json_decrypt_key); sleep(5); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 0); } diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp index 584ab3e..41fe97e 100644 --- a/tools/maat_redis_tool.cpp +++ b/tools/maat_redis_tool.cpp @@ -239,7 +239,7 @@ int main(int argc, char * argv[]) unsigned long json_file_size=0,read_size=0; long long desired_version=0; char* json_buff=NULL; - + size_t json_buff_sz=0; while((oc=getopt(argc,argv,"h:p:n:d:v:f:j:t:"))!=-1) { switch(oc) @@ -327,12 +327,12 @@ int main(int argc, char * argv[]) } else if(model==WORK_MODE_JSON) { - ret=load_file_to_memory(json_file, &json_buff); + ret=load_file_to_memory(json_file, (unsigned char**)&json_buff, &json_buff_sz); if(ret<0) { printf("open %s failed.\n", json_file); } - ret=json2iris(json_buff, json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL); + ret=json2iris(json_buff, json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, NULL); if(ret<0) { printf("Invalid json format.\n"); From 94f11e5a7b7a2357cbc605c6687e4934cbe07a5d Mon Sep 17 00:00:00 2001 From: zhengchao Date: Mon, 13 Jan 2020 19:05:24 +0800 Subject: [PATCH 27/29] =?UTF-8?q?=E6=94=AF=E6=8C=81maat=20json=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=8A=A0=E5=AF=86=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 87 +++++++++----- src/entry/Maat_command.cpp | 2 +- src/entry/Maat_rule.cpp | 24 ++-- src/entry/Maat_utils.cpp | 127 ++++++++++++++++++++- src/entry/config_monitor.cpp | 157 ++++++-------------------- src/entry/json2iris.cpp | 48 +------- src/inc_internal/Maat_rule_internal.h | 10 +- src/inc_internal/Maat_utils.h | 3 + src/inc_internal/config_monitor.h | 2 +- src/inc_internal/json2iris.h | 4 +- test/test_maatframe.cpp | 8 +- tools/maat_redis_tool.cpp | 10 +- 12 files changed, 257 insertions(+), 225 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index ba3999b..75b6082 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -508,6 +508,52 @@ int detain_last_data(char* buff,int buff_size,int detained_len,const char* data, } return ret_len; } +int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char* err_str, size_t err_str_sz) +{ + int ret=0; + struct stat fstat_buf; + char* json_buff=NULL; + + + MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module , + "Maat initial with JSON file %s, formating..", + maat_json_fn); + if(strlen(feather->decrypt_key)&&strlen(feather->decrypt_algo)) + { + ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&json_buff, err_str, err_str_sz); + } + if(json_buff==NULL)//decryption failed or no decryption. + { + ret=load_file_to_memory(maat_json_fn, &json_buff); + } + ret=json2iris(json_buff, + maat_json_fn, + feather->compile_tn, feather->group_tn, + NULL, + feather->json_ctx.iris_file, + sizeof(feather->json_ctx.iris_file), + feather->logger); + free(json_buff); + json_buff=NULL; + if(ret<0) + { + return -1; + } + strncpy(feather->json_ctx.json_file, maat_json_fn, sizeof(feather->json_ctx.json_file)); + + ret=stat(maat_json_fn, &fstat_buf); + feather->json_ctx.last_md5_time=fstat_buf.st_ctim; + + md5_file(feather->json_ctx.json_file, feather->json_ctx.effective_json_md5); + MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO, maat_module, + "JSON file %s md5: %s, generate index file %s OK.", + feather->json_ctx.json_file, + feather->json_ctx.effective_json_md5, + feather->json_ctx.iris_file); + feather->input_mode=SOURCE_JSON_FILE; + + return 0; +} Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger) { if(max_thread_num<=0) @@ -543,6 +589,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* feather->base_rgn_seq=0; feather->AUTO_NUMBERING_ON=1; feather->backgroud_update_enabled=1; + snprintf(feather->decrypt_algo, sizeof(feather->decrypt_algo), "aes-256-cbc"); snprintf(feather->foreign_cont_dir, sizeof(feather->foreign_cont_dir), "%s_files", table_info_path); pthread_mutex_init(&(feather->background_update_mutex),NULL); snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path); @@ -554,8 +601,8 @@ failed: int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size) { _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct stat attrib; int intval=0,ret=-1; + char err_str[MAX_TABLE_NAME_LEN]; switch(type) { case MAAT_OPT_ENABLE_UPDATE: @@ -630,31 +677,15 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo break; case MAAT_OPT_JSON_FILE_PATH: assert(_feather->input_mode==SOURCE_NONE); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Maat initial with JSON file %s, formating..", - (const char*)value); - - ret=json2iris((const char*)value, - _feather->compile_tn,_feather->group_tn, - NULL, - _feather->json_ctx.iris_file, - sizeof(_feather->json_ctx.iris_file), - _feather->logger); + ret=load_maat_json_file(_feather, (const char *)value, err_str, sizeof(err_str)); if(ret<0) { + MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, + "Load maat json file %s failed: %s.", + (const char*)value, err_str); return -1; } - memcpy(_feather->json_ctx.json_file, value, size); - stat(_feather->json_ctx.json_file, &attrib); - _feather->json_ctx.last_md5_time=attrib.st_ctime; - md5_file(_feather->json_ctx.json_file, _feather->json_ctx.effective_json_md5); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "JSON file %s md5: %s, generate index file %s OK.", - _feather->json_ctx.json_file, - _feather->json_ctx.effective_json_md5, - _feather->json_ctx.iris_file); - _feather->input_mode=SOURCE_JSON_FILE; break; case MAAT_OPT_STAT_ON: _feather->stat_on=1; @@ -667,10 +698,10 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo { return -1; } - memcpy(_feather->stat_file,(const char*)value,size); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Maat performance statistic output to %s." - ,(const char*)value); + memcpy(_feather->stat_file, (const char*)value, size); + MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module, + "Maat performance statistic output to %s.", + (const char*)value); _feather->stat_on=1; break; case MAAT_OPT_SCAN_DETAIL: @@ -678,9 +709,9 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo _feather->rule_scan_type=intval; break; case MAAT_OPT_INSTANCE_NAME: - snprintf(_feather->instance_name - ,sizeof(_feather->instance_name) - ,"%s", + snprintf(_feather->instance_name, + sizeof(_feather->instance_name), + "%s", (const char*)value); break; case MAAT_OPT_DECRYPT_KEY: diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 6d061fd..a51b4de 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -1824,7 +1824,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para void (*finish)(void*),//u_para void* u_para, - const unsigned char* dec_key, + const char* dec_key, _Maat_feather_t* feather) { int table_id=0, i=0, rule_num=0, empty_value_num=0, valid_column=-1; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 90373d3..3d93ad8 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -33,7 +33,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_8_20191129=1; +int MAAT_FRAME_VERSION_2_8_20200113=1; int is_valid_table_name(const char* str) { @@ -3500,7 +3500,8 @@ void *thread_rule_monitor(void *arg) int scan_dir_cnt=0; int ret=0; char md5_tmp[MD5_DIGEST_LENGTH*2+1]={0}; - char tmp_dir[MAX_TABLE_NAME_LEN]={0}; + char err_str[MAX_TABLE_NAME_LEN]={0}; + struct stat attrib; size_t total_wait_rule_cnt=0; @@ -3557,31 +3558,24 @@ void *thread_rule_monitor(void *arg) break; case SOURCE_JSON_FILE: memset(md5_tmp, 0, sizeof(md5_tmp)); - memset(tmp_dir, 0, sizeof(tmp_dir)); stat(feather->json_ctx.json_file, &attrib); - if(attrib.st_ctime!=feather->json_ctx.last_md5_time) + if(memcmp(&attrib.st_ctim, &(feather->json_ctx.last_md5_time), sizeof(attrib.st_ctim))) { - feather->json_ctx.last_md5_time=attrib.st_ctime; + feather->json_ctx.last_md5_time=attrib.st_ctim; md5_file(feather->json_ctx.json_file, md5_tmp); if(0!=strcmp(md5_tmp,feather->json_ctx.effective_json_md5)) { - ret=json2iris(feather->json_ctx.json_file, - feather->compile_tn, feather->group_tn, - NULL, - tmp_dir, - sizeof(tmp_dir), - feather->logger); + ret=load_maat_json_file(feather, feather->json_ctx.json_file, err_str, sizeof(err_str)); if(ret<0) { MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module , - "Maat re-initiate with JSON file %s failed, md5: %s", + "Maat re-initiate with JSON file %s (md5=%s)failed: %s", feather->json_ctx.json_file, - md5_tmp); + md5_tmp, + err_str); } else { - strcpy(feather->json_ctx.effective_json_md5, md5_tmp); - strcpy(feather->json_ctx.iris_file, tmp_dir); config_monitor_traverse(0, feather->json_ctx.iris_file, maat_start_cb, diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 3d40912..f4c16dd 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -1,9 +1,11 @@ #include -#include #include #include -#include #include +#include +#include +#include + #include "Maat_utils.h" pid_t gettid() { @@ -257,4 +259,125 @@ int lqueue_destroy_cb(void *data, long data_len, void *arg) assert(0); return 0; } +#define DECRYPT_BLOCK_SIZE (16*1024) +int decrypt_open(const char* filename, const char* key, const char* algorithm, unsigned char**pp_out, char* err_str, size_t err_str_sz) +{ + unsigned char inbuf[DECRYPT_BLOCK_SIZE]; + int inlen, out_blk_len=0; + int out_buff_len=0,buff_offset=0; + EVP_CIPHER_CTX *ctx; + + unsigned char cipher_key[EVP_MAX_KEY_LENGTH]; + unsigned char cipher_iv[EVP_MAX_IV_LENGTH]; + memset(cipher_key,0,sizeof(cipher_key)); + memset(cipher_iv,0,sizeof(cipher_iv)); + + const EVP_CIPHER *cipher; + const EVP_MD *dgst=NULL; + const unsigned char *salt=NULL; + int ret=0; + + FILE*in=fopen(filename, "r"); + if(in==NULL) + { + return -1; + } + + OpenSSL_add_all_algorithms(); + cipher=EVP_get_cipherbyname(algorithm); + if(cipher==NULL) + { + snprintf(err_str, err_str_sz, "Cipher %s is not supported.",algorithm); + return 0; + } + dgst=EVP_get_digestbyname("md5"); + if(dgst==NULL) + { + snprintf(err_str, err_str_sz, "Get MD5 object failed."); + return 0; + } + ret=EVP_BytesToKey(cipher, dgst, salt, (unsigned char*)key, strlen((const char*)key), 1, cipher_key, cipher_iv); + if(ret==0) + { + snprintf(err_str, err_str_sz, "Key and IV generatioin failed."); + return 0; + } + /* Don't set key or IV right away; we want to check lengths */ + ctx = EVP_CIPHER_CTX_new(); + EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL,0); + OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) % 16==0); + OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16); + + /* Now we can set key and IV */ + EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, 0); + out_buff_len=DECRYPT_BLOCK_SIZE; + *pp_out=(unsigned char*)malloc(out_buff_len*sizeof(unsigned char)); + for (;;) + { + inlen = fread(inbuf, 1, sizeof(inbuf), in); + if (inlen <= 0) + break; + + if(out_buff_len-buff_offsetcfg_path,"r"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error,open %s failed.",index->cfg_path); - return -1; - } + char* table_file_buff=NULL; + int file_sz=0, file_offset=0; + if(strlen(index->encryp_algorithm)>0) { if(key==NULL||strlen((const char*)key)==0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, no key to decrypt %s.",index->cfg_path); - fclose(fp); return -1; } - decrypt_len=decrypt_open(fp, key,index->encryp_algorithm, &decrypt_buff,logger); - if(decrypt_len==0) + file_sz=decrypt_open(index->cfg_path, key, index->encryp_algorithm, (unsigned char**)&table_file_buff, error_string, sizeof(error_string)); + if(file_sz==0) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, %s decrypt failed.",index->cfg_path); - fclose(fp); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, %s decrypt failed: %s", + index->cfg_path, error_string); return -1; } - read_nxt_line_from_buff(decrypt_buff, decrypt_len, &decrypt_offset, line, sizeof(line)); - sscanf(line,"%d\n",&cfg_num); - do_decrypt=1; } else { - fscanf(fp,"%d\n",&cfg_num); + file_sz=load_file_to_memory(index->cfg_path, &table_file_buff); + if(file_sz==0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, %s decrypt failed: %s", + index->cfg_path, error_string); + return -1; + } + } + read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line)); + sscanf(line, "%d\n", &cfg_num); + if(cfg_num!=index->cfg_num) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor ,"file %s config num not matched",index->cfg_path); - fclose(fp); + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor, "file %s config num not matched", index->cfg_path); return -1; } for(i=0;ivaluestring; } - ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,redis_write_ctx,&iris_cfg,logger); + ret=set_iris_descriptor(json_filename, json, compile_tn, group_tn, redis_write_ctx, &iris_cfg, logger); if(ret<0) { goto error_out; @@ -1163,31 +1132,24 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,r ret=create_tmp_dir(&iris_cfg); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, "create tmp folder %s error",iris_cfg.tmp_iris_dir); goto error_out; } - ret=write_iris(json,&iris_cfg,logger); + ret=write_iris(json ,&iris_cfg, logger); if(ret<0) { goto error_out; } - memcpy(iris_dir_buf,iris_cfg.tmp_iris_index_dir,MIN(strlen(iris_cfg.tmp_iris_index_dir)+1,(unsigned int)buf_len)); + memcpy(iris_dir_buf,iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir)+1, (unsigned int)buf_len)); cJSON_Delete(json); - fclose(json_fp); - free(json_buff); clear_iris_descriptor(&iris_cfg); return 0; error_out: cJSON_Delete(json); - if(json_fp!=NULL) - { - fclose(json_fp); - } - free(json_buff); clear_iris_descriptor(&iris_cfg); return -1; } diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 8ccdb81..3aed79e 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -21,7 +21,6 @@ #include #include -extern const char *maat_module; #define mr_region_id_var "SEQUENCE_REGION" @@ -287,7 +286,7 @@ struct source_json_ctx char json_file[MAX_TABLE_NAME_LEN]; char iris_file[MAX_TABLE_NAME_LEN]; char effective_json_md5[MD5_DIGEST_LENGTH*2+1]; - time_t last_md5_time; + struct timespec last_md5_time; }; struct source_redis_ctx { @@ -333,7 +332,8 @@ struct _Maat_feather_t char compile_tn[MAX_TABLE_NAME_LEN]; char group_tn[MAX_TABLE_NAME_LEN]; pthread_mutex_t background_update_mutex; - unsigned char decrypt_key[MAX_TABLE_NAME_LEN]; + char decrypt_key[MAX_TABLE_NAME_LEN]; + char decrypt_algo[MAX_TABLE_NAME_LEN]; pthread_t cfg_mon_t; int AUTO_NUMBERING_ON; @@ -450,14 +450,14 @@ void empty_serial_rules(struct serial_rule_t* rule); int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,unsigned int serial_rule_num, long long server_time, void* logger); long long redis_server_time(redisContext* ctx); redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db, void* logger); -char* md5_file(const char* filename, char* md5string); +int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char* err_str, size_t err_str_sz); void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx, void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para void (*finish)(void*),//u_para void* u_para, - const unsigned char* dec_key, + const char* dec_key, _Maat_feather_t* feather); diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index ab233a2..6f7352f 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -2,6 +2,7 @@ #include "Maat_rule.h" #include #include +#include #include #include //fstat #include //fstat @@ -73,5 +74,7 @@ int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len const char** charset_get_all_name(void); const char* charset_get_name(enum MAAT_CHARSET charset); int lqueue_destroy_cb(void *data, long data_len, void *arg); +int decrypt_open(const char* filename, const char* key, const char* algorithm, unsigned char**pp_out, char* err_str, size_t err_str_sz); +int load_file_to_memory(const char* file_name, char**pp_out); diff --git a/src/inc_internal/config_monitor.h b/src/inc_internal/config_monitor.h index faabf50..6167ce4 100644 --- a/src/inc_internal/config_monitor.h +++ b/src/inc_internal/config_monitor.h @@ -9,7 +9,7 @@ void config_monitor_traverse(long long version,const char*idx_dir, int (*update)(const char*, const char*, void*),//table name ,line ,u_para void (*finish)(void*),//u_para void* u_para, - const unsigned char* dec_key, + const char* dec_key, void* logger); #endif diff --git a/src/inc_internal/json2iris.h b/src/inc_internal/json2iris.h index f61f6e5..ab8e5c6 100644 --- a/src/inc_internal/json2iris.h +++ b/src/inc_internal/json2iris.h @@ -1,6 +1,6 @@ #ifndef H_MAAT_JSON2IRIS_H_INCLUDE #define H_MAAT_JSON2IRIS_H_INCLUDE -int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger); -int set_file_rulenum(const char* path,int rulenum,void* logger); +int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, void* logger); +int set_file_rulenum(const char* path, int rulenum, void* logger); #endif diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 469a524..39306b7 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -54,7 +54,7 @@ void wait_for_cmd_effective(Maat_feather_t feather, long long version_before) // printf("wait for %lld ms\n", wating_us/1000); } -void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old) +void scan_with_old_or_new_cfg(Maat_feather_t feather, int is_old) { const char* hit_old_data="Hello world! I'm eve."; const char* hit_new_data="Maat was borned in MESA."; @@ -72,7 +72,7 @@ void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old) ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, hit_old_data, strlen(hit_old_data), &result,NULL, 1, &mid, 0); - if(hit_old) + if(is_old) { EXPECT_EQ(ret, 1); EXPECT_TRUE(result.config_id==1); @@ -87,7 +87,7 @@ void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old) ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, hit_new_data, strlen(hit_new_data), &result,NULL, 1, &mid, 0); - if(!hit_old) + if(!is_old) { EXPECT_EQ(ret, 1); EXPECT_TRUE(result.config_id==2); @@ -141,7 +141,7 @@ TEST_F(JSONUpdate, NewCfg) sleep(2); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1); system_cmd_cp(new_json, watched_json); - sleep(2); + sleep(5); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 0); } diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp index 58ba87e..584ab3e 100644 --- a/tools/maat_redis_tool.cpp +++ b/tools/maat_redis_tool.cpp @@ -1,4 +1,5 @@ #include "Maat_rule.h" +#include "Maat_utils.h" #include "Maat_command.h" #include "Maat_rule_internal.h" #include "cJSON.h" @@ -325,8 +326,13 @@ int main(int argc, char * argv[]) read_rule_from_redis(ctx,desired_version,dump_dir, NULL); } else if(model==WORK_MODE_JSON) - { - ret=json2iris(json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL); + { + ret=load_file_to_memory(json_file, &json_buff); + if(ret<0) + { + printf("open %s failed.\n", json_file); + } + ret=json2iris(json_buff, json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL); if(ret<0) { printf("Invalid json format.\n"); From d914fa1cb224794c3a94edebce855e6ac6a3791a Mon Sep 17 00:00:00 2001 From: zhengchao Date: Wed, 22 Jan 2020 18:25:01 +0800 Subject: [PATCH 28/29] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=B8=B2=E6=9F=93iris?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E7=9A=84=E4=BB=A3=E7=A0=81=EF=BC=8C=E9=80=90?= =?UTF-8?q?=E6=9D=A1=E5=86=99=E5=85=A5=E5=86=85=E5=AD=98=E5=90=8E=EF=BC=8C?= =?UTF-8?q?=E5=86=8D=E4=B8=80=E6=AC=A1=E6=80=A7=E5=86=99=E5=85=A5=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/Maat_rule.h | 3 +- src/entry/Maat_utils.cpp | 11 ++ src/entry/json2iris.cpp | 247 ++++++++++++++-------------------- src/inc_internal/Maat_table.h | 2 +- src/inc_internal/Maat_utils.h | 2 + test/test_maatframe.cpp | 4 +- 6 files changed, 119 insertions(+), 150 deletions(-) diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index e9444d6..3a93a6e 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -31,7 +31,8 @@ enum MAAT_CHARSET CHARSET_UNICODE_NCR_HEX, //SGML Numeric character reference,hexdecimal base, e.g. "ا" CHARSET_URL_ENCODE_GB2312, //URL encode with GB2312, e.g. the chinese word "china" was encoded to %D6%D0%B9%FA CHARSET_URL_ENCODE_UTF8, //11, URL encode with UTF8,e.g. the chinese word "china" was encoded to %E4%B8%AD%E5%9B%BD - CHARSET_WINDOWS1251 + CHARSET_WINDOWS1251, + __CHARSET_MAX }; enum MAAT_ACTION { diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index f4c16dd..58da1a8 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -166,6 +166,17 @@ char* str_unescape(char* s) s[j]='\0'; return s; } +size_t memcat(void**dest, size_t offset, size_t *n_dest, const void* src, size_t n_src) +{ + if(*n_destiris_table_map, (const unsigned char*)table_name,strlen(table_name)); if(table_info==NULL) { - table_info=(struct iris_table_t*)calloc(sizeof(struct iris_table_t),1); + table_info=ALLOC(struct iris_table_t, 1); table_info->line_count=0; - memcpy(table_info->table_name,table_name,MIN(sizeof(table_info->table_name)-1, strlen(table_name))); - snprintf(table_info->table_path,sizeof(table_info->table_path),"%s/%s.local",p_iris->tmp_iris_dir,table_info->table_name); - MESA_htable_add(p_iris->iris_table_map,(const unsigned char*)table_info->table_name,strlen(table_info->table_name),table_info); + table_info->table_type=table_type; + memcpy(table_info->table_name, table_name, MIN(sizeof(table_info->table_name)-1, strlen(table_name))); + snprintf(table_info->table_path,sizeof(table_info->table_path), "%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name); + MESA_htable_add(p_iris->iris_table_map, (const unsigned char*)table_info->table_name, strlen(table_info->table_name), table_info); } return table_info; } - +void free_iris_table_info(void* p) +{ + struct iris_table_t* table=(struct iris_table_t*)p; + free(table->buff); + table->buff=NULL; + free(table); +} static int get_group_seq(struct iris_description_t* iris_cfg) { redisReply* data_reply=NULL; @@ -131,7 +142,8 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn, iris_cfg->group_name_map=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(iris_cfg->group_name_map, 0); - + + hargs.data_free = free_iris_table_info; iris_cfg->iris_table_map=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(iris_cfg->iris_table_map, 0); @@ -172,8 +184,8 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn, map_register(iris_cfg->str2int_map, "hexbin",1); map_register(iris_cfg->str2int_map, "case plain",2); - iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn); - iris_cfg->group_table=query_table_info(iris_cfg, group_tn); + iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn, TABLE_TYPE_COMPILE); + iris_cfg->group_table=query_table_info(iris_cfg, group_tn, TABLE_TYPE_GROUP); return 0; } @@ -231,14 +243,13 @@ int set_file_rulenum(const char* path,int rulenum,void* logger) fclose(fp); return 0; } -int direct_write_rule(cJSON* json,MESA_htable_handle str2int,struct traslate_command_t*cmd,int cmd_cnt,const char* path,void* logger) +int direct_write_rule(cJSON* json, MESA_htable_handle str2int, struct traslate_command_t*cmd, int cmd_cnt, struct iris_table_t* table, void* logger) { int i=0,ret=-1; cJSON* item=NULL; cJSON dummy; char *p=NULL; int int_value=0; - FILE* fp=NULL; for(i=0;ivaluestring)+1,1); - memcpy(cmd[i].json_value,item->valuestring,strlen(item->valuestring)); + cmd[i].json_value=ALLOC(char, strlen(item->valuestring)+1); + memcpy(cmd[i].json_value, item->valuestring, strlen(item->valuestring)); break; default://impossible ,already checked assert(0); @@ -289,30 +302,16 @@ int direct_write_rule(cJSON* json,MESA_htable_handle str2int,struct traslate_com } } } - fp=fopen(path,"a"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "fopen %s error %s.",path,strerror(errno)); - goto error_out; - } for(i=0;iwrite_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, cmd[i].json_value, strlen(cmd[i].json_value)); + table->write_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, "\t", 1); } - fprintf(fp,"\n"); - fclose(fp); - - for(i=0;iwrite_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, "\n", 1); + table->line_count++; + ret=0; -error_out: +error_out: for(i=0;istr2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) +int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -516,11 +516,11 @@ int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, co json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,enum MAAT_TABLE_TYPE table_type,void * logger) +int write_expr_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -534,7 +534,7 @@ int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const c json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - if(table_type==TABLE_TYPE_EXPR_PLUS) + if(table->table_type==TABLE_TYPE_EXPR_PLUS) { json_cmd[cmd_cnt].json_string="district"; json_cmd[cmd_cnt].json_type=cJSON_String; @@ -564,10 +564,10 @@ int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const c json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_intval_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +int write_intval_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -593,10 +593,10 @@ int write_intval_line(cJSON *region_json,struct iris_description_t *p_iris,const json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); } -int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) +int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -626,10 +626,10 @@ int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, con json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) +int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -655,28 +655,27 @@ int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, co json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_plugin_line(cJSON* plug_table_json,int sequence,iris_description_t* p_iris,void* logger) +int write_plugin_line(cJSON* plug_table_json, int sequence, iris_description_t* p_iris, void* logger) { - cJSON* item=NULL,*table_content=NULL,*each_line=NULL; + cJSON* item=NULL,*table_content=NULL, *each_line=NULL; struct iris_table_t* table_info=NULL; - const char* table_name=NULL,*line_content=NULL; - int ret=0,i=0,line_cnt=0; - FILE*fp=NULL; + const char* table_name=NULL, *line_content=NULL; + int i=0, line_cnt=0; - item=cJSON_GetObjectItem(plug_table_json,"table_name"); + item=cJSON_GetObjectItem(plug_table_json, "table_name"); if(item==NULL||item->type!=cJSON_String) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "The %d plugin_table's table_name not defined or format error.",sequence); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, + "The %d plugin_table's table_name not defined or format error.", sequence); return -1; } - table_name= item->valuestring; - table_info=query_table_info(p_iris, table_name); - table_content=cJSON_GetObjectItem(plug_table_json,"table_content"); + table_name=item->valuestring; + table_info=query_table_info(p_iris, table_name, TABLE_TYPE_PLUGIN); + table_content=cJSON_GetObjectItem(plug_table_json, "table_content"); if(table_content==NULL||table_content->type!=cJSON_Array) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, @@ -685,21 +684,6 @@ int write_plugin_line(cJSON* plug_table_json,int sequence,iris_description_t* p_ return -1; } line_cnt=cJSON_GetArraySize(table_content); - if(table_info->line_count==0) - { - ret=set_file_rulenum(table_info->table_path,0,logger); - if(ret<0) - { - return -1; - } - } - fp=fopen(table_info->table_path,"a"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "fopen %s error %s.",table_info->table_path,strerror(errno)); - return -1; - } for(i=0;ivaluestring; - fprintf(fp,"%s\n",line_content); + table_info->write_pos+=memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), line_content, strlen(line_content)); + table_info->write_pos+=memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), "\n", 1); table_info->line_count++; } - fclose(fp); - set_file_rulenum(table_info->table_path,table_info->line_count,logger); return 0; } -int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_description_t* p_iris,void* logger) +int write_region_rule(cJSON* region_json, int compile_id, int group_id, iris_description_t* p_iris, void* logger) { cJSON* item=NULL,*table_content=NULL; int ret=0; @@ -727,15 +710,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_EXPR; struct iris_table_t* table_info=NULL; - item=cJSON_GetObjectItem(region_json,"table_name"); - if(item==NULL||item->type!=cJSON_String) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d's region table_name not defined or format error.",compile_id); - return -1; - } - table_name=item->valuestring; - table_info=query_table_info( p_iris, table_name); + item=cJSON_GetObjectItem(region_json,"table_type"); if(item==NULL||item->type!=cJSON_String) { @@ -745,7 +720,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri return -1; } table_type_str=item->valuestring; - ret=map_str2int(p_iris->str2int_map,table_type_str,(int*)&(table_type)); + ret=map_str2int(p_iris->str2int_map, table_type_str, (int*)&(table_type)); if(ret!=1) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, @@ -753,6 +728,16 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri ,compile_id,table_name,table_type_str); return -1; } + item=cJSON_GetObjectItem(region_json,"table_name"); + if(item==NULL||item->type!=cJSON_String) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d's region table_name not defined or format error.",compile_id); + return -1; + } + table_name=item->valuestring; + table_info=query_table_info(p_iris, table_name, table_type); + table_content=cJSON_GetObjectItem(region_json,"table_content"); if(table_content==NULL||table_content->type!=cJSON_Object) { @@ -761,14 +746,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri ,compile_id,table_name); return -1; } - if(table_info->line_count==0) - { - ret=set_file_rulenum(table_info->table_path,0,logger); - if(ret<0) - { - return -1; - } - } + region_id=get_region_seq(p_iris); cJSON_AddNumberToObject(table_content, "region_id", region_id); cJSON_AddNumberToObject(table_content, "group_id", group_id); @@ -778,32 +756,27 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - ret=write_expr_line(table_content, p_iris, table_info->table_path,table_type, logger); + ret=write_expr_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_IP: - ret=write_ip_line(table_content, p_iris, table_info->table_path, logger); + ret=write_ip_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_IP_PLUS: - write_ip_plus_line(table_content, p_iris, table_info->table_path, logger); + write_ip_plus_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_INTERVAL: - ret=write_intval_line(table_content, p_iris, table_info->table_path, logger); + ret=write_intval_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_DIGEST: - ret=write_digest_line(table_content, p_iris, table_info->table_path, logger); + ret=write_digest_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_SIMILARITY: - write_similar_line(table_content, p_iris,table_info->table_path, logger); + ret=write_similar_line(table_content, p_iris, table_info, logger); break; default: assert(0); break; } - if(ret>=0) - { - table_info->line_count++; - set_file_rulenum(table_info->table_path,table_info->line_count,logger); - } return ret; } @@ -860,18 +833,10 @@ int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void * } else { - table_info=query_table_info(p_iris,item->valuestring); + table_info=query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE); } - if(table_info->line_count==0) - { - ret=set_file_rulenum(table_info->table_path, 0,logger); - if(ret<0) - { - return -1; - } - } - ret=direct_write_rule(compile, p_iris->str2int_map,compile_cmd,cmd_cnt, table_info->table_path,logger); + ret=direct_write_rule(compile, p_iris->str2int_map,compile_cmd,cmd_cnt, table_info, logger); if(ret<0) { return -1; @@ -884,41 +849,29 @@ int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void * return -1; } compile_id=item->valueint; - table_info->line_count++; - set_file_rulenum(table_info->table_path,table_info->line_count,logger); return compile_id; } int write_group_line(int group_id, int parent_id, int group_not_flag, int parent_type, const char* virtual_table, struct iris_description_t *p_iris, void * logger) { - FILE*fp=NULL; - int ret=0; - - if(p_iris->group_table->line_count==0) - { - ret=set_file_rulenum(p_iris->group_table->table_path,0,logger); - if(ret<0) - { - return -1; - } - } - fp=fopen(p_iris->group_table->table_path,"a"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "fopen %s error %s.",p_iris->group_table->table_path,strerror(errno)); - return -1; - } - fprintf(fp,"%d\t%d\t1\t%d\t%d\t%s\n",group_id, parent_id, group_not_flag, parent_type, virtual_table); - fclose(fp); - p_iris->group_table->line_count++; - ret=set_file_rulenum(p_iris->group_table->table_path,p_iris->group_table->line_count,logger); + char buff[1024*4]; + struct iris_table_t* table=p_iris->group_table; + snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%d\t%s\n", group_id, parent_id, group_not_flag, parent_type, virtual_table); + table->write_pos+=memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); + table->line_count++; return 0; } void table_idx_write_cb(const uchar * key, uint size, void * data, void * user) { - struct iris_table_t* p_table=(struct iris_table_t*)data; - FILE* fp=(FILE*)user; - fprintf(fp,"%s\t%d\t%s\n",p_table->table_name,p_table->line_count,p_table->table_path); + FILE* fp=NULL; + struct iris_table_t* table=(struct iris_table_t*)data; + fp=fopen(table->table_path, "w"); + fprintf(fp,"%d\n", table->line_count); + fwrite(table->buff, table->write_pos, 1, fp); + fclose(fp); + + fp=(FILE*)user; + fprintf(fp,"%s\t%d\t%s\n", table->table_name, table->line_count, table->table_path); + } int write_index_file(struct iris_description_t *p_iris,void* logger) { @@ -990,10 +943,10 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac { group_info=ALLOC(struct group_info_t, 1); group_info->group_id=get_group_seq(p_iris); - MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info); + MESA_htable_add(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name), group_info); } } - ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, virtual_table, p_iris, logger); + ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, virtual_table, p_iris, logger); if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, @@ -1092,7 +1045,7 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) } } } - ret=write_index_file(p_iris,logger); + ret=write_index_file(p_iris, logger); if(ret<0) { return -1; diff --git a/src/inc_internal/Maat_table.h b/src/inc_internal/Maat_table.h index 27112d2..d056b2d 100644 --- a/src/inc_internal/Maat_table.h +++ b/src/inc_internal/Maat_table.h @@ -6,7 +6,7 @@ #define MAX_COMPILE_EX_DATA_NUM 2 #define MAX_FOREIGN_CLMN_NUM 8 #define MAX_PLUGIN_PER_TABLE 32 -#define MAX_CHARSET_NUM 16 +#define MAX_CHARSET_NUM __CHARSET_MAX #define MAX_CONJUNCTION_TABLE_NUM 8 #define MAX_TABLE_NAME_LEN 256 diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index 6f7352f..faf5a67 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -64,6 +64,8 @@ char* str_tolower(char* string); char *strtok_r_esc(char *s, const char delim, char **save_ptr); char *str_unescape_and(char*s); char* str_unescape(char* s); +size_t memcat(void**dest, size_t offset, size_t *n_dest, const void* src, size_t n_src); + pid_t gettid(void); int system_cmd_mkdir(const char* path); int system_cmd_rm(const char* src_file); diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 39306b7..665c21b 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -112,10 +112,12 @@ class JSONUpdate : public testing::Test protected: static void SetUpTestCase() { + const char* decrypt_key="himaat!"; system_cmd_cp(old_json, watched_json); _shared_feather_j=Maat_feather(g_iThreadNum, table_info_path, g_logger); + // Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_DECRYPT_KEY, decrypt_key, strlen(decrypt_key)+1); Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_JSON_FILE_PATH, watched_json, strlen(watched_json)+1); - Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms)); + Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms)); Maat_initiate_feather(_shared_feather_j); From 987cb5708a088b6365337404b6591dcb9c86104f Mon Sep 17 00:00:00 2001 From: zhengchao Date: Wed, 22 Jan 2020 20:49:45 +0800 Subject: [PATCH 29/29] =?UTF-8?q?maat=20json=E6=96=87=E4=BB=B6=E6=94=AF?= =?UTF-8?q?=E6=8C=81aes-256-cbc=E5=8A=A0=E5=AF=86=EF=BC=8C=E5=AF=86?= =?UTF-8?q?=E7=A0=81=E9=80=9A=E8=BF=87MAAT=5FOPT=5FDECRYPT=5FKEY=E9=80=89?= =?UTF-8?q?=E9=A1=B9=E6=8C=87=E5=AE=9A=EF=BC=8C=E5=8F=AA=E5=9C=A8=E5=86=85?= =?UTF-8?q?=E5=AD=98=E4=B8=AD=E8=A7=A3=E5=AF=86=EF=BC=8Ciris=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E7=9A=84=E4=B8=AD=E9=97=B4=E7=8A=B6=E6=80=81=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E4=B9=9F=E8=A2=AB=E5=8A=A0=E5=AF=86=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 8 ++- src/entry/Maat_utils.cpp | 109 +++++++++++++++++----------------- src/entry/config_monitor.cpp | 18 +++--- src/entry/json2iris.cpp | 79 ++++++++++++++++-------- src/inc_internal/Maat_utils.h | 12 ++-- src/inc_internal/json2iris.h | 2 +- test/test_maatframe.cpp | 14 +++-- tools/maat_redis_tool.cpp | 6 +- 8 files changed, 144 insertions(+), 104 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 75b6082..7f0a637 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -513,18 +513,18 @@ int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char int ret=0; struct stat fstat_buf; char* json_buff=NULL; - + size_t buff_sz=0; MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module , "Maat initial with JSON file %s, formating..", maat_json_fn); if(strlen(feather->decrypt_key)&&strlen(feather->decrypt_algo)) { - ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&json_buff, err_str, err_str_sz); + ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&json_buff, &buff_sz, err_str, err_str_sz); } if(json_buff==NULL)//decryption failed or no decryption. { - ret=load_file_to_memory(maat_json_fn, &json_buff); + ret=load_file_to_memory(maat_json_fn, (unsigned char**)&json_buff, &buff_sz); } ret=json2iris(json_buff, maat_json_fn, @@ -532,6 +532,8 @@ int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char NULL, feather->json_ctx.iris_file, sizeof(feather->json_ctx.iris_file), + strlen(feather->decrypt_key)?feather->decrypt_key:NULL, + strlen(feather->decrypt_algo)?feather->decrypt_algo:NULL, feather->logger); free(json_buff); json_buff=NULL; diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 58da1a8..8462ba7 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -173,7 +173,7 @@ size_t memcat(void**dest, size_t offset, size_t *n_dest, const void* src, size_t *n_dest=(offset+n_src)*2; *dest=realloc(*dest, sizeof(char)*(*n_dest)); } - memcpy(*dest+offset, src, n_src); + memcpy((char*)*dest+offset, src, n_src); return n_src; } @@ -212,16 +212,21 @@ int system_cmd_mkdir(const char* path) int system_cmd_mv(const char* src_file,const char*dst_file) { char cmd[MAX_SYSTEM_CMD_LEN] = { 0 }; - snprintf(cmd,sizeof(cmd), "mv %s %s", src_file, dst_file); + snprintf(cmd, sizeof(cmd), "mv %s %s", src_file, dst_file); return system(cmd); } int system_cmd_cp(const char* src_file,const char*dst_file) { char cmd[MAX_SYSTEM_CMD_LEN] = { 0 }; - snprintf(cmd,sizeof(cmd), "cp -f %s %s", src_file, dst_file); + snprintf(cmd, sizeof(cmd), "cp -f %s %s", src_file, dst_file); + return system(cmd); +} +int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password) +{ + char cmd[MAX_SYSTEM_CMD_LEN] = { 0 }; + snprintf(cmd,sizeof(cmd), "openssl enc -e -aes-256-cbc -k %s -p -nosalt -in %s -out %s", password, src_file, dst_file); return system(cmd); } - int system_cmd_rm(const char* src_file) { char cmd[MAX_SYSTEM_CMD_LEN] = { 0 }; @@ -270,13 +275,11 @@ int lqueue_destroy_cb(void *data, long data_len, void *arg) assert(0); return 0; } -#define DECRYPT_BLOCK_SIZE (16*1024) -int decrypt_open(const char* filename, const char* key, const char* algorithm, unsigned char**pp_out, char* err_str, size_t err_str_sz) -{ - unsigned char inbuf[DECRYPT_BLOCK_SIZE]; - int inlen, out_blk_len=0; - int out_buff_len=0,buff_offset=0; +int crypt_memory(const unsigned char* inbuf, size_t inlen, unsigned char** pp_out, size_t *out_sz, const char* key, const char* algorithm, int do_encrypt, char* err_str, size_t err_str_sz) +{ + int ret=0, out_blk_len=0; + int out_buff_len=0, out_buff_offset=0; EVP_CIPHER_CTX *ctx; unsigned char cipher_key[EVP_MAX_KEY_LENGTH]; @@ -287,19 +290,12 @@ int decrypt_open(const char* filename, const char* key, const char* algorithm, u const EVP_CIPHER *cipher; const EVP_MD *dgst=NULL; const unsigned char *salt=NULL; - int ret=0; - - FILE*in=fopen(filename, "r"); - if(in==NULL) - { - return -1; - } OpenSSL_add_all_algorithms(); cipher=EVP_get_cipherbyname(algorithm); if(cipher==NULL) { - snprintf(err_str, err_str_sz, "Cipher %s is not supported.",algorithm); + snprintf(err_str, err_str_sz, "Cipher %s is not supported.", algorithm); return 0; } dgst=EVP_get_digestbyname("md5"); @@ -316,52 +312,39 @@ int decrypt_open(const char* filename, const char* key, const char* algorithm, u } /* Don't set key or IV right away; we want to check lengths */ ctx = EVP_CIPHER_CTX_new(); - EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL,0); + EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, do_encrypt); OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) % 16==0); OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16); /* Now we can set key and IV */ - EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, 0); - out_buff_len=DECRYPT_BLOCK_SIZE; + //It should be set to 1 for encryption, 0 for decryption and -1 to leave the value unchanged (the actual value of 'enc' being supplied in a previous call). + EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, -1); + out_buff_len=inlen+EVP_CIPHER_block_size(cipher)-1; *pp_out=(unsigned char*)malloc(out_buff_len*sizeof(unsigned char)); - for (;;) + if (!EVP_CipherUpdate(ctx, *pp_out+out_buff_offset, &out_blk_len, inbuf, inlen)) { - inlen = fread(inbuf, 1, sizeof(inbuf), in); - if (inlen <= 0) - break; - - if(out_buff_len-buff_offsetencryp_algorithm)>0) { @@ -294,20 +294,20 @@ int cm_read_table_file(struct cm_table_info_t* index, MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, no key to decrypt %s.",index->cfg_path); return -1; } - file_sz=decrypt_open(index->cfg_path, key, index->encryp_algorithm, (unsigned char**)&table_file_buff, error_string, sizeof(error_string)); - if(file_sz==0) + ret=decrypt_open(index->cfg_path, key, index->encryp_algorithm, (unsigned char**)&table_file_buff, &file_sz, error_string, sizeof(error_string)); + if(ret<0) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, %s decrypt failed: %s", + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, decrypt %s failed: %s", index->cfg_path, error_string); return -1; } } else { - file_sz=load_file_to_memory(index->cfg_path, &table_file_buff); - if(file_sz==0) + ret=load_file_to_memory(index->cfg_path, (unsigned char**)&table_file_buff, &file_sz); + if(ret<0) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, %s decrypt failed: %s", + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, open %s failed: %s", index->cfg_path, error_string); return -1; } diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 60ff5e4..31943f3 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -51,6 +51,9 @@ struct iris_description_t MESA_htable_handle iris_table_map; MESA_htable_handle str2int_map; redisContext *redis_write_ctx; + char* encrypt_key; + char* encrypt_algo; + FILE* idx_fp; }; struct traslate_command_t { @@ -119,7 +122,7 @@ static int get_region_seq(struct iris_description_t* iris_cfg) return sequence; } -int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,const char* group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger) +int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_key, const char* encrypt_algo, const char*compile_tn,const char* group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger) { memset(iris_cfg,0,sizeof(struct iris_description_t)); snprintf(iris_cfg->tmp_iris_dir,sizeof(iris_cfg->tmp_iris_dir),"%s_iris_tmp",json_file); @@ -186,6 +189,12 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn, iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn, TABLE_TYPE_COMPILE); iris_cfg->group_table=query_table_info(iris_cfg, group_tn, TABLE_TYPE_GROUP); + + if(encrypt_key && encrypt_algo) + { + iris_cfg->encrypt_key=_maat_strdup(encrypt_key); + iris_cfg->encrypt_algo=_maat_strdup(encrypt_algo); + } return 0; } @@ -200,6 +209,8 @@ void clear_iris_descriptor(struct iris_description_t *iris_cfg) MESA_htable_destroy(iris_cfg->iris_table_map, NULL); } map_destroy(iris_cfg->str2int_map); + free(iris_cfg->encrypt_algo); + free(iris_cfg->encrypt_key); return; } int create_tmp_dir(struct iris_description_t *p) @@ -862,29 +873,50 @@ int write_group_line(int group_id, int parent_id, int group_not_flag, int parent } void table_idx_write_cb(const uchar * key, uint size, void * data, void * user) { - FILE* fp=NULL; + struct iris_description_t *p_iris=(struct iris_description_t *)user; struct iris_table_t* table=(struct iris_table_t*)data; - fp=fopen(table->table_path, "w"); - fprintf(fp,"%d\n", table->line_count); - fwrite(table->buff, table->write_pos, 1, fp); - fclose(fp); - - fp=(FILE*)user; - fprintf(fp,"%s\t%d\t%s\n", table->table_name, table->line_count, table->table_path); + FILE* table_fp=NULL; + char line_cnt_str[32], err_str[256]; + snprintf(line_cnt_str, sizeof(line_cnt_str), "%010d\n", table->line_count); + int ret=0; + size_t table_file_sz=strlen(line_cnt_str)+table->write_pos; + unsigned char* buff=ALLOC(unsigned char, table_file_sz); + unsigned char* encrypt_buff=NULL; + size_t encrypt_buff_sz=0; + memcpy(buff, line_cnt_str, strlen(line_cnt_str)); + memcpy(buff+strlen(line_cnt_str), table->buff, table->write_pos); + table_fp=fopen(table->table_path, "w"); + if(p_iris->encrypt_key) + { + ret=crypt_memory(buff, table_file_sz, &encrypt_buff, &encrypt_buff_sz, p_iris->encrypt_key, p_iris->encrypt_algo, 1, err_str, sizeof(err_str)); + assert(ret==0); + fwrite(encrypt_buff, encrypt_buff_sz, 1, table_fp); + fprintf(p_iris->idx_fp,"%s\t%d\t%s\t%s\n", table->table_name, table->line_count, table->table_path, p_iris->encrypt_algo); + } + else + { + fwrite(buff, table_file_sz, 1, table_fp); + fprintf(p_iris->idx_fp,"%s\t%d\t%s\n", table->table_name, table->line_count, table->table_path); + } + fclose(table_fp); + free(buff); + buff=NULL; + + } int write_index_file(struct iris_description_t *p_iris,void* logger) { - FILE*fp=NULL; - fp=fopen(p_iris->index_path,"w"); - if(fp==NULL) + p_iris->idx_fp=fopen(p_iris->index_path,"w"); + if(p_iris->idx_fp==NULL) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "index file %s fopen error %s.",p_iris->index_path,strerror(errno)); + "index file %s fopen error %s.",p_iris->index_path, strerror(errno)); return -1; } - MESA_htable_iterate(p_iris->iris_table_map, table_idx_write_cb, fp); - fclose(fp); + MESA_htable_iterate(p_iris->iris_table_map, table_idx_write_cb, p_iris); + fclose(p_iris->idx_fp); + p_iris->idx_fp=NULL; return 0; } int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger) @@ -1053,31 +1085,30 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) return 0; } // redis_write_ctx is used by maat_redis_tool to write json to redis. -int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, void* logger) +int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger) { cJSON *json=NULL, *tmp_obj=NULL; int ret=-1; struct iris_description_t iris_cfg; - memset(&iris_cfg,0,sizeof(iris_cfg)); - + memset(&iris_cfg, 0, sizeof(iris_cfg)); json=cJSON_Parse(json_buff); if (!json) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,"Error before: %-200.200s",cJSON_GetErrorPtr()); goto error_out; } - tmp_obj=cJSON_GetObjectItem(json,"compile_table"); + tmp_obj=cJSON_GetObjectItem(json, "compile_table"); if(tmp_obj) { compile_tn=tmp_obj->valuestring; } - tmp_obj=cJSON_GetObjectItem(json,"group_table"); + tmp_obj=cJSON_GetObjectItem(json, "group_table"); if(tmp_obj) { group_tn=tmp_obj->valuestring; } - ret=set_iris_descriptor(json_filename, json, compile_tn, group_tn, redis_write_ctx, &iris_cfg, logger); + ret=set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, compile_tn, group_tn, redis_write_ctx, &iris_cfg, logger); if(ret<0) { goto error_out; @@ -1086,15 +1117,15 @@ int json2iris(const char* json_buff, const char* json_filename, const char*compi if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, - "create tmp folder %s error",iris_cfg.tmp_iris_dir); + "create tmp folder %s error", iris_cfg.tmp_iris_dir); goto error_out; } - ret=write_iris(json ,&iris_cfg, logger); + ret=write_iris(json, &iris_cfg, logger); if(ret<0) { goto error_out; } - memcpy(iris_dir_buf,iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir)+1, (unsigned int)buf_len)); + memcpy(iris_dir_buf, iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir)+1, (unsigned int)buf_len)); cJSON_Delete(json); clear_iris_descriptor(&iris_cfg); diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index faf5a67..ac4054b 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -69,14 +69,18 @@ size_t memcat(void**dest, size_t offset, size_t *n_dest, const void* src, size_t pid_t gettid(void); int system_cmd_mkdir(const char* path); int system_cmd_rm(const char* src_file); -int system_cmd_mv(const char* src_file,const char*dst_file); -int system_cmd_cp(const char* src_file,const char*dst_file); +int system_cmd_mv(const char* src_file, const char*dst_file); +int system_cmd_cp(const char* src_file, const char*dst_file); +int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password); + char* md5_file(const char* filename, char* md5string); int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len); const char** charset_get_all_name(void); const char* charset_get_name(enum MAAT_CHARSET charset); int lqueue_destroy_cb(void *data, long data_len, void *arg); -int decrypt_open(const char* filename, const char* key, const char* algorithm, unsigned char**pp_out, char* err_str, size_t err_str_sz); -int load_file_to_memory(const char* file_name, char**pp_out); +int decrypt_open(const char* file_name, const char* key, const char* algorithm, unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz); +int load_file_to_memory(const char* file_name, unsigned char**pp_out, size_t *out_sz); +//do_encrypt: 1 for encryption, 0 for decryption. +int crypt_memory(const unsigned char* inbuf, size_t inlen, unsigned char** pp_out, size_t *out_sz, const char* key, const char* algorithm, int do_encrypt, char* err_str, size_t err_str_sz); diff --git a/src/inc_internal/json2iris.h b/src/inc_internal/json2iris.h index ab8e5c6..247a191 100644 --- a/src/inc_internal/json2iris.h +++ b/src/inc_internal/json2iris.h @@ -1,6 +1,6 @@ #ifndef H_MAAT_JSON2IRIS_H_INCLUDE #define H_MAAT_JSON2IRIS_H_INCLUDE -int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, void* logger); +int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger); int set_file_rulenum(const char* path, int rulenum, void* logger); #endif diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 665c21b..b027377 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -32,6 +32,7 @@ extern int my_scandir(const char *dir, struct dirent ***namelist, int(*compar)(const void *, const void *)); extern char* md5_file(const char* filename, char* md5string); extern int system_cmd_cp(const char* src_file,const char*dst_file); +extern int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password); Maat_feather_t g_feather=NULL; void *g_logger=NULL; int g_iThreadNum=4; @@ -105,6 +106,7 @@ const char* watched_json="./json_update/maat.json"; const char* old_json="./json_update/old.json"; const char* new_json="./json_update/new.json"; const char* corrupted_json="./json_update/corrupted.json"; +const char* json_decrypt_key="himaat!"; class JSONUpdate : public testing::Test { @@ -112,10 +114,11 @@ class JSONUpdate : public testing::Test protected: static void SetUpTestCase() { - const char* decrypt_key="himaat!"; - system_cmd_cp(old_json, watched_json); + + system_cmd_encrypt(old_json, watched_json, json_decrypt_key); + _shared_feather_j=Maat_feather(g_iThreadNum, table_info_path, g_logger); - // Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_DECRYPT_KEY, decrypt_key, strlen(decrypt_key)+1); + Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_DECRYPT_KEY, json_decrypt_key, strlen(json_decrypt_key)+1); Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_JSON_FILE_PATH, watched_json, strlen(watched_json)+1); Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms)); @@ -139,10 +142,11 @@ TEST_F(JSONUpdate, OldCfg) } TEST_F(JSONUpdate, NewCfg) { - system_cmd_cp(corrupted_json, watched_json); + system_cmd_encrypt(corrupted_json, watched_json, json_decrypt_key); sleep(2); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1); - system_cmd_cp(new_json, watched_json); + + system_cmd_encrypt(new_json, watched_json, json_decrypt_key); sleep(5); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 0); } diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp index 584ab3e..41fe97e 100644 --- a/tools/maat_redis_tool.cpp +++ b/tools/maat_redis_tool.cpp @@ -239,7 +239,7 @@ int main(int argc, char * argv[]) unsigned long json_file_size=0,read_size=0; long long desired_version=0; char* json_buff=NULL; - + size_t json_buff_sz=0; while((oc=getopt(argc,argv,"h:p:n:d:v:f:j:t:"))!=-1) { switch(oc) @@ -327,12 +327,12 @@ int main(int argc, char * argv[]) } else if(model==WORK_MODE_JSON) { - ret=load_file_to_memory(json_file, &json_buff); + ret=load_file_to_memory(json_file, (unsigned char**)&json_buff, &json_buff_sz); if(ret<0) { printf("open %s failed.\n", json_file); } - ret=json2iris(json_buff, json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL); + ret=json2iris(json_buff, json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, NULL); if(ret<0) { printf("Invalid json format.\n");