diff --git a/inc/Maat_command.h b/inc/Maat_command.h index f0c8ce9..816dd63 100644 --- a/inc/Maat_command.h +++ b/inc/Maat_command.h @@ -10,7 +10,11 @@ enum MAAT_OPERATION MAAT_OP_ADD, MAAT_OP_RENEW_TIMEOUT //Rule expire time is changed to now+cmd->expire_after }; - +enum MAAT_GROUP_RELATION +{ + PARENT_TYPE_COMPILE=0, + PARENT_TYPE_GROUP +}; enum MAAT_REGION_TYPE { REGION_EXPR, @@ -104,8 +108,12 @@ struct Maat_region_t }; struct Maat_group_t { - int region_num; + const char* table_name; int group_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number. + int parent_id; + int not_flag; + enum MAAT_GROUP_RELATION parent_type; + int region_num; struct Maat_region_t *regions; }; struct Maat_cmd_t @@ -170,5 +178,14 @@ void Maat_cmd_key_free(struct Maat_cmd_key**keys, int number); int Maat_cmd_key_select(Maat_feather_t feather, int label_id, struct Maat_cmd_key** keys); int Maat_cmd_select(Maat_feather_t feather, int label_id, int * output_ids, unsigned int size); int Maat_cmd_flushDB(Maat_feather_t feather); + +int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num); +int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_region_t* region, int group_id); +int Maat_command_raw_set_group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_group_t* group); + +int Maat_cmd_get_new_group_id(Maat_feather_t feather); +int Maat_cmd_get_new_region_id(Maat_feather_t feather); + + #endif diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 1b2f5dc..71eb650 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -185,8 +185,8 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, enum MAAT_SCAN_OPT { - MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*,SIZE= strlen(string).DEFAULT: no default. - MAAT_SET_SCAN_LAST_REGION //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan cobination. + MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*,SIZE= strlen(string). DEFAULT: no default. + MAAT_SET_SCAN_LAST_REGION //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination. }; //return 0 if success, return -1 when failed; int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT type,const void* value,int size); @@ -209,7 +209,7 @@ int Maat_full_scan_string(Maat_feather_t feather,int table_id ,enum MAAT_CHARSET charset,const char* data,int data_len ,struct Maat_rule_t*result,int* found_pos,int rule_num ,scan_status_t* mid,int thread_num); -//hite_detail could be NULL if unconcern +//hit_detail could be NULL if not cared. int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id ,enum MAAT_CHARSET charset,const char* data,int data_len ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num @@ -220,7 +220,7 @@ int Maat_stream_scan_string(stream_para_t* stream_para ,enum MAAT_CHARSET charset,const char* data,int data_len ,struct Maat_rule_t*result,int* found_pos,int rule_num ,scan_status_t* mid); -//hited_detail could be NULL if unconcern +//hited_detail could be NULL if not cared. int Maat_stream_scan_string_detail(stream_para_t* stream_para ,enum MAAT_CHARSET charset,const char* data,int data_len ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d6a144f..483ddf3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.5) set(MAAT_FRAME_MAJOR_VERSION 2) -set(MAAT_FRAME_MINOR_VERSION 6) +set(MAAT_FRAME_MINOR_VERSION 7) set(MAAT_FRAME_PATCH_VERSION 0) set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION}) @@ -20,6 +20,7 @@ set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) target_include_directories(maat_frame_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/) #target_include_directories(maat_frame_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/hiredis) target_link_libraries(maat_frame_static hiredis-vip-static) +target_link_libraries(maat_frame_static igraph-static) # Shared Library Output add_library(maat_frame_shared SHARED ${MAAT_SRC}) @@ -32,6 +33,7 @@ set_target_properties(maat_frame_shared PROPERTIES LINK_FLAGS "-Wl,--version-scr target_include_directories(maat_frame_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/) target_include_directories(maat_frame_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/hiredis) target_link_libraries(maat_frame_shared hiredis-vip-static ${MAAT_DEPEND_DYN_LIB}) +target_link_libraries(maat_frame_shared igraph-static ${MAAT_DEPEND_DYN_LIB}) install(FILES ${PROJECT_SOURCE_DIR}/inc/Maat_rule.h DESTINATION include/MESA/) install(FILES ${PROJECT_SOURCE_DIR}/inc/Maat_command.h DESTINATION include/MESA/) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index ecdcb79..e332f45 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -16,7 +16,7 @@ #include "rulescan.h" #include "json2iris.h" -struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type) +struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int table_id, enum MAAT_TABLE_TYPE expect_type) { struct Maat_table_desc *p_table=NULL; if(table_id>MAX_TABLE_NUM) @@ -34,20 +34,20 @@ struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather,int tabl } if(p_table->table_type!=expect_type) { - if(expect_type!=TABLE_TYPE_EXPR|| - p_table->table_type!=TABLE_TYPE_EXPR_PLUS) + if((expect_type==TABLE_TYPE_EXPR && p_table->table_type!=TABLE_TYPE_EXPR_PLUS)|| + (expect_type==TABLE_TYPE_IP && p_table->table_type!=TABLE_TYPE_IP_PLUS)) { return NULL; } } return p_table; } -inline void INC_SCANNER_REF(Maat_scanner_t*scanner,int thread_num) +inline void INC_SCANNER_REF(Maat_scanner*scanner,int thread_num) { alignment_int64_array_add(scanner->ref_cnt, thread_num, 1); return; } -inline void DEC_SCANNER_REF(Maat_scanner_t*scanner,int thread_num) +inline void DEC_SCANNER_REF(Maat_scanner*scanner,int thread_num) { alignment_int64_array_add(scanner->ref_cnt, thread_num, -1); @@ -64,7 +64,8 @@ void Maat_clean_status(scan_status_t* mid) alignment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1); if(_mid->inner!=NULL) { - free(_mid->inner->hitted_group_id); + dynamic_array_destroy(_mid->inner->cur_hit_groups, NULL); + free(_mid->inner->all_hit_group_array); free(_mid->inner); alignment_int64_array_add(_mid->feather->inner_mid_cnt,_mid->thread_num,-1); } @@ -117,15 +118,17 @@ int insert_set_id(unsigned long long **set, size_t* size, size_t cnt, unsigned l } } size_t pickup_hit_region_from_compile(struct bool_expr *compile_hit, - const unsigned long long* hitted_id, size_t hit_cnt, int* region_pos, size_t size) + struct dynamic_array_t* hitted_id, size_t hit_cnt, int* region_pos, size_t size) { size_t i=0, j=0; size_t k=0; + unsigned long long group_id=0; for(i=0;iitem_num; j++) - { - if(hitted_id[i]==compile_hit->items[j].item_id) + { + if(group_id==compile_hit->items[j].item_id) { region_pos[k]=i; k++; @@ -135,114 +138,111 @@ size_t pickup_hit_region_from_compile(struct bool_expr *compile_hit, } return k; } -void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len) +void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len) { - memcpy(rule, rule_head, sizeof(struct _head_Maat_rule_t)); + memcpy(rule, rule_head, sizeof(struct Maat_rule_head)); memcpy(rule->service_defined, srv_def, MIN(srv_def_len,MAX_SERVICE_DEFINE_LEN)); return; } -static int compare_compile_id(const void *a, const void *b) +static int compare_compile_inner(const void *a, const void *b) { - struct Maat_rule_t *ra=(struct Maat_rule_t *)a; - struct Maat_rule_t *rb=(struct Maat_rule_t *)b; + const struct Maat_compile_group_relation *ra=*(const struct Maat_compile_group_relation **)a; + const struct Maat_compile_group_relation *rb=*(const struct Maat_compile_group_relation **)b; - return (rb->config_id-ra->config_id); + if(ra->group_cnt!=rb->group_cnt) + { + return (ra->group_cnt-rb->group_cnt); + } + else + { + return (rb->compile_id-ra->compile_id); + } } int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num) { - int scan_ret=0,result_cnt=0; - int ret=0,i=0; + int scan_ret=0, result_cnt=0; + int ret=0, i=0, j=0; size_t r_in_c_cnt=0; - int shortcut_avilable_cnt=0; unsigned char has_not_flag=0; struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler; - struct Maat_group_inner_t* group_rule=NULL; - struct Maat_compile_inner_t* array_mi_rule[MAX_SCANNER_HIT_NUM]; - struct Maat_compile_inner_t* _mi_rule=NULL; + struct Maat_group_inner* group_rule=NULL; + struct Maat_compile_group_relation* relation_array[MAX_SCANNER_HIT_NUM]; + struct Maat_compile_group_relation* relation=NULL; + int region_pos[MAX_SCANNER_HIT_NUM]; - _mid->cur_hit_cnt=0; + _mid->cur_hit_group_cnt=0; for(i=0;igroup_id<0) { continue; } - if(group_rule->compile_shortcut!=NULL&&group_rule->ref_cnt==1&&shortcut_avilable_cntcompile_shortcut); - shortcut_avilable_cnt++; + assert(group_rule->ref_by_children_cnt>=0); + assert(group_rule->ref_by_parent_cnt>=0); + for(j=0; jtop_group_cnt; j++) + { + if(_mid->cur_hit_group_cntcur_hit_groups, _mid->cur_hit_group_cnt, (void*)group_rule->top_groups[j]); + _mid->cur_hit_group_cnt++; + } + ret=insert_set_id(&(_mid->all_hit_group_array), + &(_mid->all_hit_group_array_sz), + _mid->all_hit_group_cnt, + group_rule->top_groups[j]); + _mid->all_hit_group_cnt+=ret; } - _mid->cur_hit_id[_mid->cur_hit_cnt]=group_rule->group_id; - _mid->cur_hit_cnt++; - ret=insert_set_id(&(_mid->hitted_group_id), - &(_mid->hit_group_size), - _mid->hit_group_cnt, - group_rule->group_id); - _mid->hit_group_cnt+=ret; } - if((region_hit_num>0 &&shortcut_avilable_cnt==region_hit_num) || - shortcut_avilable_cnt==MAX_SCANNER_HIT_NUM) + + scan_ret=bool_matcher_match(bm, thread_num, + _mid->all_hit_group_array, _mid->all_hit_group_cnt, + (void **)relation_array, MAX_SCANNER_HIT_NUM); + + if(scan_ret>1) { - //short cut for rules contains one group - scan_ret=shortcut_avilable_cnt; - alignment_int64_array_add(feather->orphan_group_saving, thread_num, 1); - } - else if(0&&shortcut_avilable_cnt==0&®ion_hit_num==1&&_mid->hit_group_cnt==1&&is_last_region==1) - { - //This shortcut is NO longger valid after bool macher support NOT-logic. - //short cut for last scan and combination rules - //region_hit_num==1 : for current scan hitted rules, one and each other group may statisfy a compile rule. - //_mid->hit_group_cnt==1: With pre scan hitted group rules, one group may staisfy a compile rule - scan_ret=0; - } - else - { - scan_ret=bool_matcher_match(bm, thread_num, - _mid->hitted_group_id, _mid->hit_group_cnt, - (void **)array_mi_rule, MAX_SCANNER_HIT_NUM); + qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation**), + compare_compile_inner); } for(i=0;irwlock))) + assert(relation->magic_num==COMPILE_RELATION_MAGIC); + if(0==pthread_rwlock_tryrdlock(&(relation->rwlock))) { - if(_mi_rule->is_valid==1) + if(relation->compile) { - if(_mi_rule->not_group_cnt>0 && !is_last_region) + if(relation->not_group_cnt>0 && !is_last_region) { _mid->not_grp_compile_hitted_flag=1; } else { - make_group_set(_mi_rule, &(rs_result[result_cnt].group_set), &has_not_flag); - r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set), _mid->cur_hit_id, _mid->cur_hit_cnt, + make_group_set(relation, &(rs_result[result_cnt].group_set), &has_not_flag); + r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set), _mid->cur_hit_groups, _mid->cur_hit_group_cnt, region_pos, MAX_SCANNER_HIT_NUM); if(r_in_c_cnt>0 || //compile config hitted becasue of new reigon - _mid->cur_hit_cnt==0) //or ever hit a compile that refer a NOT-logic group + _mid->cur_hit_group_cnt==0) //or ever hit a compile that refer a NOT-logic group { - fill_maat_rule(&(result[result_cnt]), &(_mi_rule->db_c_rule->m_rule_head), - _mi_rule->db_c_rule->service_defined ,_mi_rule->db_c_rule->m_rule_head.serv_def_len); - rs_result[result_cnt].compile_id=_mi_rule->compile_id; + fill_maat_rule(&(result[result_cnt]), &(relation->compile->head), + relation->compile->service_defined ,relation->compile->head.serv_def_len); + rs_result[result_cnt].compile_id=relation->compile_id; result_cnt++; } } } - pthread_rwlock_unlock(&(_mi_rule->rwlock)); + pthread_rwlock_unlock(&(relation->rwlock)); } } if(result_cnt>0) { - qsort(result, result_cnt, sizeof(struct Maat_rule_t), - compare_compile_id); - alignment_int64_array_add(feather->hit_cnt,thread_num,1); + alignment_int64_array_add(feather->hit_cnt, thread_num, 1); } if(region_hit_num==0&&result_cnt>0) { @@ -252,39 +252,34 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int return result_cnt; } -int exprid2region_id(struct Maat_group_inner_t* group_rule,int expr_id,int* district_id) +int exprid2region_id(struct Maat_group_inner* group_rule,int expr_id,int* district_id, Maat_scanner* scanner) { - int i=0,region_id=-1; - struct _Maat_region_inner_t* region_rule=NULL; + int region_id=-1; + struct Maat_region_inner* region_rule=NULL; assert(group_rule->group_id>=0); - pthread_mutex_lock(&(group_rule->mutex)); - for(i=0;iregion_boundary;i++) + int array_idx=(int)(long)HASH_fetch_by_id(scanner->exprid_hash, expr_id); + pthread_mutex_lock(&(group_rule->mutex)); + assert(array_idxregion_boundary); + region_rule=(struct Maat_region_inner*)dynamic_array_read(group_rule->regions, array_idx); + if(region_rule) { - region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group_rule->regions, i); - if(region_rule==NULL) - { - continue; - } - if(expr_id>=region_rule->expr_id_lb&&expr_id<=region_rule->expr_id_ub) - { - region_id=region_rule->region_id; - *district_id=region_rule->district_id; - break; - } + assert(expr_id>=region_rule->expr_id_lb&&expr_id<=region_rule->expr_id_ub); + region_id=region_rule->region_id; + *district_id=region_rule->district_id; } pthread_mutex_unlock(&(group_rule->mutex)); return region_id; } -int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num) +int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num, Maat_scanner* scanner) { - struct Maat_group_inner_t* group_rule=NULL; + struct Maat_group_inner* group_rule=NULL; int i=0; int district_id=-1,region_id=-1; int ret_region_num=region_hit_num; while(i0&&district_id!=_mid->district_id) { ret_region_num--; @@ -368,7 +363,8 @@ int hit_pos_RS2Maat(struct sub_item_pos_t* maat_sub_item,int size,rule_result_t* int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mid, scan_result_t *region_hit,int region_cnt, _compile_result_t *compile_hit,int compile_cnt, - struct Maat_hit_detail_t *hit_detail,int detail_num) + struct Maat_hit_detail_t *hit_detail,int detail_num, + struct Maat_scanner* scanner) { int i=0,j=0,k=0; char r_in_c_flag[region_cnt]; @@ -379,20 +375,20 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi memset(r_in_c_flag,0,sizeof(r_in_c_flag)); memset(region_pos,0,sizeof(region_pos)); - struct Maat_group_inner_t* group_rule=NULL; + struct Maat_group_inner* group_rule=NULL; //for each hitted compile cfg,find its region_ids for(i=0;icur_hit_id, _mid->cur_hit_cnt, + r_in_c_cnt=pickup_hit_region_from_compile(&(compile_hit[i].group_set), _mid->cur_hit_groups, _mid->cur_hit_group_cnt, region_pos, MAX_SCANNER_HIT_NUM); assert(r_in_c_cnt>0);//previous hitted compile was elimited in region_compile for(j=0,k=0;jcur_hit_cnt=0; - inner_mid->hit_group_cnt=0; - inner_mid->hit_group_size=4; - inner_mid->hitted_group_id= ALLOC(unsigned long long, inner_mid->hit_group_size); + inner_mid->cur_hit_group_cnt=0; + inner_mid->cur_hit_groups=dynamic_array_create(32, 32); + inner_mid->all_hit_group_cnt=0; + inner_mid->all_hit_group_array_sz=4; + inner_mid->all_hit_group_array= ALLOC(unsigned long long, inner_mid->all_hit_group_array_sz); return inner_mid; } struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather, int thread_num) @@ -503,7 +500,12 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* return NULL; } _Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1); - feather->table_cnt=read_table_description(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger); + feather->table_cnt=read_table_description(feather->p_table_info, MAX_TABLE_NUM, table_info_path, max_thread_num, logger); + if(feather->table_cnt==0) + { + free(feather); + return NULL; + } feather->map_tablename2id=map_create(); int i=0,j=0,ret=0; for(i=0;iouter_mid_cnt=alignment_int64_array_alloc(max_thread_num); feather->inner_mid_cnt=alignment_int64_array_alloc(max_thread_num); feather->hit_cnt=alignment_int64_array_alloc(max_thread_num); - feather->orphan_group_saving=alignment_int64_array_alloc(max_thread_num); feather->not_grp_hit_cnt=alignment_int64_array_alloc(max_thread_num); feather->maat_version=0; feather->last_full_version=0; @@ -636,12 +637,12 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo break; case MAAT_OPT_JSON_FILE_PATH: assert(_feather->input_mode==SOURCE_NONE); - ret=json2iris((const char*)value - ,_feather->compile_tn,_feather->group_tn - ,NULL - ,_feather->json_ctx.iris_file - ,sizeof(_feather->json_ctx.iris_file) - ,_feather->logger); + ret=json2iris((const char*)value, + _feather->compile_tn,_feather->group_tn, + NULL, + _feather->json_ctx.iris_file, + sizeof(_feather->json_ctx.iris_file), + _feather->logger); if(ret<0) { return -1; @@ -1042,19 +1043,21 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user) { struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user; - struct Maat_compile_inner_t *compile_inner=(struct Maat_compile_inner_t *)data; + struct Maat_compile_group_relation *relation=(struct Maat_compile_group_relation *)data; + struct Maat_compile_rule* compile_rule=relation->compile; MAAT_RULE_EX_DATA ad=NULL; - if(compile_inner->ref_table==NULL || compile_inner->ref_table->table_id!=ex_desc->table_id || compile_inner->db_c_rule==NULL) + if(compile_rule==NULL|| compile_rule->ref_table->table_id!=ex_desc->table_id) { return; } - pthread_rwlock_wrlock(&(compile_inner->rwlock)); - ad=rule_ex_data_new(&(compile_inner->db_c_rule->m_rule_head), - compile_inner->db_c_rule->service_defined, + + pthread_rwlock_wrlock(&(relation->rwlock)); + ad=rule_ex_data_new(&(relation->compile->head), + relation->compile->service_defined, ex_desc); - compile_inner->ads[ex_desc->idx]=ad; - pthread_rwlock_unlock(&(compile_inner->rwlock)); + compile_rule->ads[ex_desc->idx]=ad; + pthread_rwlock_unlock(&(relation->rwlock)); return; } int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name, @@ -1104,22 +1107,24 @@ failed: MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - struct Maat_compile_inner_t *compile_inner=NULL; + struct Maat_compile_group_relation *relation=NULL; + struct Maat_compile_rule* compile=NULL; const struct compile_table_desc* compile_desc=NULL; const struct compile_ex_data_idx* ex_desc=NULL; MAAT_RULE_EX_DATA ad=NULL; - compile_inner=(struct Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); - if(compile_inner==NULL) + relation=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); + if(relation==NULL||relation->compile==NULL) { return NULL; } - pthread_rwlock_rdlock(&(compile_inner->rwlock)); - compile_desc=&(compile_inner->ref_table->compile); + pthread_rwlock_rdlock(&(relation->rwlock)); + compile=relation->compile; + compile_desc=&(compile->ref_table->compile); assert(idxex_data_num); ex_desc=compile_desc->ex_desc+idx; - ex_desc->dup_func(ex_desc->idx, &ad, compile_inner->ads+idx, ex_desc->argl,ex_desc->argp); - pthread_rwlock_unlock(&(compile_inner->rwlock)); + ex_desc->dup_func(ex_desc->idx, &ad, compile->ads+idx, ex_desc->argl,ex_desc->argp); + pthread_rwlock_unlock(&(relation->rwlock)); return ad; } struct wrap_plugin_EX_data @@ -1152,7 +1157,7 @@ MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_ MESA_htable_create_args_t hargs; memset(&hargs,0,sizeof(hargs)); - hargs.thread_safe=0; + hargs.thread_safe=8; hargs.hash_slot_size = slot_size; hargs.max_elem_num = 0; hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; @@ -1310,7 +1315,6 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id return NULL; } table_rt= _feather->scanner->table_rt[table_id]; - pthread_rwlock_rdlock(&table_rt->plugin.rwlock); wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash, (const unsigned char*)key, strlen(key)); if(wrap_data!=NULL) @@ -1318,7 +1322,6 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata), plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); } - pthread_rwlock_unlock(&table_rt->plugin.rwlock); return exdata; } @@ -1337,7 +1340,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id struct Maat_table_desc *p_table=NULL; struct expr_table_desc* expr_desc=NULL; struct timespec start,end; - Maat_scanner_t* my_scanner=NULL; + Maat_scanner* my_scanner=NULL; if(data==NULL||data_len<=0) { return 0; @@ -1410,7 +1413,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS) { - hit_region_cnt=match_district(_mid,region_result,hit_region_cnt); + hit_region_cnt=match_district(_mid,region_result,hit_region_cnt, my_scanner); } if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid)) { @@ -1435,7 +1438,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id *detail_ret=fill_region_hit_detail(data,_mid->inner, region_result,hit_region_cnt, compile_result,compile_ret, - hit_detail,detail_num); + hit_detail,detail_num, my_scanner); } } if(_feather->perf_on==1) @@ -1477,7 +1480,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id scan_result_t *region_result=NULL; _compile_result_t compile_result[rule_num]; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_scanner_t* my_scanner=NULL; + struct Maat_scanner* my_scanner=NULL; intval_scan_data.rule_type=RULETYPE_INT; intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); intval_scan_data.int_data=intval; @@ -1567,7 +1570,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id Maat_table_desc* p_table=NULL; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_scanner_t* my_scanner=NULL; + struct Maat_scanner* my_scanner=NULL; struct timespec start,end; if(_feather->perf_on==1) { @@ -1688,7 +1691,7 @@ int Maat_scan_addr(Maat_feather_t feather,int table_id stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,int thread_num) { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_scanner_t* scanner=NULL; + struct Maat_scanner* scanner=NULL; struct Maat_table_desc *p_table=NULL; assert(thread_num<_feather->scan_thread_num); @@ -1746,7 +1749,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para ,int* detail_ret,scan_status_t* mid) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); - struct Maat_scanner_t* scanner=sp->feather->scanner; + struct Maat_scanner* scanner=sp->feather->scanner; int sub_type=0; int region_ret=0,hit_region_cnt=0,compile_ret=0; @@ -1855,7 +1858,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para } if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS) { - hit_region_cnt=match_district(_mid,region_result,hit_region_cnt); + hit_region_cnt=match_district(_mid, region_result, hit_region_cnt, scanner); } if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid)) { @@ -1882,14 +1885,14 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para *detail_ret=fill_region_hit_detail(sp->scan_buff,_mid->inner, region_result,hit_region_cnt, compile_result,compile_ret, - hit_detail,detail_num); + hit_detail,detail_num, scanner); } else { *detail_ret=fill_region_hit_detail(data,_mid->inner, region_result,hit_region_cnt, compile_result,compile_ret, - hit_detail,detail_num); + hit_detail,detail_num, scanner); } } } @@ -1929,7 +1932,7 @@ int Maat_stream_scan_string(stream_para_t* stream_para void Maat_stream_scan_string_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); - struct Maat_scanner_t* scanner=sp->feather->scanner; + struct Maat_scanner* scanner=sp->feather->scanner; struct Maat_table_runtime* table_rt=NULL; if(scanner!=NULL) { @@ -1971,7 +1974,7 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para) stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,unsigned long long total_len,int thread_num) { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_scanner_t* scanner=NULL; + struct Maat_scanner* scanner=NULL; sfh_instance_t * tmp_fuzzy_handle=NULL; struct Maat_table_desc *p_table=NULL; p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST); @@ -2130,7 +2133,7 @@ fast_out: void Maat_stream_scan_digest_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); - struct Maat_scanner_t* scanner=sp->feather->scanner; + struct Maat_scanner* scanner=sp->feather->scanner; struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1); if(scanner!=NULL) @@ -2153,11 +2156,11 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum { int ret=0; struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - struct Maat_compile_inner_t *compile_inner=NULL; + struct Maat_compile_group_relation *compile_inner=NULL; switch(type) { case MAAT_RULE_SERV_DEFINE: - compile_inner=(struct Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); + compile_inner=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); if(compile_inner==NULL) { ret=0; @@ -2165,8 +2168,8 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum else { pthread_rwlock_rdlock(&(compile_inner->rwlock)); - ret=MIN(size,compile_inner->db_c_rule->m_rule_head.serv_def_len); - memcpy(value,compile_inner->db_c_rule->service_defined,ret); + ret=MIN(size,compile_inner->compile->head.serv_def_len); + memcpy(value,compile_inner->compile->service_defined,ret); pthread_rwlock_unlock(&(compile_inner->rwlock)); } break; @@ -2227,7 +2230,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id GIE_result_t region_result[MAX_SCANNER_HIT_NUM]; _compile_result_t compile_result[rule_num]; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_scanner_t* my_scanner=NULL; + struct Maat_scanner* my_scanner=NULL; Maat_table_desc* p_table=NULL; struct timespec start,end; if(_feather->perf_on==1) diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index cbeebed..a86d303 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -192,6 +192,9 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_ case TABLE_TYPE_IP: column_seq=14; break; + case TABLE_TYPE_IP_PLUS: + column_seq=18; + break; case TABLE_TYPE_COMPILE: column_seq=8; break; @@ -239,13 +242,39 @@ int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) line[i]='0'; return 0; } -void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int size) +void serialize_group(const struct Maat_group_t* p_group, enum MAAT_OPERATION op, char* buff, size_t sz) { - UNUSED int ret=0; + if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD; + snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d", p_group->group_id, + p_group->parent_id, + op, + p_group->not_flag, + p_group->parent_type); + return; +} +void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int group_num, enum MAAT_OPERATION op, char* buff, size_t sz) +{ + if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD; + const char* service_define=huge_service_defined?huge_service_defined:(strlen(p_m_rule->service_defined)?p_m_rule->service_defined:"null"); + + snprintf(buff, sz,"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t%d\t%d", + p_m_rule->config_id, + p_m_rule->service_id, + p_m_rule->action, + p_m_rule->do_blacklist, + p_m_rule->do_log, + service_define, + op, + group_num); + return; +} +void serialize_region(const struct Maat_region_t* p, int group_id, char* buff, size_t sz) +{ + UNUSED size_t ret=0; switch(p->region_type) { case REGION_IP: - ret=snprintf(buff,size,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1" + ret=snprintf(buff,sz,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1" ,p->region_id ,group_id ,p->ip_rule.addr_type @@ -263,7 +292,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int case REGION_EXPR: if(p->expr_rule.district==NULL) { - ret=snprintf(buff,size,"%d\t%d\t%s\t%d\t%d\t%d\t1" + ret=snprintf(buff,sz,"%d\t%d\t%s\t%d\t%d\t%d\t1" ,p->region_id ,group_id ,p->expr_rule.keywords @@ -273,7 +302,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int } else //expr_plus { - ret=snprintf(buff,size,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1" + ret=snprintf(buff,sz,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1" ,p->region_id ,group_id ,p->expr_rule.keywords @@ -284,14 +313,14 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int } break; case REGION_INTERVAL: - ret=snprintf(buff,size,"%d\t%d\t%u\t%u\t1" + ret=snprintf(buff,sz,"%d\t%d\t%u\t%u\t1" ,p->region_id ,group_id ,p->interval_rule.low_boundary ,p->interval_rule.up_boundary); break; case REGION_DIGEST: - ret=snprintf(buff,size,"%d\t%d\t%llu\t%s\t%hd\t1" + ret=snprintf(buff,sz,"%d\t%d\t%llu\t%s\t%hd\t1" ,p->region_id ,group_id ,p->digest_rule.orgin_len @@ -299,7 +328,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int ,p->digest_rule.confidence_degree); break; case REGION_SIMILARITY: - ret=snprintf(buff,size,"%d\t%d\t%s\t%hd\t1" + ret=snprintf(buff,sz,"%d\t%d\t%s\t%hd\t1" ,p->region_id ,group_id ,p->similarity_rule.target @@ -308,7 +337,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int default: assert(0); } - assert(retop=op; rule->rule_id=rule_id; rule->label_id=label_id; rule->timeout=timeout; assert(strlen(table_name)table_name)); - memset(rule->table_name, 0, sizeof(rule->table_name)); - memcpy(rule->table_name,table_name,strlen(table_name)); + strncpy(rule->table_name, table_name, sizeof(rule->table_name)); if(line!=NULL) { rule->table_line=_maat_strdup(line); @@ -836,35 +865,35 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _ struct Maat_group_t* group_cmd=NULL; struct Maat_region_t* region_cmd=NULL; - struct Maat_compile_inner_t *compile_inner=NULL; - struct Maat_group_inner_t* group_inner=NULL; - struct _Maat_region_inner_t* region_inner=NULL; + struct Maat_compile_group_relation *relation=NULL; + struct Maat_group_inner* group_inner=NULL; + struct Maat_region_inner* region_inner=NULL; void* logger=feather->logger; int config_id=cmd->compile.config_id; if(feather->scanner==NULL) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_command + MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_command ,"MAAT not ready."); return -1; } - compile_inner=(struct Maat_compile_inner_t *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id); - //Operation on compile_inner is thread safe, no immediate memory free when delete a compile rule or a scanner. - //In another words, if the compile_inner is accessable from compile means, its was valid in at least 10 seconds (garbage bury). - if(compile_inner==NULL) + relation=(struct Maat_compile_group_relation *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id); + //Operation on relation is thread safe, no immediate memory free when delete a compile rule or a scanner. + //In another words, if the relation is accessable from compile means, its was valid in at least 10 seconds (garbage bury). + if(relation==NULL) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_command + MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_command ,"config %d not exist." ,config_id); return -1; } - pthread_rwlock_rdlock(&(compile_inner->rwlock)); - cmd->group_num=compile_inner->group_cnt; + pthread_rwlock_rdlock(&(relation->rwlock)); + cmd->group_num=relation->group_cnt; assert(cmd->groups==NULL); - cmd->groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),cmd->group_num); - for(i=0;igroup_boundary;i++) + cmd->groups=ALLOC(struct Maat_group_t, cmd->group_num); + for(i=0;igroup_boundary;i++) { - group_inner=(struct Maat_group_inner_t*)dynamic_array_read(compile_inner->groups,i); + group_inner=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i); if(group_inner==NULL) { continue; @@ -872,15 +901,15 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _ group_cmd=&(cmd->groups[grp_idx]); group_cmd->group_id=group_inner->group_id; - if(group_inner->ref_cnt>1) + if(group_inner->ref_by_parent_cnt>1) { continue; } group_cmd->region_num=group_inner->region_cnt; - group_cmd->regions=(struct Maat_region_t*)calloc(sizeof(struct Maat_region_t),group_cmd->region_num); + group_cmd->regions=ALLOC(struct Maat_region_t, group_cmd->region_num); for(j=0;jregion_boundary;j++) { - region_inner=(struct _Maat_region_inner_t*)dynamic_array_read(group_inner->regions,j); + region_inner=(struct Maat_region_inner*)dynamic_array_read(group_inner->regions,j); if(region_inner==NULL) { continue; @@ -893,11 +922,11 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _ } grp_idx++; } - pthread_rwlock_unlock(&(compile_inner->rwlock)); + pthread_rwlock_unlock(&(relation->rwlock)); return 0; } -int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,struct serial_rule_t* list, int size) +int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd, struct serial_rule_t* list, int size) { struct Maat_group_t* p_group=NULL; struct Maat_region_t* p_region=NULL; @@ -915,19 +944,12 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st } if(op==MAAT_OP_ADD) { - snprintf(line,sizeof(line),"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t1\t%d",p_m_rule->config_id - ,p_m_rule->service_id - ,p_m_rule->action - ,p_m_rule->do_blacklist - ,p_m_rule->do_log - ,(_cmd->huge_service_defined!=NULL)?(_cmd->huge_service_defined):(p_m_rule->service_defined) - ,cmd->group_num); - set_serial_rule(list+rule_num,MAAT_OP_ADD,cmd->compile.config_id,cmd->label_id,feather->compile_tn,line,timeout); - + serialize_compile(p_m_rule, _cmd->huge_service_defined, cmd->group_num, MAAT_OP_ADD, line, sizeof(line)); + set_serial_rule(list+rule_num, MAAT_OP_ADD, cmd->compile.config_id, cmd->label_id, feather->compile_tn, line, timeout); } else { - set_serial_rule(list+rule_num,op,cmd->compile.config_id,cmd->label_id,feather->compile_tn,NULL,timeout); + set_serial_rule(list+rule_num, op, cmd->compile.config_id, cmd->label_id, feather->compile_tn, NULL, timeout); } rule_num++; for(i=0;igroup_num;i++) @@ -940,9 +962,10 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st p_group->group_id=feather->base_grp_seq; feather->base_grp_seq++; } - snprintf(line,sizeof(line),"%d\t%d\t1",p_group->group_id - ,p_m_rule->config_id); - set_serial_rule(list+rule_num,MAAT_OP_ADD,p_group->group_id,0,feather->group_tn,line,timeout); + p_group->parent_id=p_m_rule->config_id; + p_group->parent_type=PARENT_TYPE_COMPILE; + serialize_group(p_group, MAAT_OP_ADD, line, sizeof(line)); + set_serial_rule(list+rule_num, MAAT_OP_ADD, p_group->group_id, 0, feather->group_tn, line, timeout); } else { @@ -1358,10 +1381,10 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) struct Maat_region_t* p_region=NULL; enum MAAT_TABLE_TYPE table_type; - struct Maat_compile_inner_t *compile_rule=NULL; + struct Maat_compile_group_relation *compile_rule=NULL; if(feather->scanner!=NULL) { - compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id); + compile_rule=(struct Maat_compile_group_relation*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id); if(compile_rule!=NULL) { MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module @@ -2056,8 +2079,8 @@ int Maat_cmd_set_group(Maat_feather_t feather,int group_id, const struct Maat_re { return -1; } - //struct Maat_group_inner_t* group_inner=NULL; - //group_inner=(struct Maat_group_inner_t*)HASH_fetch_by_id(_feather->scanner->group_hash, group_id); + //struct Maat_group_inner* group_inner=NULL; + //group_inner=(struct Maat_group_inner*)HASH_fetch_by_id(_feather->scanner->group_hash, group_id); //NOT implemented yet. assert(0); return 0; @@ -2371,7 +2394,7 @@ int Maat_cmd_commit(Maat_feather_t feather) if(_feather->AUTO_NUMBERING_ON==1) { - data_reply=_wrap_redisCommand(write_ctx,"INCRBY SEQUENCE_REGION %d",new_region_num); + data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_region_id_var, new_region_num); if(data_reply->type!=REDIS_REPLY_INTEGER) { freeReplyObject(data_reply); @@ -2381,7 +2404,7 @@ int Maat_cmd_commit(Maat_feather_t feather) _feather->base_rgn_seq=data_reply->integer-new_region_num; freeReplyObject(data_reply); - data_reply=_wrap_redisCommand(write_ctx,"INCRBY SEQUENCE_GROUP %d",new_group_num); + data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_group_id_var, new_group_num); if(data_reply->type!=REDIS_REPLY_INTEGER) { freeReplyObject(data_reply); @@ -2445,17 +2468,30 @@ long long Maat_cmd_incrby(Maat_feather_t feather,const char* key, int increment) freeReplyObject(data_reply); return result; } +int Maat_cmd_get_new_group_id(Maat_feather_t feather) +{ + int group_id=0; + group_id=(int) Maat_cmd_incrby(feather, mr_group_id_var, 1); + return group_id; +} +int Maat_cmd_get_new_region_id(Maat_feather_t feather) +{ + int region_id=0; + region_id=(int) Maat_cmd_incrby(feather, mr_region_id_var, 1); + return region_id; +} + void Maat_cmd_key_free(struct Maat_cmd_key**keys, int size) { int i=0; struct Maat_cmd_key* p=*keys; - for(i=0; itable_name); p->table_name=NULL; p->rule_id=0; } - free(p); + free(*keys); *keys=NULL; return; } @@ -2547,9 +2583,9 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger) append_cmd_cnt++; redisAppendCommand(ctx,"SET MAAT_PRE_VER %lld",maat_redis_version); append_cmd_cnt++; - redisAppendCommand(ctx,"SET SEQUENCE_REGION 1",maat_redis_version); + redisAppendCommand(ctx,"SET %s 1", mr_region_id_var); append_cmd_cnt++; - redisAppendCommand(ctx,"SET SEQUENCE_GROUP 1",maat_redis_version); + redisAppendCommand(ctx,"SET %s 1", mr_group_id_var); append_cmd_cnt++; redisAppendCommand(ctx,"EXEC"); append_cmd_cnt++; @@ -2571,6 +2607,41 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger) } return redis_transaction_success; } +static int _Maat_command_set_one_line(struct _Maat_feather_t* _feather, enum MAAT_OPERATION op, int id, const char* table_name, const char* line) +{ + redisContext* write_ctx=get_redis_ctx_for_write(_feather); + _feather->server_time=redis_server_time(write_ctx); + struct serial_rule_t s_rule; + set_serial_rule(&s_rule, op, id, 0, table_name, line, 0); + int transaction_success=0; + transaction_success=exec_serial_rule(write_ctx, &s_rule, 1, _feather->server_time, _feather->logger); + empty_serial_rules(&s_rule); + return transaction_success; +} +int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num) +{ + struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather; + char line[MAX_TABLE_LINE_SIZE]; + serialize_compile(compile, huge_service_defined, group_num, op, line, sizeof(line)); + int ret=_Maat_command_set_one_line(_feather, op, compile->config_id, table_name, line); + return ret; +} +int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_region_t* region, int group_id) +{ + struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather; + char line[MAX_TABLE_LINE_SIZE]; + serialize_region(region, group_id, line, sizeof(line)); + int ret=_Maat_command_set_one_line(_feather, op, region->region_id, region->table_name, line); + return ret; +} +int Maat_command_raw_set_group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_group_t* group) +{ + struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather; + char line[MAX_TABLE_LINE_SIZE]; + serialize_group(group, op, line, sizeof(line)); + int ret=_Maat_command_set_one_line(_feather, op, group->group_id*1024*1204+group->parent_id, group->table_name, line); + return ret; +} int Maat_cmd_flushDB(Maat_feather_t feather) { _Maat_feather_t* _feather=(_Maat_feather_t*)feather; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 5f983d5..0dd5571 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -32,7 +32,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_6_20190222=1; +int MAAT_FRAME_VERSION_2_7_20190629=1; const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; @@ -65,7 +65,7 @@ int is_valid_match_method(enum MAAT_MATCH_METHOD match_method) } -iconv_t maat_iconv_open(struct Maat_scanner_t* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from) +iconv_t maat_iconv_open(struct Maat_scanner* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from) { const char *from_s=CHARSET_STRING[from]; const char *to_s=CHARSET_STRING[to]; @@ -87,7 +87,7 @@ iconv_t maat_iconv_open(struct Maat_scanner_t* scanner,enum MAAT_CHARSET to,enum return cd; } -int iconv_convert(struct Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen) +int iconv_convert(struct Maat_scanner* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen) { size_t ret; int copy_len=0; @@ -206,7 +206,7 @@ int uni2ascii(const char* fmt,const char* src, const int srclen, char* dst, cons } return j; } -int universal_charset_convert(struct Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen) +int universal_charset_convert(struct Maat_scanner* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen) { int ret=0; char* tmp_buff=NULL; @@ -506,7 +506,7 @@ int HASH_delete_by_id(MESA_htable_handle hash,int id) ret=MESA_htable_del(hash,(unsigned char*)&id, sizeof(id), NULL); return ret; } -MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc) +MAAT_RULE_EX_DATA rule_ex_data_new(const struct Maat_rule_head * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc) { MAAT_RULE_EX_DATA ad=NULL; struct Maat_rule_t rule; @@ -514,7 +514,7 @@ MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, c ex_desc->new_func(ex_desc->idx, &rule, srv_def, &ad, ex_desc->argl,ex_desc->argp); return ad; } -void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc) +void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc) { struct Maat_rule_t rule; fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); @@ -692,12 +692,22 @@ int read_table_description(struct Maat_table_desc** p_table_info,int num,const c char line[MAX_TABLE_LINE_SIZE]; int i=0,ret=0,table_cnt=0; char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; - MESA_htable_handle string2int_map=map_create(); + MESA_htable_handle string2int_map=NULL;; struct Maat_table_desc*p=NULL; struct Maat_table_desc*conj_table=NULL; + fp=fopen(table_info_path,"r"); + if(fp==NULL) + { + fprintf(stderr,"Maat read table info %s error.\n",table_info_path); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s failed: %s.\n", table_info_path, strerror(errno)); + return 0; + } + string2int_map=map_create(); map_register(string2int_map,"expr", TABLE_TYPE_EXPR); - map_register(string2int_map,"ip", TABLE_TYPE_IP); + map_register(string2int_map,"ip", TABLE_TYPE_IP); + map_register(string2int_map,"ip_plus", TABLE_TYPE_IP_PLUS); map_register(string2int_map,"compile", TABLE_TYPE_COMPILE); map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN); map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL); @@ -725,13 +735,7 @@ int read_table_description(struct Maat_table_desc** p_table_info,int num,const c map_register(string2int_map,"yes", 1); map_register(string2int_map,"no", 0); - fp=fopen(table_info_path,"r"); - if(fp==NULL) - { - fprintf(stderr,"Maat read table info %s error.\n",table_info_path); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s error.\n",table_info_path); - } + i=0; while(NULL!=fgets(line,sizeof(line),fp)) { @@ -758,7 +762,7 @@ int read_table_description(struct Maat_table_desc** p_table_info,int num,const c { MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, "Maat read table info %s line %d error:invalid table type.",table_info_path,i); - goto error_jump; + goto invalid_table; } switch(p->table_type) { @@ -770,7 +774,7 @@ int read_table_description(struct Maat_table_desc** p_table_info,int num,const c fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, "Maat read table info %s line %d error:illegal column.",table_info_path,i); - goto error_jump; + goto invalid_table; } break; case TABLE_TYPE_PLUGIN: @@ -780,7 +784,7 @@ int read_table_description(struct Maat_table_desc** p_table_info,int num,const c fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, "Maat read table info %s line %d error:illegal plugin info.",table_info_path,i); - goto error_jump; + goto invalid_table; } break; case TABLE_TYPE_COMPILE: @@ -803,32 +807,32 @@ int read_table_description(struct Maat_table_desc** p_table_info,int num,const c MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, "Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,num); - goto error_jump; + goto invalid_table; } if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table; { conj_table=p_table_info[p->table_id]; if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Maat read table info %s line %d error:reach tableid %d conjunction upper limit." ,table_info_path,i,p->table_id); - goto error_jump; + goto invalid_table; } memcpy(conj_table->table_name[conj_table->conj_cnt],p->table_name[0],MAX_TABLE_NAME_LEN); conj_table->conj_cnt++; - MESA_handle_runtime_log(logger, RLOG_LV_INFO,maat_module, + MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module, "Maat read table info %s:%d:conjunction %s with %s (id=%d,total=%d)." ,table_info_path,i,p->table_name[0] ,conj_table->table_name[0],conj_table->table_id,conj_table->conj_cnt); //use goto to free the conjunctioned table_info - goto error_jump; + goto invalid_table; } p_table_info[p->table_id]=p; table_cnt++; continue; -error_jump: +invalid_table: table_info_free(p); p=NULL; } @@ -836,66 +840,116 @@ error_jump: map_destroy(string2int_map); return table_cnt; } -struct Maat_group_inner_t* create_group_rule(int group_id) +struct Maat_group_inner* create_group_rule(int group_id, int table_id, struct Maat_scanner *scanner) { - struct Maat_group_inner_t* group=(struct Maat_group_inner_t*)malloc(sizeof(struct Maat_group_inner_t)); + int ret=0; + struct Maat_group_inner* group=ALLOC(struct Maat_group_inner, 1); group->group_id=group_id; group->region_cnt=0; group->region_boundary=0; - group->ref_cnt=0; + group->ref_by_parent_cnt=0; group->regions=dynamic_array_create(1,8); - group->compile_shortcut=NULL; - group->table_id=0; + group->table_id=table_id; group->group_name=NULL; + group->vertex_id=scanner->grp_vertex_id_generator++; + assert(igraph_vcount(&scanner->group_graph)==group->vertex_id); + igraph_add_vertices(&scanner->group_graph, 1, NULL); //Add 1 vertice. + ret=HASH_add_by_id(scanner->vertex_id2group, group->vertex_id, group); + assert(ret>0); + ret=HASH_add_by_id(scanner->group_hash, group_id, group); + assert(ret>0); pthread_mutex_init(&(group->mutex), NULL); return group; } -void _destroy_group_rule(struct Maat_group_inner_t* group) +void _destroy_group_rule(struct Maat_group_inner* group) { - dynamic_array_destroy(group->regions,free); + if(group->regions) dynamic_array_destroy(group->regions,free); group->region_cnt=0; group->region_boundary=0; group->regions=NULL; - group->ref_cnt=0; + group->ref_by_parent_cnt=0; group->group_id=-1; group->table_id=-1; free(group->group_name); group->group_name=NULL; + free(group->top_groups); + group->top_groups=NULL; pthread_mutex_destroy(&(group->mutex)); free(group); } -void destroy_group_rule(struct Maat_group_inner_t* group) -{ - - if(group->ref_cnt>0||group->region_cnt>0) - { - return; - } - _destroy_group_rule(group); +size_t print_igraph_vector(igraph_vector_t *v, char* buff, size_t sz) { + long int i; + int printed=0; + for (i=0; iref_by_parent_cnt--; + break; + case DESTROY_GROUP_BY_CHILD: + group_rule->ref_by_children_cnt--; + break; + default: + assert(0); + break; + } + igraph_vector_t v; + char buff[4096]; + if(group_rule->ref_by_parent_cnt==0&&group_rule->ref_by_children_cnt==0&&group_rule->region_cnt==0) + { + HASH_delete_by_id(scanner->group_hash, group_rule->group_id); + HASH_delete_by_id(scanner->vertex_id2group, group_rule->vertex_id); + igraph_vector_init(&v, 8); + igraph_neighbors(&scanner->group_graph, &v, group_rule->vertex_id, IGRAPH_ALL); + if(igraph_vector_size(&v)>0) + { + print_igraph_vector(&v, buff, sizeof(buff)); + MESA_handle_runtime_log(scanner->logger_ref, RLOG_LV_FATAL, maat_module, + "Del group %d exception, still reached by %s.", + group_rule->vertex_id, buff); + assert(0); + } + igraph_vector_destroy(&v); + //Calling _destroy_group_rule on garbage collection to free memory. + garbage_bagging(GARBAGE_GROUP_RULE, group_rule, scanner->tomb_ref); + + } +} +void make_group_set(struct Maat_compile_group_relation* relation, struct bool_expr* a_set, unsigned char *has_not) { int i=0,j=0; - a_set->user_tag=compile_rule; - struct Maat_group_inner_t*group=NULL; - assert(compile_rule->group_cnt<=MAX_ITEMS_PER_BOOL_EXPR); - for(i=0,j=0;igroup_boundary&&juser_tag=relation; + struct Maat_group_inner*group=NULL; + assert(relation->group_cnt<=MAX_ITEMS_PER_BOOL_EXPR); + for(i=0,j=0;igroup_boundary&&jgroups, i); + group=(struct Maat_group_inner*)dynamic_array_read(relation->groups, i); if(group==NULL) { continue; } a_set->items[j].item_id=group->group_id; - a_set->items[j].not_flag=compile_rule->not_flag[j]; + a_set->items[j].not_flag=relation->not_flag[j]; if(a_set->items[j].not_flag) { *has_not=1; } j++; } - assert(j==compile_rule->group_cnt); + assert(j==relation->group_cnt); a_set->item_num=j; } struct compile_walker @@ -906,23 +960,23 @@ struct compile_walker void walk_compile_hash(const uchar * key, uint size, void * data, void * user) { struct bool_expr* one_set=NULL; - struct Maat_compile_inner_t* compile_rule=(struct Maat_compile_inner_t*)data; + struct Maat_compile_group_relation* relation=(struct Maat_compile_group_relation*)data; struct compile_walker* walker=(struct compile_walker*)user; unsigned char has_not_flag=0; MESA_lqueue_head update_q=walker->update_q; - if(compile_rule->db_c_rule==NULL) + if(relation->compile==NULL) { return; } //make sure compile rule's each group has loadded. - if((compile_rule->group_cnt==compile_rule->db_c_rule->declare_grp_num - ||compile_rule->db_c_rule->declare_grp_num==0)//for compatible old version - &&compile_rule->group_cnt>0 - &&compile_rule->group_cnt!=compile_rule->not_group_cnt) + if((relation->group_cnt==relation->compile->declared_grp_num + || relation->compile->declared_grp_num==0)//for compatible old version + && relation->group_cnt>0 + && relation->group_cnt!=relation->not_group_cnt) { one_set=ALLOC(struct bool_expr, 1); //reading compile rule is safe in update thread, mutex lock called when modified - make_group_set(compile_rule, one_set, &has_not_flag); + make_group_set(relation, one_set, &has_not_flag); if(has_not_flag) { walker->compile_has_not_flag++; @@ -994,58 +1048,89 @@ void EMPTY_FREE(void*p) { return; } -struct Maat_compile_inner_t * create_compile_rule(int compile_id) + +struct Maat_compile_rule* create_compile_rule(struct Maat_rule_head* p_head, const char* service_define, int declared_grp_num, const struct Maat_table_desc* table) { - struct Maat_compile_inner_t* p=ALLOC(struct Maat_compile_inner_t,1); + int i=0; + struct Maat_compile_rule*p=ALLOC(struct Maat_compile_rule, 1); + p->head=*p_head; + p->declared_grp_num=declared_grp_num; + p->ads=ALLOC(MAAT_RULE_EX_DATA, MAX_COMPILE_EX_DATA_NUM); + + //protect by feather->background_update_mutex + p->ref_table=table; + p->head.serv_def_len=strlen(service_define)+1; + p->service_defined=ALLOC(char, p->head.serv_def_len); + memcpy(p->service_defined, service_define, p->head.serv_def_len); + + for(i=0; icompile.ex_data_num; i++) + { + p->ads[i]=rule_ex_data_new(&p->head, p->service_defined, table->compile.ex_desc+i); + } + p->is_valid=1; + return p; +} + +void destroy_compile_rule(struct Maat_compile_rule* compile_rule) +{ + int i=0; + const struct compile_table_desc* compile_desc= &(compile_rule->ref_table->compile); + + for(i=0; iex_data_num; i++) + { + rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ads+i, compile_desc->ex_desc+i); + compile_rule->ads[i]=NULL; + } + free(compile_rule->ads); + + compile_rule->is_valid=0; + compile_rule->declared_grp_num=-1; + free(compile_rule->service_defined); + compile_rule->service_defined=NULL; + free(compile_rule); + return; +} +struct Maat_compile_group_relation * create_compile_group_relation(int compile_id, struct Maat_scanner *scanner) +{ + int ret=0; + struct Maat_compile_group_relation* p=ALLOC(struct Maat_compile_group_relation, 1); + p->magic_num=COMPILE_RELATION_MAGIC; p->compile_id=compile_id; p->group_cnt=0; p->group_boundary=1; p->groups=dynamic_array_create(1, 1); - p->ads=ALLOC(MAAT_RULE_EX_DATA, MAX_COMPILE_EX_DATA_NUM); pthread_rwlock_init(&(p->rwlock), NULL); + ret=HASH_add_by_id(scanner->compile_hash, compile_id, p); + assert(ret>0); return p; } -void _destroy_compile_rule(struct Maat_compile_inner_t * compile_rule) + +void _destroy_compile_group_relation(struct Maat_compile_group_relation * cg_relation) { - const struct Maat_table_desc* table=compile_rule->ref_table; - const struct compile_table_desc* compile_desc=&(table->compile); - struct db_compile_rule_t* db_compile_rule=compile_rule->db_c_rule; - int i=0; - compile_rule->compile_id=-1; - dynamic_array_destroy(compile_rule->groups,NULL); + assert(cg_relation->magic_num==COMPILE_RELATION_MAGIC); + pthread_rwlock_wrlock(&(cg_relation->rwlock)); + cg_relation->compile_id=-1; + dynamic_array_destroy(cg_relation->groups, NULL); + pthread_rwlock_unlock(&(cg_relation->rwlock)); - - pthread_rwlock_wrlock(&(compile_rule->rwlock)); + pthread_rwlock_destroy(&(cg_relation->rwlock)); - if(db_compile_rule!=NULL) - { - for(i=0; iex_data_num; i++) - { - rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, compile_desc->ex_desc+i); - compile_rule->ads[i]=NULL; - } - - free(db_compile_rule->service_defined); - free(db_compile_rule); - compile_rule->db_c_rule=NULL; - } - free(compile_rule->ads); - pthread_rwlock_unlock(&(compile_rule->rwlock)); - - pthread_rwlock_destroy(&(compile_rule->rwlock)); - free(compile_rule); + free(cg_relation); } -void destroy_compile_rule(struct Maat_compile_inner_t * p) +void destroy_compile_group_relation(struct Maat_compile_group_relation * p, struct Maat_scanner *scanner) { int i=0; - UNUSED struct Maat_compile_inner_t* p_group=NULL; + UNUSED struct Maat_group_inner* p_group=NULL; assert(p->group_cnt==0); + assert(p->compile==NULL); for(i=0;igroup_boundary;i++) { - p_group=(struct Maat_compile_inner_t*)dynamic_array_read(p->groups,i); + p_group=(struct Maat_group_inner*)dynamic_array_read(p->groups, i); assert(p_group==NULL); } - _destroy_compile_rule(p); + assert(p->magic_num==COMPILE_RELATION_MAGIC); + HASH_delete_by_id(scanner->compile_hash, p->compile_id); + garbage_bagging(GARBAGE_COMPILE_GOURP_RELATION, p, scanner->tomb_ref); } scan_rule_t* create_rs_str_rule(unsigned int sub_type,enum MAAT_MATCH_METHOD match_method,int is_case_sensitive,const char* string,int len,int l_offset,int r_offset) { @@ -1107,7 +1192,7 @@ void destroy_rs_ip_rule(scan_rule_t* p) { free(p); } -scan_rule_t* create_rs_intval_rule(unsigned int sub_type,struct db_intval_rule_t *intval_rule) +scan_rule_t* create_rs_intval_rule(unsigned int sub_type,struct db_intval_rule *intval_rule) { scan_rule_t *p_rule=(scan_rule_t*)calloc(sizeof(scan_rule_t),1); p_rule->rule_type=RULETYPE_INT; @@ -1173,7 +1258,7 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule) return; } GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest, - short cfds_lvl,struct Maat_group_inner_t* tag) + short cfds_lvl,struct Maat_group_inner* tag) { GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1); int digest_len=0; @@ -1221,7 +1306,6 @@ struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table { table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, table_desc->plugin.ex_desc.key2index_func); - pthread_rwlock_init(&(table_rt->plugin.rwlock), NULL); } break; default: @@ -1274,7 +1358,6 @@ void table_runtime_free(struct Maat_table_runtime* p) { MESA_htable_destroy(p->plugin.key2ex_hash, NULL); } - break; default: break; } @@ -1288,7 +1371,7 @@ void table_runtime_free(struct Maat_table_runtime* p) return; } -struct Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t *feather) +struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *feather) { int scan_thread_num=feather->scan_thread_num; // int rs_scan_type=feather->rule_scan_type; @@ -1312,22 +1395,34 @@ struct Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t hargs.data_expire_with_condition = NULL; - struct Maat_scanner_t* scanner=NULL; - scanner=ALLOC(struct Maat_scanner_t, 1); + struct Maat_scanner* scanner=NULL; + scanner=ALLOC(struct Maat_scanner, 1); //Function Maat_cmd_append will access compile_hash in user thread. - hargs.thread_safe=1; + hargs.thread_safe=8; scanner->compile_hash=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(scanner->compile_hash,0); - hargs.thread_safe=1; + hargs.thread_safe=8; scanner->group_hash=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(scanner->group_hash,0); + scanner->vertex_id2group=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(scanner->vertex_id2group,0); + + hargs.thread_safe=8; + scanner->exprid_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(scanner->exprid_hash, 0); + + hargs.thread_safe=0; + hargs.data_free = free; scanner->region_hash=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(scanner->region_hash,0); + ret=igraph_empty(&scanner->group_graph, 0, IGRAPH_DIRECTED); + assert(ret==IGRAPH_SUCCESS); + scanner->district_map=map_create(); scanner->version=version; @@ -1352,6 +1447,7 @@ struct Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t rulescan_set_param(scanner->region,RULESCAN_REGEX_GROUP,NULL,0); } scanner->tomb_ref=feather->garbage_q; + scanner->logger_ref=feather->logger; scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num); for(i=0;iregion); - MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))_destroy_compile_rule); + MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))_destroy_compile_group_relation); MESA_htable_destroy(scanner->group_hash, (void (*)(void*))_destroy_group_rule); + MESA_htable_destroy(scanner->exprid_hash, NULL); MESA_htable_destroy(scanner->region_hash, NULL); + MESA_htable_destroy(scanner->vertex_id2group, NULL); + map_destroy(scanner->district_map); scanner->district_map=NULL; assert(scanner->tmp_district_map==NULL); @@ -1413,6 +1512,7 @@ void destroy_maat_scanner(struct Maat_scanner_t*scanner) { table_runtime_free(scanner->table_rt[i]); } + igraph_destroy(&scanner->group_graph); free(scanner); return; } @@ -1494,7 +1594,7 @@ void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_sta return; } -void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,void*logger,struct Maat_scanner_t* maat_scanner) +void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,void*logger,struct Maat_scanner* maat_scanner) { long i=0,data_size=0; unsigned int j=0; @@ -1580,6 +1680,7 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, assert(table_rt->expr.regex_rule_cnt>=0); break; case TABLE_TYPE_IP: + case TABLE_TYPE_IP_PLUS: table_rt->ip.ipv4_rule_cnt+=region_counter[i].ipv4_rule_cnt; table_rt->ip.ipv6_rule_cnt+=region_counter[i].ipv6_rule_cnt; break; @@ -1602,7 +1703,7 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, free(to_update_expr); } -void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*logger,struct Maat_scanner_t* maat_scanner,int table_id) +void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*logger,struct Maat_scanner* maat_scanner,int table_id) { long i=0,data_size=0; int ret=0; @@ -1648,115 +1749,169 @@ void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*log update_array=NULL; return; } -struct Maat_group_inner_t* add_region_to_group(struct Maat_group_inner_t* group,int table_id,int region_id,int district_id,int expr_id,enum MAAT_TABLE_TYPE region_type) +struct region_group_relation { - int i=0; - struct _Maat_region_inner_t* region_rule=NULL; - for(i=0;iregion_boundary;i++) + int region_id; + int group_id; + int array_idx; +}; +int region_group_relation_add(MESA_htable_handle region_hash, int region_id, int group_id, int array_idx) +{ + struct region_group_relation* relation=ALLOC(struct region_group_relation, 1); + relation->region_id=region_id; + relation->group_id=group_id; + relation->array_idx=array_idx; + int ret=HASH_add_by_id(region_hash, region_id, relation); + if(ret<0) { - region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group->regions, i); - if(region_rule==NULL) - { - continue; - } - if(region_rule->region_id==region_id) - { - break; - } + free(relation); + return -1; } - if(i==group->region_boundary)//new region + else { - region_rule=(struct _Maat_region_inner_t*)malloc(sizeof(struct _Maat_region_inner_t)); + return 0; + } +} +struct region_group_relation* region_group_relation_get(MESA_htable_handle region_hash, int region_id) +{ + struct region_group_relation* relation=NULL; + relation=(struct region_group_relation*)HASH_fetch_by_id(region_hash, region_id); + return relation; +} +int region_group_relation_del(MESA_htable_handle region_hash, int region_id) +{ + int ret=HASH_delete_by_id(region_hash,region_id); + if(ret==-1) + { + return -1; + } + else + { + return 0; + } +} +struct Maat_group_inner* add_region_to_group(struct Maat_group_inner* group,int table_id,int region_id,int district_id,int expr_id,enum MAAT_TABLE_TYPE region_type, struct Maat_scanner* scanner) +{ + struct Maat_region_inner* region_rule=NULL; + struct region_group_relation* relation=NULL; + relation=region_group_relation_get(scanner->region_hash, region_id); + int array_idx; + pthread_mutex_lock(&(group->mutex)); + if(relation==NULL) + { + region_rule=ALLOC(struct Maat_region_inner, 1); region_rule->region_id=region_id; region_rule->expr_id_cnt=1; region_rule->expr_id_ub=region_rule->expr_id_lb=expr_id; region_rule->district_id=district_id; region_rule->table_type=region_type; region_rule->table_id=table_id; - pthread_mutex_lock(&(group->mutex)); - dynamic_array_write(group->regions,group->region_boundary,region_rule); + + dynamic_array_write(group->regions, group->region_boundary, region_rule); + array_idx=group->region_boundary; + region_group_relation_add(scanner->region_hash, region_id, group->group_id, array_idx); group->region_cnt++; group->region_boundary++; - pthread_mutex_unlock(&(group->mutex)); } else { + assert(relation->group_id==group->group_id); + assert(relation->array_idxregion_boundary); + array_idx=relation->array_idx; + region_rule=(struct Maat_region_inner*)dynamic_array_read(group->regions, array_idx); assert(expr_id==region_rule->expr_id_ub+1); region_rule->expr_id_ub=expr_id; region_rule->expr_id_cnt++; } + HASH_add_by_id(scanner->exprid_hash, expr_id, (void*)(long long)array_idx); + pthread_mutex_unlock(&(group->mutex)); + return group; } -void cancel_last_region_from_group(struct Maat_group_inner_t* group,int region_id,int expr_id) +void cancel_last_region_from_group(struct Maat_group_inner* group,int region_id,int expr_id, struct Maat_scanner* scanner) { - struct _Maat_region_inner_t* region_rule=NULL; + struct Maat_region_inner* region_rule=NULL; + struct region_group_relation* relation=NULL; + relation=region_group_relation_get(scanner->region_hash, region_id); + assert(relation->group_id==group->group_id); + assert(relation->array_idx==group->region_boundary-1); + int array_idx=relation->array_idx; pthread_mutex_lock(&(group->mutex)); - region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group->regions,group->region_boundary-1); + region_rule=(struct Maat_region_inner*)dynamic_array_read(group->regions, array_idx); assert(region_rule->expr_id_ub==expr_id&®ion_rule->region_id==region_id); if(region_rule->expr_id_cnt==1) { free(region_rule); - dynamic_array_write(group->regions,group->region_boundary,NULL); + dynamic_array_write(group->regions, group->region_boundary, NULL); group->region_cnt--; group->region_boundary--; + relation=NULL; + region_group_relation_del(scanner->region_hash, region_id); } else { region_rule->expr_id_ub--; region_rule->expr_id_cnt--; } - pthread_mutex_unlock(&(group->mutex)); + HASH_delete_by_id(scanner->exprid_hash, expr_id); + pthread_mutex_unlock(&(group->mutex)); return; } -unsigned int del_region_from_group(struct Maat_group_inner_t* group,int region_id,unsigned int *output_expr_id,int output_size) +unsigned int del_region_from_group(struct Maat_group_inner* group,int region_id,unsigned int *output_expr_id, int output_size, struct Maat_scanner* scanner) { - int i=0,j=0; - struct _Maat_region_inner_t* region_rule=NULL; - pthread_mutex_lock(&(group->mutex)); - for(i=0;iregion_boundary;i++) - { - region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group->regions, i); - if(region_rule==NULL) + int i=0, j=0, ret=0; + struct Maat_region_inner* region_rule=NULL; + struct region_group_relation* relation=NULL; + relation=region_group_relation_get(scanner->region_hash, region_id); + if(relation) + { + pthread_mutex_lock(&(group->mutex)); + assert(relation->group_id==group->group_id); + region_rule=(struct Maat_region_inner*)dynamic_array_read(group->regions, relation->array_idx); + dynamic_array_write(group->regions, relation->array_idx, NULL); + for(i=0;iexpr_id_cnt;i++) { - continue; - } - if(region_rule->region_id==region_id) - { - dynamic_array_write(group->regions, i, NULL); - for(j=0;jexpr_id_cnt;j++) - { - output_expr_id[j]=region_rule->expr_id_lb+j; - assert(output_expr_id[j]>=0); - } - assert(j<=output_size); - region_rule->region_id=0; - free(region_rule); - region_rule=NULL; - group->region_cnt--; - break; + output_expr_id[i]=region_rule->expr_id_lb+i; + assert(output_expr_id[i]>=0); } + assert(i<=output_size); + region_rule->region_id=0; + free(region_rule); + region_rule=NULL; + group->region_cnt--; + assert(group->region_cnt>=0); + relation=NULL; + ret=region_group_relation_del(scanner->region_hash, region_id); + assert(ret==0); + pthread_mutex_unlock(&(group->mutex)); } - pthread_mutex_unlock(&(group->mutex)); - return j; + for(j=0; jexprid_hash, output_expr_id[j]); + assert(ret==0); + } + + return i; } -int add_group_to_compile(struct Maat_compile_inner_t*a_compile_rule,struct Maat_group_inner_t* a_rule_group, int not_flag) +int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maat_group_inner* a_rule_group, int not_flag) { int i=0,ret=-1; int write_pos=-1; - struct Maat_group_inner_t* p=NULL; - pthread_rwlock_wrlock(&(a_compile_rule->rwlock)); - if(a_compile_rule->db_c_rule!=NULL - &&a_compile_rule->group_cnt>=a_compile_rule->db_c_rule->declare_grp_num - &&a_compile_rule->db_c_rule->declare_grp_num!=0) + struct Maat_group_inner* p=NULL; + + pthread_rwlock_wrlock(&(relation->rwlock)); + if(relation->compile!=NULL + && relation->group_cnt>=relation->compile->declared_grp_num + && relation->compile->declared_grp_num!=0) { ret=-1; goto error_out; } - for(i=0;igroup_boundary;i++) + for(i=0;igroup_boundary;i++) { - p=(struct Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i); + p=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i); if(p==NULL) { write_pos=i; @@ -1770,114 +1925,104 @@ int add_group_to_compile(struct Maat_compile_inner_t*a_compile_rule,struct Maat_ } } } - if(write_pos<0&&a_compile_rule->group_boundary==MAX_EXPR_ITEM_NUM) + if(write_pos<0&&relation->group_boundary==MAX_EXPR_ITEM_NUM) { ret=-1; goto error_out; } if(write_pos<0) { - write_pos=a_compile_rule->group_boundary; - a_compile_rule->group_boundary++; + write_pos=relation->group_boundary; + relation->group_boundary++; } - dynamic_array_write(a_compile_rule->groups, write_pos, a_rule_group); + dynamic_array_write(relation->groups, write_pos, a_rule_group); if(not_flag) { - a_compile_rule->not_flag[write_pos]=1; - a_compile_rule->not_group_cnt++; + relation->not_flag[write_pos]=1; + relation->not_group_cnt++; } else { - a_compile_rule->not_flag[write_pos]=0; - } - a_compile_rule->group_cnt++; - a_rule_group->ref_cnt++; - //member group->compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish. - if(a_rule_group->ref_cnt==1&&a_compile_rule->group_cnt==1) - { - a_rule_group->compile_shortcut=a_compile_rule; - } - else - { - a_rule_group->compile_shortcut=NULL; - } - - //update group's shortcut when compile has more than one group. - if(a_compile_rule->group_cnt!=1) - { - for(i=0;igroup_boundary;i++) - { - p=(struct Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i); - if(p!=NULL) - { - p->compile_shortcut=NULL; - } - } + relation->not_flag[write_pos]=0; } + relation->group_cnt++; + a_rule_group->ref_by_parent_cnt++; ret=1; error_out: - pthread_rwlock_unlock(&(a_compile_rule->rwlock)); + pthread_rwlock_unlock(&(relation->rwlock)); return ret; } -struct Maat_group_inner_t* del_group_from_compile(struct Maat_compile_inner_t*a_compile_rule,int group_id) +struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relation*relation, int group_id) { int i=0; - struct Maat_group_inner_t* group_rule=NULL; - pthread_rwlock_wrlock(&(a_compile_rule->rwlock)); + struct Maat_group_inner* group_rule=NULL; + pthread_rwlock_wrlock(&(relation->rwlock)); for(i=0;igroups,i); + group_rule=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i); if(group_rule==NULL) { continue; } if(group_rule->group_id==group_id) { - group_rule->ref_cnt--; - dynamic_array_write(a_compile_rule->groups,i,NULL); - if(a_compile_rule->not_flag[i]==1) + dynamic_array_write(relation->groups,i,NULL); + if(relation->not_flag[i]==1) { - a_compile_rule->not_group_cnt--; - a_compile_rule->not_flag[i]=0; - } - a_compile_rule->group_cnt--; - pthread_rwlock_unlock(&(a_compile_rule->rwlock)); - return group_rule; + relation->not_group_cnt--; + relation->not_flag[i]=0; + } + relation->group_cnt--; + break; + } + else + { + group_rule=NULL; } } - pthread_rwlock_unlock(&(a_compile_rule->rwlock)); - return NULL; + pthread_rwlock_unlock(&(relation->rwlock)); + return group_rule; } -int MAAT_MAGIC=0xaaaa; -int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_name,int is_valid,void*logger) + +int sync_region(MESA_htable_handle region_hash,int region_id, int group_id, const char* table_name, int is_valid, void*logger) { - int ret=-1; + struct region_group_relation* relation=NULL; + relation=region_group_relation_get(region_hash, region_id); if(is_valid==TRUE) { - ret=HASH_add_by_id(region_hash,region_id,&MAAT_MAGIC); - if(ret<0) + if(relation) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "region id %d of table %s is not unique.",region_id,table_name); + "region id %d of table %s is already in group %d.", + region_id, table_name, relation->group_id); return -1; } } else { - ret=HASH_delete_by_id(region_hash,region_id); - if(ret==-1) + if(!relation) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "region delete error,id %d in table %s does not exisit." - ,region_id - ,table_name); + "region delete error, id %d in table %s does not exisit.", + region_id, table_name); return -1; } + else + { + if(group_id!=relation->group_id) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "region delete error, id %d in table %s is already in group %d, but cmd want to delete from group %d.", + region_id, table_name, + relation->group_id, group_id); + return -1; + } + } } return 1; } -int get_district_id(Maat_scanner_t *scanner,const char* district_str) +int get_district_id(Maat_scanner *scanner,const char* district_str) { int map_ret=0,district_id=-1; map_ret=map_str2int(scanner->district_map, district_str,&district_id); @@ -1897,7 +2042,7 @@ int get_district_id(Maat_scanner_t *scanner,const char* district_str) } return district_id; } -int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,struct Maat_scanner_t *scanner,void* logger) +int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,struct Maat_scanner *scanner,void* logger) { unsigned int i=0,j=0; char* p=NULL,*saveptr=NULL,*region_string=NULL; @@ -1905,7 +2050,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st int expr_id=0,district_id=-1; struct expr_table_desc* expr_desc=&(table->expr); scan_rule_t*p_rule=NULL; - struct Maat_group_inner_t* group_rule=NULL; + struct Maat_group_inner* group_rule=NULL; enum MAAT_CHARSET dst_charset=CHARSET_NONE; char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM]; int key_left_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1},key_right_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1}; @@ -1916,7 +2061,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st } int sub_expr_cnt=0; struct op_expr_t *op_expr=NULL; - struct Maat_group_inner_t* u_para=NULL; + struct Maat_group_inner* u_para=NULL; if(table->table_type==TABLE_TYPE_EXPR_PLUS) { @@ -1924,11 +2069,10 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st str_unescape(db_rule->district); district_id=get_district_id(scanner, db_rule->district); } - group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id); + group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id); if(group_rule==NULL) { - group_rule=create_group_rule(db_rule->group_id); - HASH_add_by_id(scanner->group_hash, db_rule->group_id, group_rule); + group_rule=create_group_rule(db_rule->group_id, 0, scanner); } switch(db_rule->expr_type) { @@ -1985,7 +2129,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st break; case EXPR_TYPE_REGEX://it's easy,no need to charset convert expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule,table->table_id,db_rule->region_id,district_id,expr_id,TABLE_TYPE_EXPR); + u_para=add_region_to_group(group_rule, table->table_id, db_rule->region_id, district_id, expr_id, TABLE_TYPE_EXPR, scanner); if(u_para==NULL) { return -1; @@ -2050,7 +2194,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st break; } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule,table->table_id, db_rule->region_id,district_id,expr_id, table->table_type); + u_para=add_region_to_group(group_rule,table->table_id, db_rule->region_id,district_id,expr_id, table->table_type, scanner); if(u_para==NULL)//duplicate { return -1; @@ -2114,7 +2258,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st op_expr->convert_failed>0) { scanner->dedup_expr_num++; - cancel_last_region_from_group(group_rule,db_rule->region_id,op_expr->p_expr->expr_id); + cancel_last_region_from_group(group_rule,db_rule->region_id,op_expr->p_expr->expr_id, scanner); destroy_op_expr(op_expr); //redeem expr_id scanner->exprid_generator--; @@ -2130,7 +2274,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st else { expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule, table->table_id, db_rule->region_id,district_id,expr_id, table->table_type); + u_para=add_region_to_group(group_rule, table->table_id, db_rule->region_id,district_id,expr_id, table->table_type, scanner); if(u_para==NULL) { return -1; @@ -2161,23 +2305,22 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st } return 0; } -int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,struct Maat_scanner_t *scanner,void* logger) +int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,struct Maat_scanner *scanner,void* logger) { - struct Maat_group_inner_t* group_rule=NULL; + struct Maat_group_inner* group_rule=NULL; scan_rule_t* p_rule=NULL; struct op_expr_t* op_expr=NULL; - struct Maat_group_inner_t* u_para=NULL; + struct Maat_group_inner* u_para=NULL; int expr_id=0,district_id=-1; - group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id); + group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id); if(group_rule==NULL) { - group_rule=create_group_rule(db_ip_rule->group_id); - HASH_add_by_id(scanner->group_hash, db_ip_rule->group_id, group_rule); + group_rule=create_group_rule(db_ip_rule->group_id, 0, scanner); } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule, table->table_id,db_ip_rule->region_id,district_id,expr_id,TABLE_TYPE_IP); + u_para=add_region_to_group(group_rule, table->table_id, db_ip_rule->region_id, district_id, expr_id, TABLE_TYPE_IP, scanner); if(u_para==NULL) { return -1; @@ -2193,22 +2336,21 @@ int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,st MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); return 0; } -int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule_t* intval_rule,struct Maat_scanner_t *scanner,void* logger) +int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule* intval_rule,struct Maat_scanner *scanner,void* logger) { - struct Maat_group_inner_t* group_rule=NULL; + struct Maat_group_inner* group_rule=NULL; scan_rule_t* p_rule=NULL; struct op_expr_t* op_expr=NULL; - struct Maat_group_inner_t* u_para=NULL; + struct Maat_group_inner* u_para=NULL; int expr_id=0,district_id=-1; - group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id); + group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id); if(group_rule==NULL) { - group_rule=create_group_rule(intval_rule->group_id); - HASH_add_by_id(scanner->group_hash, intval_rule->group_id, group_rule); + group_rule=create_group_rule(intval_rule->group_id, 0, scanner); } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule, table->table_id,intval_rule->region_id,district_id,expr_id,TABLE_TYPE_INTERVAL); + u_para=add_region_to_group(group_rule, table->table_id, intval_rule->region_id, district_id, expr_id, TABLE_TYPE_INTERVAL, scanner); if(u_para==NULL) { return -1; @@ -2224,22 +2366,21 @@ int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule_t* intva MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); return 0; } -int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule_t* db_digest_rule,struct Maat_scanner_t *scanner,void* logger) +int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule* db_digest_rule,struct Maat_scanner *scanner,void* logger) { - struct Maat_group_inner_t* group_rule=NULL; + struct Maat_group_inner* group_rule=NULL; GIE_digest_t* digest_rule=NULL; - struct Maat_group_inner_t* u_para=NULL; + struct Maat_group_inner* u_para=NULL; struct Maat_table_runtime * table_rt=scanner->table_rt[table->table_id]; int expr_id=0,district_id=-1; - group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); + group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); if(group_rule==NULL) { - group_rule=create_group_rule(db_digest_rule->group_id); - HASH_add_by_id(scanner->group_hash, db_digest_rule->group_id, group_rule); + group_rule=create_group_rule(db_digest_rule->group_id, 0, scanner); } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule,table->table_id,db_digest_rule->region_id,district_id,expr_id,TABLE_TYPE_DIGEST); + u_para=add_region_to_group(group_rule, table->table_id, db_digest_rule->region_id, district_id, expr_id, TABLE_TYPE_DIGEST, scanner); if(u_para==NULL) { return -1; @@ -2253,33 +2394,33 @@ int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule_t* db_di ,db_digest_rule->confidence_degree ,group_rule); MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*)); - scanner->gie_total_q_size++; + scanner->gie_update_q_size++; return 0; } -int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int rule_type,struct Maat_scanner_t *maat_scanner,void* logger) +int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int rule_type,struct Maat_scanner *maat_scanner,void* logger) { int i=0; unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0}; int expr_num=0; - struct Maat_group_inner_t* group_rule=NULL; + struct Maat_group_inner* group_rule=NULL; struct op_expr_t* op_expr=NULL; GIE_digest_t* digest_rule=NULL; - group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(maat_scanner->group_hash, group_id); + group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(maat_scanner->group_hash, group_id); if(group_rule==NULL) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,table %s group id %u not exist,while delete region id %d." + "update error, table %s group id %u not exist, while delete region id %d." ,table->table_name[table->updating_name] ,group_id ,region_id); return -1; } assert(group_id==group_rule->group_id); - expr_num=del_region_from_group(group_rule,region_id, expr_id, sizeof(expr_id)/sizeof(unsigned int)); + expr_num=del_region_from_group(group_rule,region_id, expr_id, sizeof(expr_id)/sizeof(unsigned int), maat_scanner); if(expr_num==0) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "region delete error,id %d table %s region not in group id %d." + "region delete error, id %d table %s region not in group id %d." ,region_id ,table->table_name[table->updating_name] ,group_id); @@ -2288,6 +2429,7 @@ int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int switch(table->table_type) { case TABLE_TYPE_IP: + case TABLE_TYPE_IP_PLUS: case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: case TABLE_TYPE_INTERVAL: @@ -2306,159 +2448,221 @@ int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int ,0 ,NULL); MESA_lqueue_join_tail(maat_scanner->table_rt[table->table_id]->similar.update_q,&digest_rule, sizeof(void*)); - maat_scanner->gie_total_q_size++; + maat_scanner->gie_update_q_size++; break; default: assert(0); break; } - if(group_rule->region_cnt==0&&group_rule->region_cnt==0) - { - HASH_delete_by_id(maat_scanner->group_hash,group_id); - garbage_bagging(GARBAGE_GROUP_RULE, group_rule, maat_scanner->tomb_ref); + destroy_group_rule(group_rule, DESTROY_GROUP_BY_REGION, maat_scanner); - } return 0; } -int add_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct Maat_scanner_t *scanner,void* logger) +int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner *scanner, void* logger) { - struct Maat_group_inner_t* group_rule=NULL; - struct Maat_compile_inner_t*compile_rule=NULL; + struct Maat_group_inner* group_rule=NULL, *parent_group=NULL; + struct Maat_compile_group_relation*compile_rule=NULL; int ret=0; - group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); + igraph_integer_t edge_id; + group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); if(group_rule==NULL) { - group_rule=create_group_rule(db_group_rule->group_id); - group_rule->table_id=table->table_id; - - ret=HASH_add_by_id(scanner->group_hash, db_group_rule->group_id, group_rule); - assert(ret>=0); + group_rule=create_group_rule(db_group_rule->group_id, table->table_id, scanner); } - compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id); - if(compile_rule==NULL) + if(db_group_rule->parent_type==PARENT_TYPE_GROUP) { - compile_rule=create_compile_rule(db_group_rule->compile_id); - ret=HASH_add_by_id(scanner->compile_hash, db_group_rule->compile_id, compile_rule); - assert(ret>=0); - } - ret=add_group_to_compile(compile_rule, group_rule, db_group_rule->not_flag); - if(ret<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, - "update error,add group: %s %d to compile rule %d error, compile rule is full or duplicate group." - ,table->table_name[table->updating_name] - ,db_group_rule->group_id - ,db_group_rule->compile_id); - return -1; - } - return 0; - -} -void del_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct Maat_scanner_t *scanner,void* logger) -{ - struct Maat_compile_inner_t*compile_rule=NULL; - struct Maat_group_inner_t* group_rule=NULL; - compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id); - if(compile_rule==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,delete %s group rule error : compile id %d does not exisit." - ,table->table_name[table->updating_name] - ,db_group_rule->compile_id); - return; - } - group_rule=del_group_from_compile(compile_rule, db_group_rule->group_id); - if(group_rule==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,delete %s group rule error : group id %d not in compile id %d." - ,table->table_name[table->updating_name] - ,db_group_rule->group_id - ,db_group_rule->compile_id); - return; - } - if(compile_rule->group_cnt==0&&compile_rule->is_valid==0) - { - HASH_delete_by_id(scanner->compile_hash, db_group_rule->compile_id); - garbage_bagging(GARBAGE_COMPILE_RULE, compile_rule, scanner->tomb_ref); - } - //Directly delete group id will not destroyp group_rule,it 'll be destroyed when delete this group's last region. - if(group_rule->ref_cnt==0&&group_rule->region_cnt==0) - { - //Directly delete table %s group id %d, do this when delete its last region. - } - return; -} -int add_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct Maat_scanner_t *scanner,void* logger) -{ - struct Maat_compile_inner_t *compile_rule=NULL; - struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head); - int i=0; - compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id); - if(compile_rule==NULL) - { - compile_rule=create_compile_rule(p_maat_rule_head->config_id); - HASH_add_by_id(scanner->compile_hash,p_maat_rule_head->config_id,compile_rule); + parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->parent_id); + if(parent_group==NULL) + { + parent_group=create_group_rule(db_group_rule->parent_id, table->table_id, scanner); + } + group_rule->ref_by_parent_cnt++; + parent_group->ref_by_children_cnt++; + ret=igraph_get_eid(&scanner->group_graph, &edge_id, group_rule->vertex_id, parent_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); + if(edge_id>0)//if the edge was not found and error is false, then -1 will be assigned to eid. + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "update error, add sub group: %s %d to group %d error, sub group exist.", + table->table_name[table->updating_name], + db_group_rule->group_id, + db_group_rule->parent_id); + return -1; + } + //igraph allow add multiple edges between two vetex, igraph_delete_edges removes one edge per call. + igraph_add_edge(&scanner->group_graph, group_rule->vertex_id, parent_group->vertex_id); } else { - if(compile_rule->db_c_rule!=NULL)//duplicate config + group_rule->has_compile_neighbors=1; + compile_rule=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id); + if(compile_rule==NULL) + { + compile_rule=create_compile_group_relation(db_group_rule->parent_id, scanner); + } + ret=add_group_to_compile(compile_rule, group_rule, db_group_rule->not_flag); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "update error,add group: %s %d to compile rule %d error, compile rule is full or duplicate group.", + table->table_name[table->updating_name], + db_group_rule->group_id, + db_group_rule->parent_id); + return -1; + } + } + scanner->to_update_group_cnt++; + return 1; +} +int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner *scanner, void* logger) +{ + struct Maat_compile_group_relation* relation=NULL; + struct Maat_group_inner* group_rule=NULL, *parent_group=NULL; + igraph_es_t es; + int ret=0; + igraph_integer_t edge_num_before=0, edge_num_after=0; + + + if(db_group_rule->parent_type==PARENT_TYPE_GROUP) + { + group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); + parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->parent_id); + if(group_rule==NULL) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module , + "update error, delete %s group %d from parent group %d error, target group not exisit.", + table->table_name[table->updating_name], + db_group_rule->group_id, + db_group_rule->parent_id); + return 0; + } + if(parent_group==NULL) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module , + "update error, delete %s group %d from parent group %d error, parent group not exisit.", + table->table_name[table->updating_name], + db_group_rule->group_id, + db_group_rule->parent_id); + return 0; + } + edge_num_before=igraph_ecount(&scanner->group_graph); + // The edges between the given pairs of vertices will be included in the edge selection. + //The vertex pairs must be given as the arguments of the function call, the third argument + //is the first vertex of the first edge, the fourth argument is the second vertex of the + //first edge, the fifth is the first vertex of the second edge and so on. The last element + //of the argument list must be -1 to denote the end of the argument list. + //https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small + ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group_rule->vertex_id, parent_group->vertex_id, -1); + assert(ret==IGRAPH_SUCCESS); + // ignore no such edge to abort(). + igraph_set_error_handler(igraph_error_handler_ignore); + ret=igraph_delete_edges(&scanner->group_graph, es); + edge_num_after=igraph_ecount(&scanner->group_graph); + + if(ret!=IGRAPH_SUCCESS||edge_num_before-edge_num_after!=1) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module , + "update error, delete %s group %d from parent group %d error, not such relation before.", + table->table_name[table->updating_name], + db_group_rule->group_id, + db_group_rule->parent_id); + } + igraph_es_destroy(&es); + destroy_group_rule(parent_group, DESTROY_GROUP_BY_CHILD, scanner); + } + else + { + relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id); + if(relation==NULL) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "update error, delete %s group %d form compile %d error, compile does not exist.", + table->table_name[table->updating_name], + db_group_rule->group_id, + db_group_rule->parent_id); + return 0; + } + group_rule=del_group_from_compile(relation, db_group_rule->group_id); + if(group_rule==NULL) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "update error, delete %s group %d from compile %d error, target group does not in compile.", + table->table_name[table->updating_name], + db_group_rule->group_id, + db_group_rule->parent_id); + return 0; + } + if(relation->group_cnt==0 && relation->compile==NULL) + { + destroy_compile_group_relation(relation, scanner); + } + } + destroy_group_rule(group_rule, DESTROY_GROUP_BY_PARENT, scanner); + scanner->to_update_group_cnt++; + return 1; +} +int add_compile_rule(struct Maat_table_desc* table, struct Maat_compile_rule* db_compile_rule, struct Maat_scanner *scanner, void* logger) +{ + struct Maat_compile_group_relation *cg_relation=NULL; + struct Maat_rule_head *p_maat_rule_head=&(db_compile_rule->head); + + cg_relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id); + if(cg_relation==NULL) + { + cg_relation=create_compile_group_relation(p_maat_rule_head->config_id, scanner); + } + else + { + if(cg_relation->compile!=NULL)//duplicate config { return -1; } } - - pthread_rwlock_wrlock(&(compile_rule->rwlock)); - compile_rule->ref_table=table; - compile_rule->db_c_rule=db_compile_rule; - for(i=0; icompile.ex_data_num; i++) - { - compile_rule->ads[i]=rule_ex_data_new(p_maat_rule_head, db_compile_rule->service_defined, table->compile.ex_desc+i); - } - compile_rule->is_valid=1; - pthread_rwlock_unlock(&(compile_rule->rwlock)); - + cg_relation->compile=db_compile_rule; + scanner->to_update_compile_cnt++; return 0; } -int del_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct Maat_scanner_t *scanner,void* logger) +int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_scanner *scanner, void* logger) { - struct Maat_compile_inner_t *compile_rule=NULL; - compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id); - if(compile_rule==NULL) + struct Maat_compile_group_relation *cg_relation=NULL; + cg_relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, compile_id); + if(cg_relation==NULL || cg_relation->compile==NULL) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,delete %s compile rule error : congfig id %d does not exisit." + "update error, delete %s compile rule error : compile id %d does not exist." ,table->table_name[table->updating_name] - ,db_compile_rule->m_rule_head.config_id); + ,compile_id); return -1; } - pthread_rwlock_wrlock(&(compile_rule->rwlock)); - compile_rule->is_valid=0; - pthread_rwlock_unlock(&(compile_rule->rwlock)); - if(compile_rule->group_cnt==0) + pthread_rwlock_wrlock(&(cg_relation->rwlock)); + garbage_bagging(GARBAGE_COMPILE_RULE, cg_relation->compile, scanner->tomb_ref); + cg_relation->compile=NULL; + pthread_rwlock_unlock(&(cg_relation->rwlock)); + + if(cg_relation->group_cnt==0&&cg_relation->compile==NULL) { - HASH_delete_by_id(scanner->compile_hash, compile_rule->compile_id); - garbage_bagging(GARBAGE_COMPILE_RULE,compile_rule, scanner->tomb_ref); + destroy_compile_group_relation(cg_relation, scanner); } + scanner->to_update_compile_cnt++; return 1; } -void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger) +void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger) { struct db_group_rule_t db_group_rule; struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; int ret=0; memset(&db_group_rule, 0, sizeof(db_group_rule)); - ret=sscanf(table_line,"%d\t%d\t%d\t%d", &(db_group_rule.group_id), - &(db_group_rule.compile_id), + ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%d", &(db_group_rule.group_id), + &(db_group_rule.parent_id), &(db_group_rule.is_valid), - &(db_group_rule.not_flag)); - if(ret!=3&&ret!=4) + &(db_group_rule.not_flag), + &(db_group_rule.parent_type)); + if(ret!=3&&ret!=4&&ret!=5) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "update error,invalid format of group table %s:%s", + "update error, invalid format of group table %s:%s", table->table_name[table->updating_name], table_line); table->udpate_err_cnt++; return; @@ -2467,13 +2671,22 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru { db_group_rule.not_flag=0; } + if(db_group_rule.parent_type==PARENT_TYPE_GROUP && db_group_rule.not_flag) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + "update error, invalid format of group table %s:%s not operation is forbidden for non-compile parent.", + table->table_name[table->updating_name], table_line); + table->udpate_err_cnt++; + return; + } if(db_group_rule.is_valid==FALSE) { - del_group_rule(table, &db_group_rule, scanner, logger); + ret=del_group_rule(table, &db_group_rule, scanner, logger); //leave no trace when compatible_group_update calling - if(table->table_type==TABLE_TYPE_GROUP) + if(table->table_type==TABLE_TYPE_GROUP&&ret==1) { table_rt->origin_rule_num--; + assert(table_rt->origin_rule_num>=0); if(db_group_rule.not_flag) { table_rt->group.not_flag_group--; @@ -2488,7 +2701,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , "duplicate config of group table %s group_id %d compile_id %d.", table->table_name[0], db_group_rule.group_id, - db_group_rule.compile_id); + db_group_rule.parent_id); } else @@ -2507,7 +2720,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru return; } -void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct Maat_scanner_t *scanner,void* logger) +void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct Maat_scanner *scanner,void* logger) { char virtual_group_line[256]; snprintf(virtual_group_line,sizeof(virtual_group_line), @@ -2515,7 +2728,7 @@ void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int com update_group_rule(table, virtual_group_line,scanner,logger); return; } -void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) { struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); int ret=0,db_hexbin=0,rule_type=0; @@ -2605,10 +2818,11 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc table->udpate_err_cnt++; goto error_out; } - ret=sync_region(scanner->region_hash - ,maat_str_rule->region_id - ,table->table_name[table->updating_name] - ,maat_str_rule->is_valid,logger); + ret=sync_region(scanner->region_hash, + maat_str_rule->region_id, + maat_str_rule->group_id, + table->table_name[table->updating_name], + maat_str_rule->is_valid,logger); if(ret<0) { table->udpate_err_cnt++; @@ -2676,99 +2890,310 @@ error_out: free(maat_str_rule); maat_str_rule=NULL; } -void update_ip_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger,int group_mode_on) +enum MAAT_IP_FORMAT { - struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); - char src_ip[40],mask_src_ip[40],dst_ip[40],mask_dst_ip[40]; - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; - unsigned short i_src_port,i_sport_mask,i_dst_port,i_dport_mask; - int protocol=0,direction=0; - int ret=0,rule_type=0; - int ret_array[8]={1},i=0; - ret=sscanf(table_line,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t%d" - ,&(ip_rule->region_id) - ,&(ip_rule->group_id) - ,&(ip_rule->addr_type) - ,src_ip - ,mask_src_ip - ,&i_src_port - ,&i_sport_mask - ,dst_ip - ,mask_dst_ip - ,&i_dst_port - ,&i_dport_mask - ,&protocol - ,&direction - ,&(ip_rule->is_valid)); - if(ret!=14||(ip_rule->addr_type!=4&&ip_rule->addr_type!=6) - ||protocol>65535||protocol<0 - ||(direction!=0&&direction!=1)) + FORMAT_RANGE, + FORMAT_MASK, + FORMAT_CIDR, + FORMAT_UNKNOWN +}; +enum MAAT_IP_FORMAT ip_format_str2int(const char* format) +{ + if(0==strcasecmp(format, "range")) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,invalid format of ip table %s:%s" - ,table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - goto error_out; + return FORMAT_RANGE; } - if(ip_rule->addr_type==4) + else if(0==strcasecmp(format, "mask")) { - ret_array[0]=inet_pton(AF_INET,src_ip,&(ip_rule->ipv4_rule.saddr)); - ip_rule->ipv4_rule.saddr=ntohl(ip_rule->ipv4_rule.saddr); - ret_array[1]=inet_pton(AF_INET,mask_src_ip,&(ip_rule->ipv4_rule.smask)); - ip_rule->ipv4_rule.smask=ntohl(ip_rule->ipv4_rule.smask); - - ret_array[2]=inet_pton(AF_INET,dst_ip,&(ip_rule->ipv4_rule.daddr)); - ip_rule->ipv4_rule.daddr=ntohl(ip_rule->ipv4_rule.daddr); - ret_array[3]=inet_pton(AF_INET,mask_dst_ip,&(ip_rule->ipv4_rule.dmask)); - ip_rule->ipv4_rule.dmask=ntohl(ip_rule->ipv4_rule.dmask); - - ip_rule->ipv4_rule.min_sport=i_src_port&i_sport_mask; - ip_rule->ipv4_rule.max_sport=(i_src_port&i_sport_mask)+(~i_sport_mask); - - ip_rule->ipv4_rule.min_dport=i_dst_port&i_dport_mask; - ip_rule->ipv4_rule.max_dport=(i_dst_port&i_dport_mask)+(~i_dport_mask); - - ip_rule->ipv4_rule.proto=protocol; - ip_rule->ipv4_rule.direction=direction; - rule_type=RULETYPE_IPv4; + return FORMAT_MASK; + } + else if(0==strcasecmp(format, "CIDR")) + { + return FORMAT_CIDR; } else { - ret_array[0]=inet_pton(AF_INET6,src_ip,&(ip_rule->ipv6_rule.saddr)); - ipv6_ntoh(ip_rule->ipv6_rule.saddr); - ret_array[1]=inet_pton(AF_INET6,mask_src_ip,&(ip_rule->ipv6_rule.smask)); - ipv6_ntoh(ip_rule->ipv6_rule.smask); + assert(0); + } + return FORMAT_UNKNOWN; +} +int ip_format2range(int ip_type, enum MAAT_IP_FORMAT format, const char* ip1, const char* ip2, unsigned int range_begin[], unsigned int range_end[]) +{ + unsigned int ipv4_addr=0, ipv4_mask=0, ipv4_range_end=0; + unsigned int ipv6_addr[4]={0}, ipv6_mask[4]={0}, ipv6_range_end[4]={0}; + int cidr=0, bit32=0; + int ret=0, i=0; + if(ip_type!=4 && ip_type!=6) + { + assert(0); + return -1; + } + if(ip_type==4) + { + ret=inet_pton(AF_INET, ip1, &ipv4_addr); + if(ret<=0) + { + return -1; + } + ipv4_addr=ntohl(ipv4_addr); + switch (format) + { + case FORMAT_RANGE: + range_begin[0]=ipv4_addr; + ret=inet_pton(AF_INET, ip2, &ipv4_range_end); + if(ret<=0) + { + return -1; + } + ipv4_range_end=ntohl(ipv4_range_end); + range_end[0]=ipv4_range_end; + break; + case FORMAT_MASK: + ret=inet_pton(AF_INET, ip2, &ipv4_mask); + if(ret<=0) + { + return -1; + } + ipv4_mask=ntohl(ipv4_mask); + range_begin[0]=ipv4_addr&ipv4_mask; + range_end[0]=ipv4_addr|~ipv4_mask; + break; + case FORMAT_CIDR: + cidr=atoi(ip2); + if(cidr>32||cidr<0) + { + return -1; + } + ipv4_mask = (0xFFFFFFFFUL << (32 - cidr)) & 0xFFFFFFFFUL; + range_begin[0]=ipv4_addr&ipv4_mask; + range_end[0]=ipv4_addr|~ipv4_mask; + break; + default: + assert(0); + } + } + else //ipv6 + { + ret=inet_pton(AF_INET6, ip1, ipv6_addr); + if(ret<=0) + { + return -1; + } + ipv6_ntoh(ipv6_addr); + switch(format) + { + case FORMAT_RANGE: + ret=inet_pton(AF_INET6, ip2, ipv6_range_end); + if(ret<=0) + { + return -1; + } + ipv6_ntoh(ipv6_range_end); + memcpy(range_begin, ipv6_addr, sizeof(ipv6_addr)); + memcpy(range_end, ipv6_range_end, sizeof(ipv6_range_end)); + break; + case FORMAT_MASK: + ret=inet_pton(AF_INET6, ip2, ipv6_mask); + if(ret<=0) + { + return -1; + } + ipv6_ntoh(ipv6_mask); + for(i=0; i<4; i++) + { + range_begin[i]=ipv6_addr[i]&ipv6_mask[i]; + range_end[i] = ipv6_addr[i]|~ipv6_mask[i]; + } + break; + case FORMAT_CIDR: + cidr=atoi(ip2); + if(cidr>128||cidr<0) + { + return -1; + } + for(i=0; i<4; i++) + { + bit32=128-cidr-32*(3-i); + if(bit32<0) bit32=0; + ipv6_mask[i]=(0xFFFFFFFFUL << bit32) & 0xFFFFFFFFUL; + range_begin[i]=ipv6_addr[i]&ipv6_mask[i]; + range_end[i] = ipv6_addr[i]|~ipv6_mask[i]; + } + break; + default: + assert(0); + } + } + return 0; +} +void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger, int group_mode_on) +{ + struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); + char src_ip1[40]={0}, src_ip2[40]={0}, dst_ip1[40]={0}, dst_ip2[40]={0}; + char saddr_format[16]={0}, sport_format[16]={0}, daddr_format[16]={0}, dport_format[16]={0}; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + unsigned short src_port1=0, src_port2=0, dst_port1=0, dst_port2=0; + int protocol=0,direction=0; + int ret=0; + int ret_array[8]={1},i=0; - ret_array[2]=inet_pton(AF_INET6,dst_ip,&(ip_rule->ipv6_rule.daddr)); - ipv6_ntoh(ip_rule->ipv6_rule.daddr); - ret_array[3]=inet_pton(AF_INET6,mask_dst_ip,&(ip_rule->ipv6_rule.dmask)); - ipv6_ntoh(ip_rule->ipv6_rule.dmask); - - ip_rule->ipv6_rule.min_sport=i_src_port&i_sport_mask; - ip_rule->ipv6_rule.max_sport=(i_src_port&i_sport_mask)+(~i_sport_mask); - - ip_rule->ipv6_rule.min_dport=i_dst_port&i_dport_mask; - ip_rule->ipv6_rule.max_dport=(i_dst_port&i_dport_mask)+~(i_dport_mask); + switch(table->table_type) + { + case TABLE_TYPE_IP: + strncpy(saddr_format, "mask", sizeof(saddr_format)); + strncpy(sport_format, "mask", sizeof(sport_format)); + strncpy(daddr_format, "mask", sizeof(daddr_format)); + strncpy(dport_format, "mask", sizeof(dport_format)); + + ret=sscanf(table_line,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t%d", + &(ip_rule->region_id), + &(ip_rule->group_id), + &(ip_rule->addr_type), + src_ip1, + src_ip2, + &src_port1, + &src_port2, + dst_ip1, + dst_ip2, + &dst_port1, + &dst_port2, + &protocol, + &direction, + &(ip_rule->is_valid)); + if(ret!=14) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error, invalid column number of ip table %s:%s" + ,table->table_name[table->updating_name],table_line); + table->udpate_err_cnt++; + goto error_out; + } + break; + case TABLE_TYPE_IP_PLUS: + ret=sscanf(table_line,"%d\t%d\t%d\t%s\t%s\t%s\t%s\t%hu\t%hu\t%s\t%s\t%s\t%s\t%hu\t%hu\t%d\t%d\t%d", + &(ip_rule->region_id), + &(ip_rule->group_id), + &(ip_rule->addr_type), + saddr_format, + src_ip1, + src_ip2, + sport_format, + &src_port1, + &src_port2, + daddr_format, + dst_ip1, + dst_ip2, + dport_format, + &dst_port1, + &dst_port2, + &protocol, + &direction, + &(ip_rule->is_valid)); + if(ret!=18) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error, invalid column number of ip_plus table %s:%s" + ,table->table_name[table->updating_name],table_line); + table->udpate_err_cnt++; + goto error_out; + } + break; + default: + table->udpate_err_cnt++; + goto error_out; + break; + } + if(ip_rule->addr_type!=4&&ip_rule->addr_type!=6) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, + "update error, invalid addr type %d of ip/ip_plus table %s:%s", + ip_rule->addr_type, + table->table_name[table->updating_name], table_line); + table->udpate_err_cnt++; + goto error_out; + } + if(protocol>65535 || protocol<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, + "update error, invalid protocol value %d of ip/ip_plus table %s:%s", + protocol, + table->table_name[table->updating_name],table_line); + table->udpate_err_cnt++; + goto error_out; + } + if(direction!=0 && direction!=1) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, + "update error, invalid direction value %d of ip/ip_plus table %s:%s", + direction, + table->table_name[table->updating_name],table_line); + table->udpate_err_cnt++; + goto error_out; + } + if(FORMAT_UNKNOWN==ip_format_str2int(saddr_format)|| + FORMAT_UNKNOWN==ip_format_str2int(sport_format)|| + FORMAT_UNKNOWN==ip_format_str2int(daddr_format)|| + FORMAT_UNKNOWN==ip_format_str2int(dport_format)) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, + "update error, invalid addr format of ip/ip_plus table %s:%s, should be range, mask or CIDR", + table->table_name[table->updating_name], table_line); + table->udpate_err_cnt++; + goto error_out; + } + + if(ip_rule->addr_type==4) + { + ret_array[0]=ip_format2range(ip_rule->addr_type, ip_format_str2int(saddr_format), src_ip1, src_ip2, &ip_rule->ipv4_rule.min_saddr, &ip_rule->ipv4_rule.max_saddr); + ret_array[1]=ip_format2range(ip_rule->addr_type, ip_format_str2int(daddr_format), dst_ip1, dst_ip2, &ip_rule->ipv4_rule.min_daddr, &ip_rule->ipv4_rule.max_daddr); + if(FORMAT_MASK==ip_format_str2int(dport_format)) + { + ip_rule->ipv4_rule.min_dport=dst_port1&dst_port2; + ip_rule->ipv4_rule.max_dport=dst_port1|~dst_port2; + } + else + { + ip_rule->ipv4_rule.min_dport=dst_port1; + ip_rule->ipv4_rule.max_dport=dst_port2; + } + ip_rule->ipv4_rule.proto=protocol; + ip_rule->ipv4_rule.direction=direction; + } + else + { + ret_array[0]=ip_format2range(ip_rule->addr_type, ip_format_str2int(saddr_format), src_ip1, src_ip2, ip_rule->ipv6_rule.min_saddr, ip_rule->ipv6_rule.max_saddr); + ret_array[1]=ip_format2range(ip_rule->addr_type, ip_format_str2int(daddr_format), dst_ip1, dst_ip2, ip_rule->ipv6_rule.min_daddr, ip_rule->ipv6_rule.max_daddr); + + if(FORMAT_MASK==ip_format_str2int(dport_format)) + { + ip_rule->ipv6_rule.min_dport=dst_port1&dst_port2; + ip_rule->ipv6_rule.max_dport=dst_port1|~dst_port2; + } + else + { + ip_rule->ipv6_rule.min_sport=dst_port1; + ip_rule->ipv6_rule.max_sport=dst_port2; + } ip_rule->ipv6_rule.proto=protocol; ip_rule->ipv6_rule.direction=direction; - rule_type=RULETYPE_IPv6; } for(i=0;i<4;i++) { - if(ret_array[i]<=0) + if(ret_array[i]<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,invalid format of ip table %s:%s" + "update error, invalid IP address format of ip table %s:%s" ,table->table_name[table->updating_name],table_line); table->udpate_err_cnt++; goto error_out; } } - ret=sync_region(scanner->region_hash - ,ip_rule->region_id - ,table->table_name[table->updating_name] - ,ip_rule->is_valid,logger); + ret=sync_region(scanner->region_hash, + ip_rule->region_id, + ip_rule->group_id, + table->table_name[table->updating_name], + ip_rule->is_valid,logger); if(ret<0) { table->udpate_err_cnt++; @@ -2776,19 +3201,19 @@ void update_ip_rule(struct Maat_table_desc* table,const char* table_line,struct } if(group_mode_on==FALSE)//for compatible old version { - compatible_group_udpate(table - ,ip_rule->region_id - ,ip_rule->group_id - ,ip_rule->is_valid - ,scanner - ,logger); + compatible_group_udpate(table, + ip_rule->region_id, + ip_rule->group_id, + ip_rule->is_valid, + scanner, + logger); ip_rule->group_id=ip_rule->region_id; } if(ip_rule->is_valid==FALSE) { - ret=del_region_rule(table - ,ip_rule->region_id,ip_rule->group_id,rule_type - ,scanner, logger); + ret=del_region_rule(table, + ip_rule->region_id, ip_rule->group_id, ip_rule->addr_type==6?RULETYPE_IPv6:RULETYPE_IPv4, + scanner, logger); if(ret<0) { table->udpate_err_cnt++; @@ -2801,7 +3226,7 @@ void update_ip_rule(struct Maat_table_desc* table,const char* table_line,struct else { - ret=add_ip_rule(table, ip_rule,scanner, logger); + ret=add_ip_rule(table, ip_rule, scanner, logger); if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , @@ -2820,9 +3245,9 @@ error_out: ip_rule=NULL; } -void update_intval_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_intval_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) { - struct db_intval_rule_t* intval_rule=ALLOC(struct db_intval_rule_t, 1); + struct db_intval_rule* intval_rule=ALLOC(struct db_intval_rule, 1); struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; int ret=0; ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id) @@ -2839,10 +3264,11 @@ void update_intval_rule(struct Maat_table_desc* table,const char* table_line,str table->udpate_err_cnt++; goto error_out; } - ret=sync_region(scanner->region_hash - ,intval_rule->region_id - ,table->table_name[table->updating_name] - ,intval_rule->is_valid,logger); + ret=sync_region(scanner->region_hash, + intval_rule->region_id, + intval_rule->group_id, + table->table_name[table->updating_name], + intval_rule->is_valid, logger); if(ret<0) { table->udpate_err_cnt++; @@ -2895,96 +3321,96 @@ error_out: intval_rule=NULL; } -void update_compile_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger) +void update_compile_rule(struct Maat_table_desc* table,const char* table_line ,struct Maat_scanner *scanner, const struct rule_tag* tags, int n_tags,void* logger) { struct compile_table_desc* compile_desc=&(table->compile); struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; - struct db_compile_rule_t *p_compile=ALLOC(struct db_compile_rule_t, 1); - struct _head_Maat_rule_t* p_m_rule=&(p_compile->m_rule_head); - char user_region[MAX_TABLE_LINE_SIZE]={0}; + struct Maat_compile_rule *p_compile=NULL; + struct Maat_rule_head m_rule_tmp; + memset(&m_rule_tmp, 0, sizeof(m_rule_tmp)); + + char service_define[MAX_TABLE_LINE_SIZE]={0}; char tag_str[MAX_TABLE_LINE_SIZE]={0}; int ret=0; - p_compile->declare_grp_num=0; - ret=sscanf(table_line,"%d\t%d\t%hhd\t%hhd\t%hhd\t%s\t%s\t%d\t%d",&(p_m_rule->config_id) - ,&(p_m_rule->service_id) - ,&(p_m_rule->action) - ,&(p_m_rule->do_blacklist) - ,&(p_m_rule->do_log) - ,tag_str - ,user_region - ,&(p_compile->is_valid) - ,&(p_compile->declare_grp_num)); - if((ret!=8&&ret!=9)||p_compile->declare_grp_num>MAAT_MAX_EXPR_ITEM_NUM) + int is_valid=0, declared_grp_num=0; + ret=sscanf(table_line,"%d\t%d\t%hhd\t%hhd\t%hhd\t%s\t%s\t%d\t%d",&(m_rule_tmp.config_id), + &(m_rule_tmp.service_id), + &(m_rule_tmp.action), + &(m_rule_tmp.do_blacklist), + &(m_rule_tmp.do_log), + tag_str, + service_define, + &is_valid, + &declared_grp_num); + if((ret!=8&&ret!=9)||declared_grp_num>MAAT_MAX_EXPR_ITEM_NUM) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,invalid format of compile table %s:%s" + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module , + "update error, invalid format of compile table %s:%s" ,table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - goto no_save; + goto error_out; } if(n_tags>0&&strlen(tag_str)>2) { ret=compare_accept_tag(tag_str, tags, n_tags); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,invalid tag format of compile table %s:%s" + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module , + "update error, invalid tag format of compile table %s:%s" ,table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - goto no_save; + goto error_out; } if(ret==0) { table->unmatch_tag_cnt++; - goto no_save; + return; } } switch(compile_desc->user_region_encoding) { case USER_REGION_ENCODE_ESCAPE: - str_unescape(user_region); + str_unescape(service_define); break; default: break; } - p_m_rule->serv_def_len=strlen(user_region)+1; - p_compile->service_defined= ALLOC(char, p_m_rule->serv_def_len); - memcpy(p_compile->service_defined,user_region,p_m_rule->serv_def_len); - if(p_compile->is_valid==FALSE) + if(is_valid==FALSE) { - ret=del_compile_rule(table,p_compile,scanner, logger); - table_rt->origin_rule_num--; - goto no_save; + ret=del_compile_rule(table, m_rule_tmp.config_id, scanner, logger); + if(ret>0) + { + table_rt->origin_rule_num--; + } + goto error_out; } else { - ret=add_compile_rule(table, p_compile, scanner,logger); + p_compile=create_compile_rule(&m_rule_tmp, service_define, declared_grp_num, table); + ret=add_compile_rule(table, p_compile, scanner, logger); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "duplicate config of compile table %s config_id=%d" - ,table->table_name[table->updating_name],p_m_rule->config_id); - table->udpate_err_cnt++; - goto no_save; + MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module, + "duplicate config of compile table %s config_id=%d", + table->table_name[table->updating_name], m_rule_tmp.config_id); + table->udpate_err_cnt++; + destroy_compile_rule(p_compile); + p_compile=NULL; + goto error_out; } table_rt->origin_rule_num++; } return; -no_save: - free(p_compile->service_defined); - p_compile->service_defined=NULL; - free(p_compile); - p_compile=NULL; +error_out: + table->udpate_err_cnt++; return; } -void update_digest_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_digest_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) { struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; - struct db_digest_rule_t* digest_rule=ALLOC(struct db_digest_rule_t, 1); + struct db_digest_rule* digest_rule=ALLOC(struct db_digest_rule, 1); int ret=0; char digest_buff[MAX_TABLE_LINE_SIZE]={'\0'}; if(table->table_type==TABLE_TYPE_DIGEST) @@ -3018,10 +3444,11 @@ void update_digest_rule(struct Maat_table_desc* table,const char* table_line,str table->udpate_err_cnt++; goto error_out; } - ret=sync_region(scanner->region_hash - ,digest_rule->region_id - ,table->table_name[table->updating_name] - ,digest_rule->is_valid,logger); + ret=sync_region(scanner->region_hash, + digest_rule->region_id, + digest_rule->group_id, + table->table_name[table->updating_name], + digest_rule->is_valid, logger); if(ret<0) { table->udpate_err_cnt++; @@ -3127,10 +3554,13 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger) switch(bag->type) { case GARBAGE_COMPILE_RULE: - destroy_compile_rule(bag->compile_rule); + destroy_compile_rule(bag->compile_rule); + break; + case GARBAGE_COMPILE_GOURP_RELATION: + _destroy_compile_group_relation(bag->compile_group_relation); break; case GARBAGE_GROUP_RULE: - destroy_group_rule(bag->group_rule); + _destroy_group_rule(bag->group_rule); break; case GARBAGE_SCANNER: ref_cnt=alignment_int64_array_sum(bag->scanner->ref_cnt, bag->scanner->max_thread_num); @@ -3185,7 +3615,7 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger) q_cnt,bury_cnt); } } -void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Maat_scanner_t* scanner, const struct rule_tag* tags, int n_tags, void* logger) +void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) { int i=0, ret=1, matched_tag=1; unsigned int len=strlen(table_line)+1; @@ -3240,8 +3670,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Ma if(plugin_desc->have_exdata) { ret=Maat_helper_read_column(table_line, plugin_desc->valid_flag_column, &is_valid_offset, &valid_len); - - pthread_rwlock_wrlock(&(table_rt->plugin.rwlock)); + //thread safe is protected by background_update_mutex if(atoi(table_line+is_valid_offset)==1) { plugin_EX_data_new(table, table_line, table_rt->plugin.key2ex_hash, logger); @@ -3250,7 +3679,6 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Ma { plugin_EX_data_free(table, table_line, table_rt->plugin.key2ex_hash, logger); } - pthread_rwlock_unlock(&(table_rt->plugin.rwlock)); } if(plugin_desc->cb_plug_cnt>0) { @@ -3270,7 +3698,67 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Ma table_rt->plugin.cache_line_num++; } } -void do_scanner_update(struct Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger) +void vector_print(igraph_vector_t *v) { + long int i; + for (i=0; igroup_graph), group_rule->vertex_id, IGRAPH_OUT, + 0, &vids, NULL, NULL, NULL, NULL, NULL, NULL); + + + long int i=0; + long long* temp_group_ids=ALLOC(long long, igraph_vector_size(&vids)); + size_t top_group_cnt=0; + for(i=0; ivertex_id2group, tmp_vid); + if(parent_group->has_compile_neighbors)//including itself + { + temp_group_ids[top_group_cnt]=parent_group->group_id; + top_group_cnt++; + } + } + pthread_mutex_lock(&(group_rule->mutex)); + free(group_rule->top_groups); + group_rule->top_group_cnt=top_group_cnt; + group_rule->top_groups=ALLOC(long long, group_rule->top_group_cnt); + memcpy(group_rule->top_groups, temp_group_ids, sizeof(long long)*group_rule->top_group_cnt); + if(group_rule->top_group_cnt>scanner->max_presented_top_group_cnt) + { + scanner->max_presented_top_group_cnt=group_rule->top_group_cnt; + scanner->most_popular_sub_group=group_rule->group_id; + } + pthread_mutex_unlock(&(group_rule->mutex)); + igraph_vector_destroy(&vids); + free(temp_group_ids); + temp_group_ids=NULL; + return; +} + +void find_group_paths(struct Maat_scanner* scanner) +{ + MESA_htable_iterate(scanner->group_hash, walk_group_hash, scanner); + return; +} +void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, int scan_thread_num, void* logger) { struct bool_matcher *tmp1=NULL,*tmp2=NULL; MESA_htable_handle tmp_map=NULL; @@ -3280,6 +3768,17 @@ void do_scanner_update(struct Maat_scanner_t* scanner,MESA_lqueue_head garbage_q GIE_create_para_t para; para.gram_value=7; para.position_accuracy=10; + igraph_bool_t is_dag; + igraph_is_dag(&(scanner->group_graph), &is_dag); + if(!is_dag) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Sub group cycle detected! Version %d", + scanner->version); + return; + } + find_group_paths(scanner); + tmp1=create_bool_matcher(scanner->compile_hash, scan_thread_num, logger); @@ -3292,7 +3791,11 @@ void do_scanner_update(struct Maat_scanner_t* scanner,MESA_lqueue_head garbage_q garbage_bagging(GARBAGE_BOOL_MATCHER, tmp2, garbage_q); } MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "Version %d dedup string rule %lu",scanner->version,scanner->dedup_expr_num); + "Version %d: dedup string rule %lu, sub group %d presents %d top groups", + scanner->version, + scanner->dedup_expr_num, + scanner->most_popular_sub_group, + scanner->max_presented_top_group_cnt); scanner->dedup_expr_num=0; rulescan_batch_update(scanner->region, scanner->region_update_q, @@ -3341,7 +3844,6 @@ void do_scanner_update(struct Maat_scanner_t* scanner,MESA_lqueue_head garbage_q } } - scanner->gie_total_q_size=0; if(scanner->tmp_district_map!=NULL) { tmp_map=scanner->district_map; @@ -3350,7 +3852,9 @@ void do_scanner_update(struct Maat_scanner_t* scanner,MESA_lqueue_head garbage_q garbage_bagging(GARBAGE_MAP_STR2INT, tmp_map, garbage_q); } scanner->last_update_time=time(NULL); - + scanner->gie_update_q_size=0; + scanner->to_update_group_cnt=0; + scanner->to_update_compile_cnt=0; return; } @@ -3400,7 +3904,7 @@ void maat_start_cb(long long new_version,int update_type,void*u_para) } return; } -long long scanner_rule_num(struct Maat_scanner_t *scanner) +long long scanner_rule_num(struct Maat_scanner *scanner) { long long total=0; struct Maat_table_runtime* table_rt=NULL; @@ -3471,7 +3975,7 @@ void maat_finish_cb(void* u_para) feather->scanner->cfg_num=scanner_rule_num(feather->scanner); feather->scanner->version=feather->maat_version; expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); - feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_total_q_size; + feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size; if(time(NULL)-feather->scanner->last_update_time>=feather->effect_interval_ms/1000) { do_scanner_update(feather->scanner @@ -3505,7 +4009,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; int ret=-1,i=0; int table_id=-1; - Maat_scanner_t* scanner=NULL; + Maat_scanner* scanner=NULL; struct Maat_table_desc* p_table=NULL; if(feather->update_tmp_scanner!=NULL) { @@ -3515,10 +4019,11 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) { scanner=feather->scanner; } +// MESA_handle_runtime_log(feather->logger, RLOG_LV_DEBUG, maat_module, "Maat table %s input: %s", table_name, line); ret=map_str2int(feather->map_tablename2id,table_name,&table_id); if(ret<0) { - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module ,"update warning,unknown table name %s",table_name); + MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s",table_name); return -1; } p_table=feather->p_table_info[table_id]; @@ -3538,6 +4043,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) update_expr_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); break; case TABLE_TYPE_IP: + case TABLE_TYPE_IP_PLUS: update_ip_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); break; case TABLE_TYPE_INTERVAL: @@ -3564,13 +4070,14 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) void *thread_rule_monitor(void *arg) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)arg; - struct Maat_scanner_t* old_scanner=NULL; + struct Maat_scanner* old_scanner=NULL; long expr_wait_q_cnt=0; int scan_dir_cnt=0; int ret=0; char md5_tmp[MD5_DIGEST_LENGTH*2+1]={0}; char tmp_dir[MAX_TABLE_NAME_LEN]={0}; struct stat attrib; + size_t total_wait_rule_cnt=0; char maat_name[16];//Defined by prctl: The name can be up to 16 bytes long,and should // be null terminated if it contains fewer bytes. @@ -3704,8 +4211,9 @@ void *thread_rule_monitor(void *arg) if(feather->scanner!=NULL) { expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); - feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_total_q_size; - if(feather->postpone_q_size>0&&time(NULL)-feather->scanner->last_update_time>=feather->effect_interval_ms/1000) + feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size; + total_wait_rule_cnt=feather->postpone_q_size+feather->scanner->to_update_compile_cnt+feather->scanner->to_update_group_cnt; + if(total_wait_rule_cnt>0&&time(NULL)-feather->scanner->last_update_time>=feather->effect_interval_ms/1000) { do_scanner_update(feather->scanner ,feather->garbage_q @@ -3745,7 +4253,6 @@ void *thread_rule_monitor(void *arg) alignment_int64_array_free(feather->inner_mid_cnt); alignment_int64_array_free(feather->outer_mid_cnt); alignment_int64_array_free(feather->hit_cnt); - alignment_int64_array_free(feather->orphan_group_saving); alignment_int64_array_free(feather->not_grp_hit_cnt); if(feather->input_mode==SOURCE_REDIS) { diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index c5e4daf..f0573eb 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -21,7 +21,6 @@ enum MAAT_FS_STATUS{ STATUS_ICONV_ERR_CNT, STATUS_SCAN_ERR_CNT, STATUS_ZOMBIE_RS_STREAM, - STATUS_ORPHAN_GROUP_SAVING, STATUS_NOT_GROUP_HIT, STATUS_CMD_NUM, STATUS_CMD_Q_SIZE, @@ -73,7 +72,6 @@ void maat_stat_init(struct _Maat_feather_t* feather) feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid"); feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"z_stream"); - feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"shortcut_sv"); feather->fs_status_id[STATUS_NOT_GROUP_HIT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"nt_grp_hit"); feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes"); @@ -168,7 +166,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0; long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0; long long outer_mid_cnt=0,inner_mid_cnt=0; - long long orphan_group_saving=0, not_grp_hit_cnt=0; + long long not_grp_hit_cnt=0; long long total_update_error=0,total_iconv_error=0; long long compile_rule_num=0, group_rule_num=0, not_group_rule_num=0, plugin_cache_num=0, plugin_acc_num=0; int i=0; @@ -183,7 +181,6 @@ void maat_stat_output(struct _Maat_feather_t* feather) active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num); inner_mid_cnt=alignment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num); - orphan_group_saving=alignment_int64_array_sum(feather->orphan_group_saving,feather->scan_thread_num); not_grp_hit_cnt=alignment_int64_array_sum(feather->not_grp_hit_cnt,feather->scan_thread_num); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version); @@ -191,7 +188,6 @@ void maat_stat_output(struct _Maat_feather_t* feather) FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING], 0,FS_OP_SET,orphan_group_saving); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0,FS_OP_SET,not_grp_hit_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_NUM], 0,FS_OP_SET,feather->cmd_acc_num); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_Q_SIZE], 0,FS_OP_SET,feather->cmd_q_cnt); @@ -234,6 +230,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) total_iconv_error=p_table->expr.iconv_err_cnt; break; case TABLE_TYPE_IP: + case TABLE_TYPE_IP_PLUS: table_regex_ipv6_num=table_rt->ip.ipv6_rule_cnt; break; default: diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 0a5bac8..5d45c85 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -83,7 +83,7 @@ static int get_group_seq(struct iris_description_t* iris_cfg) } else { - data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_GROUP 1"); + data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY %s 1", mr_group_id_var); sequence=(int)data_reply->integer-1; freeReplyObject(data_reply); } @@ -100,7 +100,7 @@ static int get_region_seq(struct iris_description_t* iris_cfg) } else { - data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_REGION 1"); + data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY %s 1", mr_region_id_var); sequence=(int)data_reply->integer-1; freeReplyObject(data_reply); } @@ -141,6 +141,7 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn, map_register(iris_cfg->str2int_map, "no",0); map_register(iris_cfg->str2int_map, "ip",TABLE_TYPE_IP); + map_register(iris_cfg->str2int_map, "ip_plus",TABLE_TYPE_IP_PLUS); map_register(iris_cfg->str2int_map, "string",TABLE_TYPE_EXPR); map_register(iris_cfg->str2int_map, "expr",TABLE_TYPE_EXPR); map_register(iris_cfg->str2int_map, "expr_plus",TABLE_TYPE_EXPR_PLUS); @@ -321,7 +322,7 @@ error_out: } return -1; } -int write_ip_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +int write_ip_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -407,7 +408,119 @@ int write_ip_rule(cJSON *region_json,struct iris_description_t *p_iris,const cha return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); } -int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,enum MAAT_TABLE_TYPE table_type,void * logger) +int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) +{ + struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; + int cmd_cnt=0; + memset(json_cmd,0,sizeof(json_cmd)); + + json_cmd[cmd_cnt].json_string="region_id"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="group_id"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="addr_type"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].str2int_flag=1; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="saddr_format"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_string="mask"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="src_ip1"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_string="0.0.0.0"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="src_ip2"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_string="255.255.255.255"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="sport_format"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_string="mask"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="src_port1"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_string="0"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="src_port2"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_string="65535"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="daddr_format"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_string="mask"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="dst_ip1"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_string="0.0.0.0"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="dst_ip2"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_string="255.255.255.255"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="dport_format"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_string="mask"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="dst_port1"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_string="0"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="dst_port2"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_string="65535"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="protocol"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_int=0; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="direction"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].str2int_flag=1; + json_cmd[cmd_cnt].empty_allowed=1; + json_cmd[cmd_cnt].default_string="double"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="is_valid"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + +} + +int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,enum MAAT_TABLE_TYPE table_type,void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -454,7 +567,7 @@ int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const c return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); } -int write_intval_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +int write_intval_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -483,7 +596,7 @@ int write_intval_rule(cJSON *region_json,struct iris_description_t *p_iris,const return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); } -int write_digest_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -516,7 +629,7 @@ int write_digest_rule(cJSON *region_json,struct iris_description_t *p_iris,const return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); } -int write_similar_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -546,7 +659,7 @@ int write_similar_rule(cJSON *region_json,struct iris_description_t *p_iris,cons } -int write_plugin_table(cJSON* plug_table_json,int sequence,iris_description_t* p_iris,void* logger) +int write_plugin_line(cJSON* plug_table_json,int sequence,iris_description_t* p_iris,void* logger) { cJSON* item=NULL,*table_content=NULL,*each_line=NULL; struct iris_table_t* table_info=NULL; @@ -665,19 +778,22 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - ret=write_expr_rule(table_content, p_iris, table_info->table_path,table_type, logger); + ret=write_expr_line(table_content, p_iris, table_info->table_path,table_type, logger); break; case TABLE_TYPE_IP: - ret=write_ip_rule(table_content, p_iris, table_info->table_path, logger); + ret=write_ip_line(table_content, p_iris, table_info->table_path, logger); + break; + case TABLE_TYPE_IP_PLUS: + write_ip_plus_line(table_content, p_iris, table_info->table_path, logger); break; case TABLE_TYPE_INTERVAL: - ret=write_intval_rule(table_content, p_iris, table_info->table_path, logger); + ret=write_intval_line(table_content, p_iris, table_info->table_path, logger); break; case TABLE_TYPE_DIGEST: - ret=write_digest_rule(table_content, p_iris, table_info->table_path, logger); + ret=write_digest_line(table_content, p_iris, table_info->table_path, logger); break; case TABLE_TYPE_SIMILARITY: - write_similar_rule(table_content, p_iris,table_info->table_path, logger); + write_similar_line(table_content, p_iris,table_info->table_path, logger); break; default: assert(0); @@ -691,7 +807,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri return ret; } -int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * logger) +int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void * logger) { int compile_id=-1,cmd_cnt=0,ret=-1; cJSON* item=NULL; @@ -772,7 +888,7 @@ int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * l set_file_rulenum(table_info->table_path,table_info->line_count,logger); return compile_id; } -int write_group_rule(int compile_id ,int group_id, int group_not_flag, struct iris_description_t *p_iris,void * logger) +int write_group_line(int group_id, int parent_id, int group_not_flag, int parent_type, struct iris_description_t *p_iris, void * logger) { FILE*fp=NULL; int ret=0; @@ -792,7 +908,7 @@ int write_group_rule(int compile_id ,int group_id, int group_not_flag, struct ir "fopen %s error %s.",p_iris->group_table->table_path,strerror(errno)); return -1; } - fprintf(fp,"%d\t%d\t1\t%d\n",group_id, compile_id, group_not_flag); + fprintf(fp,"%d\t%d\t1\t%d\t%d\n",group_id, parent_id, group_not_flag, parent_type); fclose(fp); p_iris->group_table->line_count++; ret=set_file_rulenum(p_iris->group_table->table_path,p_iris->group_table->line_count,logger); @@ -818,17 +934,104 @@ int write_index_file(struct iris_description_t *p_iris,void* logger) fclose(fp); return 0; } -int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger) +int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger) { - int i=0,j=0,k=0; - int compile_id=-1,compile_cnt=0,group_cnt=0,region_cnt=0,plug_table_cnt=0; + const char* _str_parent_type[2]={"compile", "group"}; + int i=0; + int sub_group_cnt=0, region_cnt=0; int ret=0; int group_not_flag=0; - cJSON *c_rules=NULL,*g_rules=NULL,*r_rules=NULL,*item=NULL,*plug_tables=NULL; - cJSON *compile_rule=NULL,*group_rule=NULL,*region_rule=NULL,*each_plug_table=NULL; + cJSON *region_json=NULL, *item=NULL; + cJSON *sub_groups=NULL, *region_rule=NULL; const char* group_name=NULL; struct group_info_t *group_info=NULL; struct group_info_t untitled_group; + + item=cJSON_GetObjectItem(group_json, "group_name"); + if(item==NULL||item->type!=cJSON_String) + { + group_name=untitled_group_name; + } + else + { + group_name=item->valuestring; + } + item=cJSON_GetObjectItem(group_json,"not_flag"); + if(item==NULL||item->type!=cJSON_Number) + { + group_not_flag=0; + } + else + { + group_not_flag=item->valueint; + } + if(parent_type==PARENT_TYPE_GROUP) + { + group_not_flag=0; + } + + group_info=(struct group_info_t*)MESA_htable_search(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name)); + if(group_info==NULL)//exist group name, region already read + { + if(0==strncasecmp(group_name, untitled_group_name, strlen(untitled_group_name))) + { + group_info=&untitled_group; + group_info->group_id=get_group_seq(p_iris); + } + else + { + group_info=ALLOC(struct group_info_t, 1); + group_info->group_id=get_group_seq(p_iris); + MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info); + } + } + ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, p_iris, logger); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "%s rule %d write group error.", _str_parent_type[parent_type], parent_id); + return -1; + } + region_json=cJSON_GetObjectItem(group_json,"regions"); + if(region_json!=NULL) + { + region_cnt=cJSON_GetArraySize(region_json); + for(i=0; igroup_id, p_iris, logger); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d write region error.", tracking_compile_id); + return -1; + } + } + } + sub_groups=cJSON_GetObjectItem(group_json,"sub_groups"); + if(sub_groups!=NULL) + { + //recursively + sub_group_cnt=cJSON_GetArraySize(sub_groups); + for(i=0; igroup_id, PARENT_TYPE_GROUP, tracking_compile_id, p_iris, logger); + if(ret<0) + { + return -1; + } + } + } + return 0; +} +int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) +{ + int i=0,j=0; + int compile_id=-1, compile_cnt=0, group_cnt=0, plug_table_cnt=0; + int ret=0; + cJSON *c_rules=NULL, *g_rules=NULL, *plug_tables=NULL; + cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL; plug_tables=cJSON_GetObjectItem(json,"plugin_table"); if(NULL!=plug_tables) { @@ -836,7 +1039,7 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger) for(i=0;itype!=cJSON_Number) - { - group_not_flag=0; - } - else - { - group_not_flag=item->valueint; - } - item=cJSON_GetObjectItem(group_rule,"group_name"); - if(item==NULL||item->type!=cJSON_String) - { - group_name=untitled_group_name; - } - else - { - group_name=item->valuestring; - } - group_info=(struct group_info_t*)MESA_htable_search(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name)); - if(group_info!=NULL)//exist group name ,region already read - { - ret=write_group_rule(compile_id, group_info->group_id, group_not_flag, p_iris, logger); - if(ret<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d write group error.",compile_id); - return -1; - } - r_rules=cJSON_GetObjectItem(group_rule,"regions"); - if(r_rules!=NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_json, - "compile rule %d's %s declared in previous compile rule, regions NOT take effect." - ,compile_id,group_name); - } - continue; - } - if(0==strncasecmp(group_name,untitled_group_name,strlen(untitled_group_name))) - { - group_info=&untitled_group; - group_info->group_id=get_group_seq(p_iris); - } - else - { - group_info=(struct group_info_t*)malloc(sizeof(struct group_info_t)); - group_info->group_id=get_group_seq(p_iris); - MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info); - } - r_rules=cJSON_GetObjectItem(group_rule,"regions"); - if(r_rules==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d's %s has no region.",compile_id,group_name); - return -1; - } - region_cnt=cJSON_GetArraySize(r_rules); - if(region_cnt<=0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d's %s has no region.",compile_id,group_name); - return -1; - } - for(k=0;kgroup_id, p_iris, logger); - if(ret<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d write region error.",compile_id); - return -1; - } - } - ret=write_group_rule(compile_id, group_info->group_id, group_not_flag, p_iris, logger); + ret=write_group_rule(group_rule, compile_id, PARENT_TYPE_COMPILE, compile_id, p_iris, logger); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d write group error.",compile_id); return -1; } - } } ret=write_index_file(p_iris,logger); @@ -971,6 +1098,7 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger) } return 0; } +// redis_write_ctx is used by maat_redis_tool to write json to redis. int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger) { FILE* json_fp=NULL; diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 22ef3fb..dcb31a0 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -12,7 +12,7 @@ #include "dynamic_array.h" #include "bool_matcher.h" #include "hiredis.h" - +#include "igraph/igraph.h" #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" #include "alignment_int64.h" @@ -23,6 +23,8 @@ extern const char *maat_module; +#define mr_region_id_var "SEQUENCE_REGION" +#define mr_group_id_var "SEQUENCE_GROUP" #define MAX_TABLE_NUM 256 @@ -69,14 +71,14 @@ struct db_ip_rule_t }; int is_valid; }; -struct db_intval_rule_t +struct db_intval_rule { int region_id; int group_id; interval_rule_t intval; int is_valid; }; -struct db_digest_rule_t +struct db_digest_rule { int region_id; int group_id; @@ -85,7 +87,7 @@ struct db_digest_rule_t short confidence_degree; int is_valid; }; -struct _head_Maat_rule_t +struct Maat_rule_head { int config_id; int service_id; @@ -95,20 +97,23 @@ struct _head_Maat_rule_t char resevered; int serv_def_len; }; -struct db_compile_rule_t +struct Maat_compile_rule { - struct _head_Maat_rule_t m_rule_head;// fix len of Maat_rule_t + struct Maat_rule_head head;// fix len of Maat_rule_t char* service_defined; - long long effective_range; int is_valid; - int declare_grp_num; + int declared_grp_num; + const struct Maat_table_desc* ref_table; + MAAT_RULE_EX_DATA* ads; }; + struct db_group_rule_t { int group_id; - int compile_id; + int parent_id; int is_valid; int not_flag; + int parent_type; //PARENT_TYPE_**, 0:compile, 1: group. }; struct op_expr_t { @@ -120,7 +125,7 @@ struct op_expr_t int rule_type; }; -struct _Maat_region_inner_t +struct Maat_region_inner { int region_id; int district_id; @@ -130,32 +135,38 @@ struct _Maat_region_inner_t int expr_id_ub; enum MAAT_TABLE_TYPE table_type; }; -struct Maat_group_inner_t +#define COMPILE_RELATION_MAGIC 0x1a2b3c4d +struct Maat_compile_group_relation +{ + long long magic_num; + struct Maat_compile_rule *compile; + dynamic_array_t *groups; //element is struct Maat_group_inner* + char not_flag[MAX_ITEMS_PER_BOOL_EXPR]; + int compile_id;//equal to compile->m_rule.config_id + int group_boundary; + int group_cnt; + int not_group_cnt; + pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read + +}; + +struct Maat_group_inner { int group_id; int table_id; int region_boundary; int region_cnt; - int ref_cnt; + int ref_by_parent_cnt; + int ref_by_children_cnt; char* group_name; + int has_compile_neighbors; + int vertex_id; + int top_group_cnt; + long long* top_groups; dynamic_array_t *regions; - void* compile_shortcut; pthread_mutex_t mutex; }; -struct Maat_compile_inner_t -{ - struct db_compile_rule_t *db_c_rule; - dynamic_array_t *groups; //element is struct Maat_group_inner_t* - char not_flag[MAX_ITEMS_PER_BOOL_EXPR]; - char is_valid; - int compile_id;//equal to db_c_rule->m_rule.config_id - const struct Maat_table_desc* ref_table; - int group_boundary; - int group_cnt; - int not_group_cnt; - MAAT_RULE_EX_DATA* ads; - pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read -}; + struct _compile_result_t { int compile_id; @@ -163,11 +174,11 @@ struct _compile_result_t }; struct _INNER_scan_status_t { - size_t cur_hit_cnt; - size_t hit_group_cnt; - size_t hit_group_size; - unsigned long long cur_hit_id[MAX_SCANNER_HIT_NUM]; - unsigned long long *hitted_group_id; + size_t cur_hit_group_cnt; + size_t all_hit_group_cnt; + size_t all_hit_group_array_sz; + struct dynamic_array_t* cur_hit_groups; + unsigned long long *all_hit_group_array; char not_grp_compile_hitted_flag; }; struct _OUTER_scan_status_t @@ -184,6 +195,7 @@ enum maat_garbage_type GARBAGE_SCANNER=0, GARBAGE_GROUP_RULE, GARBAGE_COMPILE_RULE, + GARBAGE_COMPILE_GOURP_RELATION, GARBAGE_BOOL_MATCHER, GARBAGE_MAP_STR2INT, GARBAGE_FOREIGN_FILE @@ -218,13 +230,13 @@ struct similar_runtime GIE_handle_t* gie_handle; MESA_lqueue_head update_q; }; + struct plugin_runtime { dynamic_array_t *cache_lines; long long cache_line_num; long long acc_line_num; long long cache_size; - pthread_rwlock_t rwlock; MESA_htable_handle key2ex_hash; }; struct expr_runtime @@ -252,7 +264,7 @@ struct Maat_table_runtime struct plugin_runtime plugin; struct expr_runtime expr; struct ip_runtime ip; - struct group_runtime group; + struct group_runtime group; void * other; }; mcore_long_t scan_cnt; @@ -266,21 +278,33 @@ struct rule_tag char* tag_name; char* tag_val; }; -struct Maat_scanner_t +struct Maat_scanner { long long version; time_t last_update_time; mcore_long_t ref_cnt; rule_scanner_t region; - long gie_total_q_size; + size_t gie_update_q_size; + size_t to_update_group_cnt; + size_t to_update_compile_cnt; struct Maat_table_runtime* table_rt[MAX_TABLE_NUM]; - MESA_htable_handle region_hash; - MESA_htable_handle group_hash; - MESA_htable_handle compile_hash; + MESA_htable_handle region_hash; //key: region_id, value: struct region_group_relation* + MESA_htable_handle exprid_hash; //key: expr_id, value: int array_idx of Maat_group_inner->regions; + MESA_htable_handle group_hash; //key: group_id, value: struct Maat_group_inner* + MESA_htable_handle compile_hash;//key: compile_id, value: struct Maat_compile_group_relation * + + MESA_htable_handle district_map; MESA_htable_handle tmp_district_map; + MESA_htable_handle vertex_id2group; + + igraph_t group_graph; + int grp_vertex_id_generator; + int most_popular_sub_group; + long long max_presented_top_group_cnt; + unsigned int district_num; unsigned int cfg_num; unsigned int exprid_generator; @@ -288,6 +312,7 @@ struct Maat_scanner_t MESA_lqueue_head region_update_q; struct bool_matcher * bool_matcher_expr_compiler; scan_result_t *region_rslt_buff; + void* logger_ref; MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q int max_thread_num; @@ -323,8 +348,8 @@ struct source_redis_ctx }; struct _Maat_feather_t { - struct Maat_scanner_t *scanner; - struct Maat_scanner_t *update_tmp_scanner; + struct Maat_scanner *scanner; + struct Maat_scanner *update_tmp_scanner; MESA_lqueue_head garbage_q; int table_cnt; int DEFERRED_LOAD_ON; @@ -390,7 +415,6 @@ struct _Maat_feather_t mcore_long_t inner_mid_cnt; mcore_long_t hit_cnt; mcore_long_t thread_call_cnt;//size indicate by scan_thread_num, - mcore_long_t orphan_group_saving; mcore_long_t not_grp_hit_cnt; long long total_scan_bytes; long long total_scan_cnt; @@ -411,9 +435,10 @@ struct _maat_garbage_t int ok_times; union { - struct Maat_scanner_t* scanner; - struct Maat_group_inner_t* group_rule; - struct Maat_compile_inner_t* compile_rule; + struct Maat_scanner* scanner; + struct Maat_group_inner* group_rule; + struct Maat_compile_rule* compile_rule; + struct Maat_compile_group_relation * compile_group_relation; struct bool_matcher* bool_matcher; void * raw; MESA_htable_handle str2int_map; @@ -443,7 +468,7 @@ int parse_accept_tag(const char* value, struct rule_tag** result, void* logger); void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); -void make_group_set(struct Maat_compile_inner_t* compile_rule, struct bool_expr* a_set, unsigned char *has_not); +void make_group_set(struct Maat_compile_group_relation* compile_rule, struct bool_expr* a_set, unsigned char *has_not); int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); void maat_start_cb(long long new_version,int update_type,void*u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para); @@ -466,9 +491,9 @@ int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_lis void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, int print_fn, void *logger); void rewrite_table_line_with_foreign(struct serial_rule_t*p); -void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len); -MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc); -void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc); +void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len); +MAAT_RULE_EX_DATA rule_ex_data_new(const struct Maat_rule_head * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc); +void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc); MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index); int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger); int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger); diff --git a/src/inc_internal/Maat_table_description.h b/src/inc_internal/Maat_table_description.h index 4e3fa3a..b924c34 100644 --- a/src/inc_internal/Maat_table_description.h +++ b/src/inc_internal/Maat_table_description.h @@ -18,6 +18,7 @@ enum MAAT_TABLE_TYPE { TABLE_TYPE_EXPR=0, TABLE_TYPE_IP, + TABLE_TYPE_IP_PLUS, TABLE_TYPE_INTERVAL, TABLE_TYPE_DIGEST, TABLE_TYPE_EXPR_PLUS, diff --git a/src/inc_internal/view_only/rulescan.h b/src/inc_internal/view_only/rulescan.h index f7a9a42..4397f18 100644 --- a/src/inc_internal/view_only/rulescan.h +++ b/src/inc_internal/view_only/rulescan.h @@ -28,7 +28,9 @@ extern "C" { RULESCAN_DETAIL_RESULT=1, /* ־λʾϸλõϢ, optvalΪNULLoptlenΪ0ĬϲϸϢ*/ RULESCAN_REGEX_GROUP =2, /* ־λʾʽƥķϢֶΣҪRULESCAN_DETAIL_RESULT־λ,optvalΪNULLoptlenΪ0ĬϲطϢ */ - RULESCAN_QUICK_SCAN /* Ҫɨģʽsub_type, ûԼ壬optvalֵΪ0-4096optlenΪ4Ĭͨɨģʽ*/ + RULEACAN_ERRLOG_CLOSE, /* ־λʾرRulescan־optvalΪNULLoptlenΪ0õĻĬϴRulescan־ */ + RULESCAN_ERRLOG_FILE_PATH, /* Rulescan־·ļû룬optvalֵΪļ־·optlenΪ·ȡû趨 + ־Ĭϴ洢ڿִгǰĿ¼µrulescan_tmp */ }; #define MAX_REGEX_GROUP_NUM 5 /* ʽֵ֧ĸ */ @@ -45,7 +47,7 @@ extern "C" const unsigned int RULETYPE_IPv6 = 4; /* IPv6 */ const unsigned int MAX_RULETYPE = 5; /* */ - const unsigned int MAX_SUB_RULETYPE = 4096; /* */ + const unsigned int MAX_SUB_RULETYPE = 4096; /* */ /* ַ͹򣨿ɱʾıַַʽ */ typedef struct _string_rule_t @@ -65,34 +67,34 @@ extern "C" unsigned int ub; /* ½磨ubĬΪ0 */ }interval_rule_t; - /* IPv4 */ + /* IPv4 */ typedef struct _ipv4_rule_t { - unsigned int saddr; /* ԴIPַ0ʾԱֶ */ - unsigned int smask; /* ԴIPַ룻0ʾ̶IP=saddr */ - unsigned int daddr; /* ĿIPַ0ʾԱֶ */ - unsigned int dmask; /* ĿIPַ룻0ʾ̶IP=daddr */ - unsigned short int min_sport; /* Դ˿ڷΧ½磻0ʾԱֶ */ - unsigned short int max_sport; /* Դ˿ڷΧϽ磻0ʾ̶˿=min_sport */ - unsigned short int min_dport; /* ĿĶ˿ڷΧ½磻0ʾԱֶ */ - unsigned short int max_dport; /* ĿĶ˿ڷΧϽ磻0ʾ̶˿=min_dport */ - unsigned short int proto; /* Э飬6ʾTCP17ʾUDP0ʾԱֶ */ - unsigned short int direction; /* 0ʾ˫1ʾ */ + unsigned int min_saddr; /* Դַ½磻0ʾԱֶ */ + unsigned int max_saddr; /* ԴַϽ磻0ʾ̶IP=min_saddr */ + unsigned int min_daddr; /* Ŀĵַ½磻0ʾԱֶ */ + unsigned int max_daddr; /* ĿĵַϽ磻0ʾ̶IP=min_daddr */ + unsigned short min_sport; /* Դ˿ڷΧ½磻0ʾԱֶ */ + unsigned short max_sport; /* Դ˿ڷΧϽ磻0ʾ̶˿=min_sport */ + unsigned short min_dport; /* ĿĶ˿ڷΧ½磻0ʾԱֶ */ + unsigned short max_dport; /* ĿĶ˿ڷΧϽ磻0ʾ̶˿=min_dport */ + unsigned short proto; /* Э飬6ʾTCP17ʾUDP0ʾԱֶ */ + unsigned short direction; /* 0ʾ˫1ʾ */ }ipv4_rule_t; - /* IPv6 */ + /* IPv6 */ typedef struct _ipv6_rule_t { - unsigned int saddr[4]; /* ԴIPַ0ʾԱֶ */ - unsigned int smask[4]; /* ԴIPַ룻0ʾ̶IP=saddr */ - unsigned int daddr[4]; /* ĿIPַ0ʾԱֶ */ - unsigned int dmask[4]; /* ĿIPַ룻0ʾ̶IP=daddr */ - unsigned short int min_sport; /* Դ˿ڷΧ½磻0ʾԱֶ */ - unsigned short int max_sport; /* Դ˿ڷΧϽ磻0ʾ̶˿=min_sport */ - unsigned short int min_dport; /* ĿĶ˿ڷΧ½磻0ʾԱֶ */ - unsigned short int max_dport; /* ĿĶ˿ڷΧϽ磻0ʾ̶˿=min_dport */ - unsigned short int proto; /* Э飬6ʾTCP17ʾUDPĬΪ0 */ - unsigned short int direction; /* 0ʾ˫1ʾ */ + unsigned int min_saddr[4]; /* Դַ½磻ȫ0ʾԱֶ */ + unsigned int max_saddr[4]; /* ԴַϽ磻ȫ0ʾ̶IP=min_saddr */ + unsigned int min_daddr[4]; /* Ŀĵַ½磻ȫ0ʾԱֶ */ + unsigned int max_daddr[4]; /* ĿĵַϽ磻ȫ0ʾ̶IP=min_daddr */ + unsigned short min_sport; /* Դ˿ڷΧ½磻0ʾԱֶ */ + unsigned short max_sport; /* Դ˿ڷΧϽ磻0ʾ̶˿=min_sport */ + unsigned short min_dport; /* ĿĶ˿ڷΧ½磻0ʾԱֶ */ + unsigned short max_dport; /* ĿĶ˿ڷΧϽ磻0ʾ̶˿=min_dport */ + unsigned short proto; /* Э飬6ʾTCP17ʾUDPĬΪ0 */ + unsigned short direction; /* 0ʾ˫1ʾ */ }ipv6_rule_t; /* ͨõĹ */ @@ -188,7 +190,7 @@ extern "C" unsigned int length[MAX_MATCH_POS_NUM]; /* ùнijȣнǰݰеģöӦlength=0, IP࣬Ϊ0*/ }rule_result_t; - /* ʽɨͣquickģʽֻexpr_idԼtagЧ */ + /* ʽɨ */ typedef struct _scan_result_t { unsigned int expr_id; /* ʽID */ diff --git a/src/version.map b/src/version.map index 7f11e6b..a17b821 100644 --- a/src/version.map +++ b/src/version.map @@ -1,6 +1,7 @@ VERS_2.4{ global: extern "C++" { + *MAAT_FRAME_VERSION_*; *Maat_*; *SFH_*; *GIE_*; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ef8db6a..02e8590 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,6 +1,9 @@ add_executable(maat_demo maat_demo.cpp) target_link_libraries(maat_demo maat_frame_shared) +add_executable(test_igraph test_igraph.cpp) +target_link_libraries(test_igraph igraph-static) + add_executable(test_maatframe test_maatframe.cpp) target_link_libraries(test_maatframe maat_frame_shared gtest) diff --git a/test/maat_json.json b/test/maat_json.json index d9dbbe6..ee03144 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -12,7 +12,7 @@ "is_valid": "yes", "groups": [ { - "group_name": "IP_group", + "group_name": "123_IP_group", "regions": [ { "table_name": "IP_CONFIG", @@ -77,7 +77,7 @@ "is_valid": "yes", "groups": [ { - "group_name": "IP_group" + "group_name": "123_IP_group" }, { "group_name": "Untitled", @@ -144,7 +144,7 @@ ] }, { - "group_name": "Untitled", + "group_name": "126_interval_group", "regions": [ { "table_name": "CONTENT_SIZE", @@ -555,7 +555,7 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "user_region": "anything", + "user_region": "StringScan.UTF8EncodedURL", "is_valid": "yes", "groups": [ { @@ -580,7 +580,7 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "user_region": "NOT-logic-test1", + "user_region": "NOTLogic.OneRegion", "is_valid": "yes", "groups": [ { @@ -595,7 +595,7 @@ "match_method": "sub", "format": "uncase plain" } - } + } ] }, { @@ -621,7 +621,7 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "user_region": "NOT-logic-test2", + "user_region": "NOTLogic.ScanNotAtLast", "is_valid": "yes", "groups": [ { @@ -662,7 +662,7 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "user_region": "NOT-logic-test3", + "user_region": "NOTLogic.ScanNotIP", "is_valid": "yes", "groups": [ { @@ -682,7 +682,7 @@ }, { "not_flag":1, - "group_name": "IP_group" + "group_name": "123_IP_group" } ] }, @@ -692,7 +692,7 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "user_region": "anything", + "user_region": "StringScan.Regex", "is_valid": "yes", "groups": [ { @@ -717,7 +717,7 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "user_region": "anything", + "user_region": "StringScan.UTF8EncodedURL", "is_valid": "yes", "groups": [ { @@ -744,7 +744,7 @@ "compile_id": 148, "service": 0, "do_blacklist": 0, - "user_region": "APP_ID=100001;BEHAV_ID=100002", + "user_region": "StringScan.ExprPlusWithOffset", "groups": [ { "regions": [ @@ -770,7 +770,7 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "user_region": "anything", + "user_region": "StringScan.Regex", "is_valid": "yes", "groups": [ { @@ -796,7 +796,7 @@ "do_blacklist": 0, "do_log": 0, "effective_rage": 0, - "user_region": "0", + "user_region": "StringScan.BugReport20190325", "is_valid": "yes", "groups": [ { @@ -838,7 +838,7 @@ "do_blacklist": 0, "do_log": 0, "effective_rage": 0, - "user_region": "suffix_hit", + "user_region": "StringScan.PrefixAndSuffix", "is_valid": "yes", "groups": [ { @@ -865,7 +865,7 @@ "do_blacklist": 0, "do_log": 0, "effective_rage": 0, - "user_region": "prefix_hit", + "user_region": "StringScan.PrefixAndSuffix", "is_valid": "yes", "groups": [ { @@ -879,9 +879,267 @@ "format": "uncase plain", "match_method": "prefix" } + }, + { + "table_type": "expr", + "table_name": "MAIL_ADDR", + "table_content": { + "keywords": "ceshi6@mailhost.cn", + "expr_type": "none", + "format": "uncase plain", + "match_method": "prefix" + } } ], - "group_name": "Untitled" + "group_name": "152_mail_addr" + }, + { + "group_name": "interval_group_refered", + "sub_groups": [ + {"group_name": "126_interval_group"} + ] + } + ] + }, + { + "compile_id": 153, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "Policy.SubGroup", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "expr", + "table_name": "MAIL_ADDR", + "table_content": { + "keywords": "ceshi4@mailhost.cn", + "expr_type": "none", + "format": "uncase plain", + "match_method": "prefix" + } + } + ], + "group_name": "Untitled", + "sub_groups": [ + {"group_name": "152_mail_addr"} + ], + "not_flag" : 0 + }, + { + "group_name": "IP_group_refered", + "sub_groups": [ + {"group_name": "123_IP_group"} + ] + } + ] + }, + { + "compile_id": 154, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ipv4_plus", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "range", + "src_ip1": "10.0.7.100", + "src_ip2": "10.0.7.106", + "sport_format": "range", + "src_port1": "5000", + "src_port2": "5001", + "daddr_format": "mask", + "dst_ip1": "123.56.104.218", + "dst_ip2": "255.255.255.0", + "dport_format": "range", + "dst_port1": "7400", + "dst_port2": "7400", + "protocol": 6, + "direction": "double" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 155, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ipv6_plus", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv6", + "saddr_format": "range", + "src_ip1": "1001:da8:205:1::101", + "src_ip2": "1001:da8:205:1::201", + "sport_format": "mask", + "src_port1": "5210", + "src_port2": "65520", + "daddr_format": "mask", + "dst_ip1": "3001:da8:205:1::401", + "dst_ip2": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:0000", + "dport_format": "mask", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 6, + "direction": "double" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 156, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "ExprPlusWithHex", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_REGION", + "table_type": "expr_plus", + "table_content": { + "district": "Content-Type", + "keywords": "2f68746d6c", + "expr_type": "none", + "match_method": "sub", + "format": "hexbin" + } + } + ] + } + ] + }, + { + "compile_id": 157, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "StringScan.StreamScanUTF8", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "expr", + "table_name": "TROJAN_PAYLOAD", + "table_content": { + "keywords": "我的订单", + "expr_type": "none", + "format": "none", + "match_method": "sub" + } + } + ] + } + ] + }, + { + "compile_id": 158, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "IPScan.IPv4_CIDR", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "CIDR", + "src_ip1": "192.168.0.1", + "src_ip2": "32", + "sport_format": "range", + "src_port1": "5210", + "src_port2": "65520", + "daddr_format": "CIDR", + "dst_ip1": "10.0.6.1", + "dst_ip2": "24", + "dport_format": "mask", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 6, + "direction": "double" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 159, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "IPScan.IPv6_CIDR", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv6", + "saddr_format": "CIDR", + "src_ip1": "2001:db8::", + "src_ip2": "120", + "sport_format": "mask", + "src_port1": "5210", + "src_port2": "65520", + "daddr_format": "CIDR", + "dst_ip1": "2001:4860:4860::8888", + "dst_ip2": "65", + "dport_format": "mask", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 6, + "direction": "double" + } + } + ], + "not_flag" : 0 } ] } diff --git a/test/table_info.conf b/test/table_info.conf index 6bace4d..5119ef6 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -1,7 +1,7 @@ #each collumn seperate with '\t' #id (0~65535) #name string -#type one of ip,expr,expr_plus,digest,intval,compile or plugin +#type one of ip, ip_plus, expr, expr_plus, digest, intval, compile or plugin #src_charset one of GBK,BIG5,UNICODE,UTF8 #dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/' #do_merege [yes/no] @@ -24,7 +24,7 @@ 5 CONTENT_SIZE intval -- 6 QD_ENTRY_INFO plugin 4 -- 7 FILE_DIGEST digest -- -8 HTTP_REGION expr_plus GBK GBK no 0 +8 HTTP_REGION expr_plus GBK GBK yes 0 9 SIM_URL similar -- 10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff 11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} -- @@ -34,4 +34,5 @@ 15 IR_INTERCEPT_IP plugin {"valid":14,"tag":18} 16 APP_PAYLOAD expr_plus UTF8 UTF8 yes 0 quickoff 17 TROJAN_PAYLOAD expr UTF8 UTF8 yes 0 quickoff -18 MAIL_ADDR expr UTF8 UTF8 yes 0 quickoff \ No newline at end of file +18 MAIL_ADDR expr UTF8 UTF8 yes 0 quickoff +19 IP_PLUS_CONFIG ip_plus -- \ No newline at end of file diff --git a/test/test_igraph.cpp b/test/test_igraph.cpp new file mode 100644 index 0000000..a4c4d24 --- /dev/null +++ b/test/test_igraph.cpp @@ -0,0 +1,62 @@ +#include "igraph/igraph.h" +#include + +void print_vector(igraph_vector_t *v, FILE *f) { + long int i; + for (i=0; i0); + + ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, v[0], v[1], -1); + assert(ret==IGRAPH_SUCCESS); + edge_num_before=igraph_ecount(&g); + ret=igraph_delete_edges(&g, es); + edge_num_after=igraph_ecount(&g); + assert(edge_num_before-edge_num_after==1); + assert(ret==IGRAPH_SUCCESS); + igraph_es_destroy(&es); + + ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, v[3], v[4], -1); + assert(ret==IGRAPH_SUCCESS); + edge_num_before=igraph_ecount(&g); + /* error test, no such edge to delete */ + igraph_set_error_handler(igraph_error_handler_ignore); + + ret=igraph_delete_edges(&g, es); + edge_num_after=igraph_ecount(&g); + assert(edge_num_before=edge_num_after); + assert(ret!=IGRAPH_SUCCESS); + igraph_es_destroy(&es); + + + igraph_destroy(&g); + + return 0; +} + diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index cdb545e..64f062e 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -42,14 +42,16 @@ void wait_for_cmd_effective(Maat_feather_t feather, long long version_before) { long long version_after=version_before; int is_updating=1; + long long wating_us=0, sleep_us=1000*100; while(is_updating||version_before==version_after) { Maat_read_state(feather,MAAT_STATE_IN_UPDATING, &is_updating, sizeof(is_updating)); Maat_read_state(feather,MAAT_STATE_VERSION, &version_after, sizeof(version_after)); - usleep(1000*100);//waiting for commands go into effect + usleep(sleep_us);//waiting for commands go into effect + wating_us+=sleep_us; } - +// printf("wait for %lld ms\n", wating_us/1000); } void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old) @@ -305,41 +307,48 @@ TEST(StringScan, BugReport20190325) EXPECT_EQ(result[0].config_id, 150); } +#define PrefixAndSuffixTest TEST(StringScan, PrefixAndSuffix) { int ret=0; - int table_id=0; + int mail_addr_table=0, cont_sz_table=0; struct Maat_rule_t result[4]; int found_pos[4]; - const char* table_name="MAIL_ADDR"; scan_status_t mid=NULL; const char* hit_twice="ceshi3@mailhost.cn"; const char* hit_suffix="11111111111ceshi3@mailhost.cn"; const char* hit_prefix="ceshi3@mailhost.cn11111111111"; - table_id=Maat_table_register(g_feather,table_name); - ASSERT_GT(table_id, 0); + - ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, hit_twice, strlen(hit_twice), + cont_sz_table=Maat_table_register(g_feather, "CONTENT_SIZE"); + ret=Maat_scan_intval(g_feather, cont_sz_table, 2015, + result, 4, &mid, 0); + mail_addr_table=Maat_table_register(g_feather, "MAIL_ADDR"); + ASSERT_GT(mail_addr_table, 0); + ret=Maat_full_scan_string(g_feather, mail_addr_table,CHARSET_GBK, hit_twice, strlen(hit_twice), result,found_pos, 4, &mid, 0); EXPECT_EQ(ret, 2); - EXPECT_EQ(result[0].config_id, 152); - EXPECT_EQ(result[1].config_id, 151); + EXPECT_EQ(result[0].config_id, 151);//compile has more groups is priority + EXPECT_EQ(result[1].config_id, 152); Maat_clean_status(&mid); - ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, hit_suffix, strlen(hit_suffix), + ret=Maat_full_scan_string(g_feather, mail_addr_table,CHARSET_GBK, hit_suffix, strlen(hit_suffix), result,found_pos, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 151); Maat_clean_status(&mid); - ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, hit_prefix, strlen(hit_prefix), + ret=Maat_scan_intval(g_feather, cont_sz_table, 2015, + result, 4, &mid, 0); + + ret=Maat_full_scan_string(g_feather, mail_addr_table,CHARSET_GBK, hit_prefix, strlen(hit_prefix), result,found_pos, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 152); Maat_clean_status(&mid); } - +#define ExprPlusTest 1 TEST(StringScan, ExprPlus) { int ret=0; @@ -363,6 +372,8 @@ TEST(StringScan, ExprPlus) &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 128); + + Maat_clean_status(&mid); return; } @@ -414,9 +425,35 @@ TEST(StringScan, ExprPlusWithOffset) &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 148); + + Maat_clean_status(&mid); return; } -TEST(IPScan, IPv4) + +TEST(StringScan, ExprPlusWithHex) +{ + int table_id=0,ret=0; + struct Maat_rule_t result[4]; + const char* scan_data="text/html; charset=UTF-8"; + const char* region_name="Content-Type"; + int found_pos[4]; + table_id=Maat_table_register(g_feather, "HTTP_REGION"); + ASSERT_GT(table_id, 0); + scan_status_t mid=NULL; + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, region_name, strlen(region_name)); + ASSERT_EQ(ret, 0); + ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), + result, found_pos, 4, + &mid, 0); + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 156); + Maat_clean_status(&mid); + + return; + +} +#define TEST_IPSCAN +TEST(IPScan, IPv4_mask) { int table_id=0,ret=0; const char* table_name="IP_CONFIG"; @@ -449,7 +486,7 @@ TEST(IPScan, IPv4) Maat_clean_status(&mid); return; } -TEST(IPScan, IPv6) +TEST(IPScan, IPv6_mask) { int table_id=0,ret=0; struct Maat_rule_t result[4]; @@ -474,6 +511,119 @@ TEST(IPScan, IPv6) Maat_clean_status(&mid); return; } +TEST(IPScan, IPv4_range) +{ + int table_id=0,ret=0; + const char* table_name="IP_PLUS_CONFIG"; + struct Maat_rule_t result[4]; + scan_status_t mid=NULL; + struct ipaddr ipv4_addr; + struct stream_tuple4_v4 v4_addr; + ipv4_addr.addrtype=ADDR_TYPE_IPV4; + inet_pton(AF_INET, "10.0.7.106", &(v4_addr.saddr)); + v4_addr.source=htons(5000); + inet_pton(AF_INET, "123.56.104.254", &(v4_addr.daddr)); + v4_addr.dest=htons(7400); + ipv4_addr.v4=&v4_addr; + + + table_id=Maat_table_register(g_feather, table_name); + + EXPECT_GT(table_id, 0); + + ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 154); + + Maat_clean_status(&mid); + return; +} + +TEST(IPScan, IPv6_range) +{ + int table_id=0,ret=0; + struct Maat_rule_t result[4]; + struct ipaddr ipv6_addr; + struct stream_tuple4_v6 v6_addr; + scan_status_t mid=NULL; + + ipv6_addr.addrtype=ADDR_TYPE_IPV6; + inet_pton(AF_INET6,"1001:da8:205:1::151",&(v6_addr.saddr)); + v6_addr.source=htons(5204);//5200~5299? + inet_pton(AF_INET6,"3001:da8:205:1::901",&(v6_addr.daddr)); + v6_addr.dest=htons(80);//any + ipv6_addr.v6=&v6_addr; + const char* table_name="IP_PLUS_CONFIG"; + table_id=Maat_table_register(g_feather,table_name); + EXPECT_GT(table_id, 0); + + //for improving performance. + Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION,NULL, 0); + ret=Maat_scan_proto_addr(g_feather, table_id, &ipv6_addr, 6, result,4, &mid, 0); + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 155); + Maat_clean_status(&mid); + return; + +} +TEST(IPScan, IPv4_CIDR) +{ + int table_id=0,ret=0; + const char* table_name="IP_PLUS_CONFIG"; + struct Maat_rule_t result[4]; + scan_status_t mid=NULL; + struct ipaddr ipv4_addr; + struct stream_tuple4_v4 v4_addr; + ipv4_addr.addrtype=ADDR_TYPE_IPV4; + inet_pton(AF_INET, "192.168.0.1", &(v4_addr.saddr)); + v4_addr.source=htons(5210); + inet_pton(AF_INET, "10.0.6.210", &(v4_addr.daddr)); + v4_addr.dest=htons(7400); + ipv4_addr.v4=&v4_addr; + + + table_id=Maat_table_register(g_feather, table_name); + + EXPECT_GT(table_id, 0); + + ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 158); + + Maat_clean_status(&mid); + return; +} +TEST(IPScan, IPv6_CIDR) +{ + int table_id=0,ret=0; + struct Maat_rule_t result[4]; + struct ipaddr ipv6_addr; + struct stream_tuple4_v6 v6_addr; + scan_status_t mid=NULL; + + ipv6_addr.addrtype=ADDR_TYPE_IPV6; + inet_pton(AF_INET6,"2001:db8::00fe",&(v6_addr.saddr)); + v6_addr.source=htons(5204);//5200~5299? + inet_pton(AF_INET6,"2001:4860:4860::8888",&(v6_addr.daddr)); + v6_addr.dest=htons(80);//any + ipv6_addr.v6=&v6_addr; + const char* table_name="IP_PLUS_CONFIG"; + table_id=Maat_table_register(g_feather,table_name); + EXPECT_GT(table_id, 0); + + //for improving performance. + Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION,NULL, 0); + ret=Maat_scan_proto_addr(g_feather, table_id, &ipv6_addr, 6, result,4, &mid, 0); + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 159); + Maat_clean_status(&mid); + return; + +} +#define TEST_NOTLogic 1 + TEST(NOTLogic, OneRegion) { const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-143."; @@ -489,15 +639,15 @@ TEST(NOTLogic, OneRegion) ASSERT_GT(table_id, 0); Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0); - ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_hit, strlen(string_should_hit), - result,found_pos, 4, &mid, 0); + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, string_should_hit, strlen(string_should_hit), + result, found_pos, 4, &mid, 0); EXPECT_GE(ret, 1); Maat_clean_status(&mid); Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0); - ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_not_hit, strlen(string_should_not_hit), - result,found_pos, 4, &mid, 0); + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, string_should_not_hit, strlen(string_should_not_hit), + result, found_pos, 4, &mid, 0); EXPECT_EQ(ret, -2); Maat_clean_status(&mid); @@ -713,7 +863,7 @@ TEST(StringScan, UTF8EncodedURL) ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url_utf8_qs, strlen(url_utf8_qs), result,found_pos, 4, &mid, 0); - EXPECT_GT(ret, 1); + EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 142); Maat_clean_status(&mid); @@ -843,6 +993,8 @@ TEST(StringScan, StreamInput) Maat_clean_status(&mid); return; } + + TEST(SimilarScan, Pure) { int ret=0; @@ -946,6 +1098,51 @@ TEST(StringScan, OffsetChunk1460) test_offset_str_scan_with_chunk(1460); return; } +#define StringScan_StreamScanUTF8 1 +TEST(StringScan, StreamScanUTF8) +{ + //Bug report: https://mantis.mesalab.cn/view.php?id=712 + int table_id=0,ret=0; + int read_size=0,pass_flag=0; + struct Maat_rule_t result[4]; + scan_status_t mid=NULL; + const char* table_name="TROJAN_PAYLOAD"; + const char* fn="./testdata/jd.com.html"; + FILE* fp=fopen(fn,"r"); + ASSERT_FALSE(fp==NULL); + + char scan_data[2048]; + table_id=Maat_table_register(g_feather,table_name); + ASSERT_GT(table_id, 0); + + + struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10); + stream_para_t sp=Maat_stream_scan_string_start(g_feather,table_id,0); + int detail_ret=0; + + ASSERT_FALSE(sp==NULL); + + while(0==feof(fp)) + { + read_size=fread(scan_data,1,sizeof(scan_data),fp); + ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data,read_size + ,result,4,hit_detail,10 + ,&detail_ret,&mid); + if(ret>0) + { + pass_flag=1; + break; + } + } + EXPECT_EQ(pass_flag, 1); + EXPECT_EQ(result[0].config_id, 157); + + Maat_stream_scan_string_end(&sp); + free(hit_detail); + fclose(fp); + Maat_clean_status(&mid); + return; +} void accept_tags_entry_cb(int table_id,const char* table_line,void* u_para) { @@ -959,7 +1156,7 @@ void accept_tags_entry_cb(int table_id,const char* table_line,void* u_para) return; } -TEST(RuleTags, Plugin1) +TEST(Policy, PluginRuleTags1) { #define RuleTags_Plugin int ret=0; @@ -985,7 +1182,7 @@ void accept_tags_entry2_cb(int table_id,const char* table_line,void* u_para) return; } -TEST(RuleTags, Plugin2) +TEST(Policy, PluginRuleTags2) { #define RuleTags_Plugin2 int ret=0; @@ -1005,7 +1202,7 @@ TEST(RuleTags, Plugin2) return; } -TEST(RuleTags, Compile) +TEST(Policy, CompileRuleTags) { #define RuleTags_Compile int ret=0; @@ -1086,7 +1283,7 @@ void compile_ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *fro -TEST(EX_DATA_INDEX, MaatRuleEXData) +TEST(Policy, CompileEXData) { #define rule_EX_data_index @@ -1120,6 +1317,42 @@ TEST(EX_DATA_INDEX, MaatRuleEXData) Maat_clean_status(&mid); return; } +TEST(Policy, SubGroup) +{ +#define TestSubGroup + int ret=0, table_id=0; + const char* scan_string="ceshi6@mailhost.cn"; + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + struct ipaddr ipv4_addr; + struct stream_tuple4_v4 v4_addr; + ipv4_addr.addrtype=ADDR_TYPE_IPV4; + inet_pton(AF_INET,"10.0.6.205",&(v4_addr.saddr)); + v4_addr.source=htons(50001); + inet_pton(AF_INET,"10.0.6.201",&(v4_addr.daddr)); + v4_addr.dest=htons(80); + ipv4_addr.v4=&v4_addr; + + table_id=Maat_table_register(g_feather,"MAIL_ADDR"); + ASSERT_GT(table_id, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, scan_string, strlen(scan_string), + result,NULL, 4, &mid, 0); + EXPECT_EQ(ret, -2); + + table_id=Maat_table_register(g_feather, "IP_CONFIG"); + ASSERT_GT(table_id, 0); + + ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid,0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 153); + + Maat_clean_status(&mid); + return; +} TEST(StreamFuzzyHash, Pure) @@ -1374,6 +1607,7 @@ protected: static int linger_timeout; }; Maat_feather_t MaatCmdTest::_shared_feather; + void* MaatCmdTest::logger; int MaatCmdTest::linger_timeout; @@ -1475,7 +1709,7 @@ TEST_F(MaatCmdTest, SetIP) EXPECT_GE(ret, 0); Maat_free_cmd(cmd); cmd=NULL; - + wait_for_cmd_effective(feather, version_before); struct ipaddr ipv4_addr; struct stream_tuple4_v4 v4_addr; @@ -1503,10 +1737,11 @@ TEST_F(MaatCmdTest, SetIP) //reset timeout. cmd=Maat_create_cmd(&rule, 0); - cmd->expire_after=10; + cmd->expire_after=timeout+5; ret=Maat_cmd(feather, cmd, MAAT_OP_RENEW_TIMEOUT); EXPECT_EQ(ret ,1); - + +// RENEW_TIMEOUT doesn't change MAAT_VERSION, thus we cannot use wait_for_cmd_effective. // wait_for_cmd_effective(feather, version_before); Maat_free_cmd(cmd); @@ -1608,7 +1843,7 @@ TEST_F(MaatCmdTest, RuleIDRecycle) struct Maat_rule_t result; scan_status_t mid=NULL; - + memset(&result, 0, sizeof(result)); int table_id=0; table_id=Maat_table_register(feather,table_name); ASSERT_GT(table_id, 0); @@ -1658,7 +1893,7 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder) int table_id=0; - table_id=Maat_table_register(feather,table_name); + table_id=Maat_table_register(feather, table_name); ASSERT_GT(table_id, 0); int i=0, repeat_times=4; @@ -1677,6 +1912,7 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder) ret=Maat_cmd_commit(feather); EXPECT_TRUE(ret>=0); usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect + memset(&result, 0, sizeof(result)); ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), result, NULL, 8, &mid, 0); @@ -1689,7 +1925,256 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder) } return; } +#define MaatCmdTest_SubGroup 0 +TEST_F(MaatCmdTest, SubGroup) +{ + const char* table_name="HTTP_URL"; + const char* group_table_name="GROUP"; + const char* compile_table_name="COMPILE"; + const char* scan_data1="www.v2ex.com/t/573028#程序员的核心竞争力是什么"; + const char* keyword1="程序员&核心竞争力"; + + const char* scan_data2="https://ask.leju.com/bj/detail/12189672562229248/?bi=tg&type=sina-pc&pos=index-dbtlwzl&wt_campaign=M_5CE750003F393&wt_source=PDPS_514ACACFD9E770"; +// const char* keyword2="ask.leju.com/bj/detail/12189672562229248/?bi=tg&type=sina-pc&pos=index-dbtlwzl&wt_campaign=M_5CE750003F393&wt_source=PDPS_514ACACFD9E770"; + const char* keyword2="ask.leju.com/b&/detail/12189672562229248/?&?bi=tg\\&type=sina-pc\\&&\\&pos=index-dbtlwzl\\&&\\&type=sina-pc\\&pos=index-dbtlwzl\\&"; + Maat_feather_t feather=MaatCmdTest::_shared_feather; + int table_id=0; + table_id=Maat_table_register(feather, table_name); + struct Maat_rule_t compile1, compile2; + struct Maat_group_t group1, group2, group3; + struct Maat_region_t region1,region2; + memset(&compile1, 0, sizeof(compile1)); + compile1.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1); + Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1); + + memset(&compile2, 0, sizeof(compile2)); + compile2.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1); + Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile2, compile_table_name, NULL, 1); + + //group1->compile1 + memset(&group1, 0, sizeof(group1)); + group1.group_id=Maat_cmd_get_new_group_id(feather); + group1.table_name=group_table_name; + group1.parent_id=compile1.config_id; + group1.parent_type=PARENT_TYPE_COMPILE; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1); + + //group1->compile2 + group1.parent_id=compile2.config_id; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1); + + /*group2->group1->compile1 + \ + \__compile2 + */ + memset(&group2, 0, sizeof(group2)); + group2.group_id=Maat_cmd_get_new_group_id(feather); + group2.table_name=group_table_name; + group2.parent_id=group1.group_id; + group2.parent_type=PARENT_TYPE_GROUP; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2); + + /*region1->group2->group1->compile1 + \ + \_compile2 + */ + memset(®ion1, 0, sizeof(region1)); + region1.region_id=Maat_cmd_get_new_region_id(feather); + region1.region_type=REGION_EXPR; + region1.table_name=table_name; + region1.expr_rule.keywords=keyword1; + region1.expr_rule.expr_type=EXPR_TYPE_AND; + Maat_command_raw_set_region(feather, MAAT_OP_ADD, ®ion1, group2.group_id); + sleep(1); + struct Maat_rule_t result[4]; + memset(&result, 0, sizeof(result)); + + scan_status_t mid=NULL; + int ret=0; + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1), + result, NULL, 4, + &mid, 0); + EXPECT_EQ(ret, 2); + EXPECT_EQ(result[0].config_id, compile2.config_id); + EXPECT_EQ(result[1].config_id, compile1.config_id); + Maat_clean_status(&mid); + + + /*region1->group2->group1->compile1 + \ + \--X--compile2 + */ + Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1); + sleep(1); + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1), + result, NULL, 4, + &mid, 0); + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, compile1.config_id); + Maat_clean_status(&mid); + + + /*region1->group2->group1--X-- + \ + \->compile2 + */ + group1.parent_id=compile1.config_id; + Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1); + Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile1, compile_table_name, NULL, 1); + group2.parent_type=PARENT_TYPE_COMPILE; + group2.parent_id=compile2.config_id; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2); + + sleep(1); + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1), + result, NULL, 4, + &mid, 0); + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, compile2.config_id); + Maat_clean_status(&mid); + + + /*region1->group2->group1--X-- + \ + \->compile2 + region2->group3 + */ + + memset(&group3, 0, sizeof(group3)); + group3.group_id=Maat_cmd_get_new_group_id(feather); + group3.parent_id=group1.group_id; + group3.parent_type=PARENT_TYPE_GROUP; + group3.table_name=group_table_name; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group3); + + memset(®ion2, 0, sizeof(region2)); + region2.region_id=Maat_cmd_get_new_region_id(feather); + region2.region_type=REGION_EXPR; + region2.table_name=table_name; + region2.expr_rule.keywords=keyword2; + region2.expr_rule.expr_type=EXPR_TYPE_AND; + Maat_command_raw_set_region(feather, MAAT_OP_ADD, ®ion2, group3.group_id); + + sleep(1); + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data2, strlen(scan_data2), + result, NULL, 4, + &mid, 0); + EXPECT_EQ(ret, -2); + Maat_clean_status(&mid); + + /*region1->group2->group1--X-- + \ + \->compile2 + region2->group3 + */ + Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1); + group1.parent_id=compile1.config_id; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1); + + Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile1, compile_table_name, NULL, 1); + group1.parent_id=compile1.config_id; + Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1); + + sleep(1); + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1), + result, NULL, 4, + &mid, 0); + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, compile2.config_id); + + Maat_clean_status(&mid); + +} +#define MaatCmdTest_RefGroup 0 +TEST_F(MaatCmdTest, RefGroup) +{ + + const char* table_name="HTTP_URL"; + const char* group_table_name="GROUP"; + const char* compile_table_name="COMPILE"; + const char* scan_data1="m.facebook.com/help/2297503110373101?helpref=hc_nav&refid=69"; + const char* keyword1="something-should-not-hit"; + const char* keyword2="facebook.com/help/2297503110373101"; + + Maat_feather_t feather=MaatCmdTest::_shared_feather; + int table_id=0; + table_id=Maat_table_register(feather, table_name); + struct Maat_rule_t compile1; + struct Maat_group_t group1, group2; + struct Maat_region_t region1, region2; + + int value=0; + Maat_set_feather_opt(feather, MAAT_OPT_ENABLE_UPDATE, &value, sizeof(value)); + + memset(&compile1, 0, sizeof(compile1)); + compile1.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1); + Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1); + + //group1->compile1 + memset(&group1, 0, sizeof(group1)); + group1.group_id=Maat_cmd_get_new_group_id(feather); + group1.table_name=group_table_name; + group1.parent_id=compile1.config_id; + group1.parent_type=PARENT_TYPE_COMPILE; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1); + + + /*region1->group1->compile1 + */ + memset(®ion1, 0, sizeof(region1)); + region1.region_id=Maat_cmd_get_new_region_id(feather); + region1.region_type=REGION_EXPR; + region1.table_name=table_name; + region1.expr_rule.keywords=keyword1; + region1.expr_rule.expr_type=EXPR_TYPE_AND; + Maat_command_raw_set_region(feather, MAAT_OP_ADD, ®ion1, group1.group_id); + value=1; + Maat_set_feather_opt(feather, MAAT_OPT_ENABLE_UPDATE, &value, sizeof(value)); + + sleep(1); + + value=0; + Maat_set_feather_opt(feather, MAAT_OPT_ENABLE_UPDATE, &value, sizeof(value)); + //region1->group1-X-compile1 + //region2->group2->compile1 + memset(&group2, 0, sizeof(group2)); + group2.group_id=Maat_cmd_get_new_group_id(feather); + group2.table_name=group_table_name; + group2.parent_id=compile1.config_id; + group2.parent_type=PARENT_TYPE_COMPILE; + + memset(®ion2, 0, sizeof(region2)); + region2.region_id=Maat_cmd_get_new_region_id(feather); + region2.region_type=REGION_EXPR; + region2.table_name=table_name; + region2.expr_rule.keywords=keyword2; + region2.expr_rule.expr_type=EXPR_TYPE_AND; + Maat_command_raw_set_region(feather, MAAT_OP_ADD, ®ion2, group2.group_id); + + + Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile1, compile_table_name, NULL, 1); + Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1); + Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1); + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2); + value=1; + Maat_set_feather_opt(feather, MAAT_OPT_ENABLE_UPDATE, &value, sizeof(value)); + + sleep(1); + + struct Maat_rule_t result[4]; + memset(&result, 0, sizeof(result)); + + scan_status_t mid=NULL; + int ret=0; + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1), + result, NULL, 4, + &mid, 0); + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, compile1.config_id); + Maat_clean_status(&mid); + +} TEST_F(MaatCmdTest, SetLines) { @@ -2105,7 +2590,7 @@ int main(int argc, char ** argv) const char* decrypt_key="mesa2017wy"; const char* accept_tags="{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; // const char* subsitute_acc_tags="{\"tags\":[{\"tag\":\"location\",\"value\":\"Astana\"}]}"; - int scan_detail=0; + int scan_detail=0, ret=0; ::testing::InitGoogleTest(&argc, argv); g_logger=MESA_create_runtime_log_handle(log_file, 0); @@ -2113,8 +2598,8 @@ int main(int argc, char ** argv) g_feather=Maat_feather(g_iThreadNum, table_info_path, g_logger); Maat_set_feather_opt(g_feather, MAAT_OPT_INSTANCE_NAME, "demo", strlen("demo")+1); Maat_set_feather_opt(g_feather, MAAT_OPT_DECRYPT_KEY, decrypt_key, strlen(decrypt_key)+1); - Maat_set_feather_opt(g_feather, MAAT_OPT_JSON_FILE_PATH, json_path, strlen(json_path)+1); - + ret=Maat_set_feather_opt(g_feather, MAAT_OPT_JSON_FILE_PATH, json_path, strlen(json_path)+1); + assert(ret==0); Maat_set_feather_opt(g_feather, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms)); //Set a short intevral for testing. Maat_set_feather_opt(g_feather, MAAT_OPT_EFFECT_INVERVAL_MS, &effective_interval_ms, sizeof(effective_interval_ms)); @@ -2127,7 +2612,7 @@ int main(int argc, char ** argv) Maat_initiate_feather(g_feather); printf("Maat initiating, see %s\n",log_file); - int ret=RUN_ALL_TESTS(); + ret=RUN_ALL_TESTS(); Maat_burn_feather(g_feather); MESA_destroy_runtime_log_handle(g_logger); return ret; diff --git a/test/testdata/jd.com.html b/test/testdata/jd.com.html new file mode 100644 index 0000000..1650566 --- /dev/null +++ b/test/testdata/jd.com.html @@ -0,0 +1,968 @@ + + + + + 京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+
    +
  • +
+ + +
+
+ + + + +
+ HTC手机哪款好 + 双卡双待手机哪款好 + 8GB手机排行榜 + 联想天逸510s 台式机 + 海尔滚筒式洗衣机 + 网络赚钱 + 白酒 + 立白洗衣液 + 青少年牙刷 + 洗面奶 + 唐狮拼色 连衣裙 + 苹果手机 + 琵琶 + 女士鞋 + 实木沙发 + 索尼电视 + 卫浴品牌 + 奥克斯空调 + 百香果 + 索尼耳机 + 冰箱镶入墙里 + 康佳家电下乡冰箱 + 奥克斯（AUX）BCD-116AD + 三星三门 + 120升冰箱 + 容声冰箱销量 + 创维（Skyworth）定频冰箱 + 容升晶弘冰箱 + 奥马（Homa）风冷（无霜）定频冰箱 + KG23N1116W + 西门孑小型对开门 + 迷你冰箱50l + 256升冰箱 + HNBX多门风冷冰箱 + 长城冰箱 + 松下对开门电冰箱 + 德努希变频冰箱 + 冰箱双开门排名 + 冰鹭（BINGLU）BCD-78S + 三星冰箱285wmq +
+ +
+
+
+
+ + +
+ +
+ +
+
+
+
+
+ +
+
+
+ + +
+
+
+ + +
+ + + + + + + + + + + + diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index bbfd508..e235b8b 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -8,4 +8,9 @@ target_link_libraries(maat_redis_tool maat_frame_static ${MAAT_DEPEND_DYN_LIB}) target_include_directories(maat_redis_tool PRIVATE ${PROJECT_SOURCE_DIR}/src/inc_internal/) target_include_directories(maat_redis_tool PRIVATE hiredis-vip-static) +add_executable(maat_debug_tool maat_debug_tool.cpp) +add_dependencies(maat_debug_tool maat_frame_shared) +target_link_libraries(maat_debug_tool maat_frame_static ${MAAT_DEPEND_DYN_LIB}) +target_include_directories(maat_debug_tool PRIVATE ${PROJECT_SOURCE_DIR}/src/inc_internal/) + install(TARGETS maat_redis_tool DESTINATION /usr/local/bin/) diff --git a/tools/maat_debug_tool.cpp b/tools/maat_debug_tool.cpp new file mode 100644 index 0000000..5b8d3be --- /dev/null +++ b/tools/maat_debug_tool.cpp @@ -0,0 +1,174 @@ +#include "Maat_rule.h" +#include "Maat_command.h" +#include + +#include /* for printf */ +#include /* for exit */ +#include +#include + + +int debug_maat_str_scan(Maat_feather_t feather, const char* table_name, const char* district, char* data, size_t sz) +{ + int table_id=0,ret=0; + int i=0; + int scan_result_max=64; + struct Maat_rule_t result[scan_result_max]; + scan_status_t mid=NULL; + + + table_id=Maat_table_register(feather, table_name); + if(table_id<0) + { + printf("Unkown table %s\n", table_name); + } + + struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10); + + enum MAAT_CHARSET maat_charset=CHARSET_UTF8; + + if(strlen(district)>0) + { + ret=Maat_set_scan_status(feather, &mid, MAAT_SET_SCAN_DISTRICT, district, strlen(district)); + } + + ret=Maat_full_scan_string(feather, table_id, maat_charset, data, sz, + result, NULL, scan_result_max, + &mid, 0); + printf("Scan table %s ", table_name); + if(ret==-1) + { + printf("error.\n"); + } + else if(ret==-2) + { + printf("hits group, but not compile.\n"); + } + else if(ret==0) + { + printf("not hit."); + } + else + { + printf("hits "); + for(i=0; i0) + { + Maat_set_feather_opt(feather, MAAT_OPT_DECRYPT_KEY, arg_value[ARG_DECRYPT_KEY], strlen(arg_value[ARG_DECRYPT_KEY])+1); + } + if(strlen(arg_value[ARG_ACCEPT_TAGS])>0) + { + Maat_set_feather_opt(feather, MAAT_OPT_ACCEPT_TAGS, arg_value[ARG_ACCEPT_TAGS], strlen(arg_value[ARG_ACCEPT_TAGS])+1); + } + ret=Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, arg_value[ARG_INPUT_JSON], strlen(arg_value[ARG_INPUT_JSON])+1); + if(ret!=0) + { + printf("Read %s failed, invalid maat json.\n", arg_value[ARG_INPUT_JSON]); + ret=-1; + goto clean_up; + } + Maat_set_feather_opt(feather, MAAT_OPT_SCAN_DETAIL, &scan_detail, sizeof(scan_detail)); + Maat_initiate_feather(feather); + + if(strlen(arg_value[ARG_SCAN_FILE])>0) + { + + if(stat(arg_value[ARG_SCAN_FILE], &file_info)<0) + { + printf("Stat %s failed.\n", arg_value[ARG_SCAN_FILE]); + goto clean_up; + } + file_size=file_info.st_size; + fp=fopen(arg_value[ARG_SCAN_FILE], "r"); + if(fp==NULL) + { + printf("Open %s failed.\n", arg_value[ARG_SCAN_FILE]); + goto clean_up; + } + file_buff=(char*)malloc(file_size); + fread(file_buff,1,file_size,fp); + fclose(fp); + + debug_maat_str_scan(feather, arg_value[ARG_TABLE_NAME], arg_value[ARG_SCAN_DISTRICT], file_buff, file_size); + free(file_buff); + } +clean_up: + Maat_burn_feather(feather); + MESA_destroy_runtime_log_handle(g_logger); + return ret; +} + + diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index 9f5a378..d57f53e 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -30,3 +30,19 @@ file(MAKE_DIRECTORY ${HIREDIS_INCLUDE_DIRECTORIES}) add_library(hiredis-vip-static STATIC IMPORTED GLOBAL) set_property(TARGET hiredis-vip-static PROPERTY IMPORTED_LOCATION ${SOURCE_DIR}/libhiredis_vip.a) set_property(TARGET hiredis-vip-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${SOURCE_DIR}) + +ExternalProject_Add(igraph PREFIX igraph + URL ${CMAKE_CURRENT_SOURCE_DIR}/igraph-0.7.1.tar.gz + URL_MD5 4f6e7c16b45fce8ed423516a9786e4e8 + CONFIGURE_COMMAND ./configure --prefix= --disable-shared --with-pic + BUILD_COMMAND make + INSTALL_COMMAND make install + BUILD_IN_SOURCE 1) + +ExternalProject_Get_Property(igraph INSTALL_DIR) +file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + +add_library(igraph-static STATIC IMPORTED GLOBAL) +add_dependencies(igraph-static igraph) +set_property(TARGET igraph-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libigraph.a) +set_property(TARGET igraph-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) diff --git a/vendor/igraph-0.7.1.tar.gz b/vendor/igraph-0.7.1.tar.gz new file mode 100644 index 0000000..9340f6b Binary files /dev/null and b/vendor/igraph-0.7.1.tar.gz differ