diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index b8d3e66..42a13d5 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -124,7 +124,8 @@ Maat_feather_t Maat_summon_feather_json(int max_thread_num, Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger); int Maat_initiate_feather(Maat_feather_t feather); -enum MAAT_INIT_OPT{ +enum MAAT_INIT_OPT +{ MAAT_OPT_SCANDIR_INTERVAL_MS=1, //VALUE is interger,SIZE=sizeof(int). DEFAULT:1,000 milliseconds. MAAT_OPT_EFFECT_INVERVAL_MS, //VALUE is interger,SIZE=sizeof(int). DEFAULT:60,000 milliseconds. MAAT_OPT_FULL_CFG_DIR, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. @@ -149,7 +150,12 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, Maat_finish_callback_t *finish,//u_para void* u_para); - +enum MAAT_SCAN_OPT +{ + MAAT_SET_SCAN_DISTRICT=1 //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. +}; +//return 0 if success, return -1 when failed; +int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT,const void* value,int size); //Return hit rule number, return -1 when error occurs,return -2 when hit current region //mid MUST set NULL before fist call diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index d87cc88..a01e9d1 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -26,10 +26,18 @@ struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int t return NULL; } p_table=_feather->p_table_info[table_id]; - if(p_table==NULL||p_table->table_type!=expect_type) + if(p_table==NULL) { return NULL; } + if(p_table->table_type!=expect_type) + { + if(expect_type!=TABLE_TYPE_EXPR|| + p_table->table_type!=TABLE_TYPE_EXPR_PLUS) + { + return NULL; + } + } return p_table; } inline void INC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num) @@ -90,14 +98,14 @@ int pickup_hit_region_from_compile(universal_bool_expr_t *compile_hit,const unsi } return k; } -int region_compile(struct _scan_status_t *_mid,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size) +int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size) { int scan_ret=0,result_cnt=0; int ret=0,i=0,j=0; int r_in_c_cnt=0; - void* expr_compiler=_mid->feather->scanner->expr_compiler; int shortcut_avilable_cnt=0; + void* bool_matcher=feather->scanner->expr_compiler; struct _Maat_group_rule_t* group_rule=NULL; struct _Maat_compile_rule_t* array_mi_rule[MAX_SCANNER_HIT_NUM]; struct _Maat_compile_rule_t* _mi_rule=NULL; @@ -130,7 +138,7 @@ int region_compile(struct _scan_status_t *_mid,void* region_hit,int region_type_ } else { - scan_ret=boolexpr_match(expr_compiler,_mid->thread_num, + scan_ret=boolexpr_match(bool_matcher,_mid->thread_num, _mid->hitted_group_id,_mid->hit_group_cnt, (void **)array_mi_rule, MAX_SCANNER_HIT_NUM); } @@ -161,14 +169,14 @@ int region_compile(struct _scan_status_t *_mid,void* region_hit,int region_type_ } if(result_cnt>0) { - _mid->feather->hit_cnt++; + feather->hit_cnt++; } return result_cnt; } -int exprid2region_id(struct _Maat_group_rule_t* group_rule,int expr_id) +int exprid2region_id(struct _Maat_group_rule_t* group_rule,int expr_id,int* district_id) { - int i=0,region_id=-1;; + int i=0,region_id=-1; struct _Maat_region_rule_t* region_rule=NULL; assert(group_rule->group_id>=0); pthread_mutex_lock(&(group_rule->mutex)); @@ -182,11 +190,34 @@ int exprid2region_id(struct _Maat_group_rule_t* group_rule,int expr_id) if(region_rule->expr_id==expr_id) { region_id=region_rule->region_id; + *district_id=region_rule->district_id; } } pthread_mutex_unlock(&(group_rule->mutex)); return region_id; } +int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num) +{ + struct _Maat_group_rule_t* group_rule=NULL; + int i=0; + int district_id=-1,region_id=-1; + int ret_region_num=region_hit_num; + while(i0&&district_id!=_mid->district_id) + { + ret_region_num--; + memmove(&(region_hit[i]),&(region_hit[i+1]),sizeof(scan_result_t)*(ret_region_num-i)); + } + else + { + i++; + } + } + return ret_region_num; +} int fill_regex_pos(struct regex_pos_t *regex_pos,int size,rule_result_t *rs_result,const char* buff) @@ -255,7 +286,7 @@ int hit_pos_RS2Maat(struct sub_item_pos_t* maat_sub_item,int size,rule_result_t* } return k; } -int fill_region_hit_detail(const char* scan_buff,const _scan_status_t* _mid, +int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mid, scan_result_t *region_hit,int region_cnt, _compile_result_t *compile_hit,int compile_cnt, struct Maat_hit_detail_t *hit_detail,int detail_num) @@ -265,6 +296,7 @@ int fill_region_hit_detail(const char* scan_buff,const _scan_status_t* _mid, int region_pos[MAX_SCANNER_HIT_NUM],pos=0; int r_in_c_cnt=0; int region_id=-1; + int district_id=-1; //Indifferenc memset(r_in_c_flag,0,sizeof(r_in_c_flag)); memset(region_pos,0,sizeof(region_pos)); @@ -281,7 +313,7 @@ int fill_region_hit_detail(const char* scan_buff,const _scan_status_t* _mid, pos=region_pos[j]; r_in_c_flag[pos]=1; group_rule=(struct _Maat_group_rule_t*)(region_hit[pos].tag); - region_id=exprid2region_id(group_rule,region_hit[pos].expr_id); + region_id=exprid2region_id(group_rule,region_hit[pos].expr_id,&district_id); if(region_id<0) { continue; @@ -303,7 +335,7 @@ int fill_region_hit_detail(const char* scan_buff,const _scan_status_t* _mid, group_rule=(struct _Maat_group_rule_t*)(region_hit[k].tag); hit_detail[j].config_id=-2; hit_detail[j].hit_region_cnt=1; - hit_detail[j].region_pos[0].region_id=exprid2region_id(group_rule,region_hit[k].expr_id); + hit_detail[j].region_pos[0].region_id=exprid2region_id(group_rule,region_hit[k].expr_id,&district_id); hit_detail[j].region_pos[0].sub_item_num=region_hit[k].rnum; hit_pos_RS2Maat(hit_detail[j].region_pos[0].sub_item_pos,MAAT_MAX_EXPR_ITEM_NUM, region_hit[k].result,region_hit[k].rnum,scan_buff); @@ -312,20 +344,48 @@ int fill_region_hit_detail(const char* scan_buff,const _scan_status_t* _mid, } return j; } -struct _scan_status_t* _Maat_make_status(struct _Maat_feather_t* feather,int thread_num) +struct _INNER_scan_status_t* _make_inner_status(int thread_num) { - struct _scan_status_t* _mid=NULL; - _mid=(struct _scan_status_t*)calloc(sizeof(struct _scan_status_t),1); - _mid->feather=feather; - _mid->thread_num=thread_num; - _mid->cur_hit_cnt=0; - _mid->hit_group_cnt=0; - _mid->hit_group_size=4; - _mid->hitted_group_id=(unsigned int*)malloc(sizeof(unsigned int)*_mid->hit_group_size); - feather->mid_cnt++; + struct _INNER_scan_status_t* inner_mid=NULL; + inner_mid=(struct _INNER_scan_status_t*)calloc(sizeof(struct _INNER_scan_status_t),1); + inner_mid->thread_num=thread_num; + inner_mid->cur_hit_cnt=0; + inner_mid->hit_group_cnt=0; + inner_mid->hit_group_size=4; + inner_mid->hitted_group_id=(unsigned int*)malloc(sizeof(unsigned int)*inner_mid->hit_group_size); + return inner_mid; +} +struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather) +{ + struct _OUTER_scan_status_t* outer_mid=NULL; + outer_mid=(struct _OUTER_scan_status_t*)calloc(sizeof(struct _OUTER_scan_status_t),1); + outer_mid->feather=feather; + outer_mid->district_id=-1; + feather->outer_mid_cnt++; + return outer_mid; +} +struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* feather,int thread_num,int is_hit_region) +{ + struct _OUTER_scan_status_t* _mid=NULL; + if(*raw_mid!=NULL) + { + _mid=(struct _OUTER_scan_status_t*)(*raw_mid); + } + if(is_hit_region==1) + { + if(_mid==NULL) + { + _mid=_make_outer_status(feather); + *raw_mid=_mid; + } + if(_mid->inner==NULL) + { + _mid->inner=_make_inner_status(thread_num); + feather->inner_mid_cnt++; + } + } return _mid; } - int detain_last_data(char* buff,int buff_size,int detained_len,const char* data,int data_len) { int to_copy_size=0,foward_offset=0; @@ -352,7 +412,6 @@ int detain_last_data(char* buff,int buff_size,int detained_len,const char* data, } return ret_len; } - Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger) { _Maat_feather_t* feather=(_Maat_feather_t*)calloc(sizeof(struct _Maat_feather_t),1); @@ -633,7 +692,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id int region_ret=0,compile_ret=0,hit_region_cnt=0; unsigned int sub_type=0; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct _scan_status_t* _mid=NULL; + struct _OUTER_scan_status_t* _mid=(struct _OUTER_scan_status_t*)(*mid); scan_result_t *region_result=NULL; _compile_result_t compile_result[rule_num];//dynamic array @@ -643,6 +702,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id { gettimeofday(&start,NULL); } + _mid=grab_mid(mid,_feather, thread_num, 0); p_table=acqurie_table(_feather, table_id,TABLE_TYPE_EXPR); if(p_table==NULL) { @@ -652,6 +712,10 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id { return 0; } + if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) + { + return -1; + } if(p_table->do_charset_merge==1) { sub_type=make_sub_type(table_id,CHARSET_NONE,0); @@ -696,25 +760,21 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } } + if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS) + { + hit_region_cnt=match_district(_mid,region_result,hit_region_cnt); + } if(hit_region_cnt>0) { p_table->hit_cnt++; - if(*mid==NULL) - { - _mid=_Maat_make_status(_feather,thread_num); - *mid=_mid; - } - else - { - _mid=(struct _scan_status_t*)(*mid); - } - compile_ret=region_compile(_mid, + _mid=grab_mid(mid,_feather,thread_num, 1); + compile_ret=region_compile(_feather,_mid->inner, region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), hit_region_cnt, result,compile_result,rule_num); if(hit_detail!=NULL) { - *detail_ret=fill_region_hit_detail(data,_mid, + *detail_ret=fill_region_hit_detail(data,_mid->inner, region_result,hit_region_cnt, compile_result,compile_ret, hit_detail,detail_num); @@ -750,7 +810,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id ,scan_status_t *mid,int thread_num) { int region_ret=0,compile_ret=0; - struct _scan_status_t* _mid=NULL; + struct _OUTER_scan_status_t* _mid=NULL; scan_data_t intval_scan_data; scan_result_t *region_result=NULL; _compile_result_t compile_result[rule_num]; @@ -792,16 +852,8 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id else if(region_ret>0) { p_table->hit_cnt++; - if(*mid==NULL) - { - _mid=_Maat_make_status(_feather,thread_num); - *mid=_mid; - } - else - { - _mid=(struct _scan_status_t*)(*mid); - } - compile_ret=region_compile(_mid, + _mid=grab_mid(mid, _feather, thread_num, 1); + compile_ret=region_compile(_feather,_mid->inner, region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), region_ret, result,compile_result,rule_num); @@ -827,7 +879,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id ,scan_status_t *mid,int thread_num) { int region_ret=0,compile_ret=0; - struct _scan_status_t* _mid=NULL; + struct _OUTER_scan_status_t* _mid=NULL; scan_data_t ip_scan_data; scan_result_t *region_result=NULL; _compile_result_t compile_result[rule_num]; @@ -892,16 +944,8 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id else if(region_ret>0) { p_table->hit_cnt++; - if(*mid==NULL) - { - _mid=_Maat_make_status(_feather,thread_num); - *mid=_mid; - } - else - { - _mid=(struct _scan_status_t*)(*mid); - } - compile_ret=region_compile(_mid, + _mid=grab_mid(mid, _feather, thread_num, 1); + compile_ret=region_compile(_feather,_mid->inner, region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), region_ret, result,compile_result,rule_num); @@ -987,15 +1031,17 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para int sub_type=0; int region_ret=0,hit_region_cnt=0,compile_ret=0; - struct _scan_status_t* _mid=(struct _scan_status_t*)(*mid); + struct _OUTER_scan_status_t* _mid=NULL; scan_result_t *region_result; _compile_result_t compile_result[rule_num];//dynamic array scan_data_t region_scan_data; + _Maat_table_info_t* p_table=NULL; struct timeval start,end; if(sp->feather->perf_on==1) { gettimeofday(&start,NULL); } + _mid=grab_mid(mid, sp->feather, sp->thread_num,0); if(data==NULL||data_len==0) { return 0; @@ -1008,10 +1054,15 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para { return 0; } - if(sp->feather->p_table_info[sp->table_id]->cfg_num==0) + p_table=sp->feather->p_table_info[sp->table_id]; + if(p_table->cfg_num==0) { return 0; } + if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) + { + return -1; + } region_result=scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*sp->thread_num; *detail_ret=0; if(sp->do_merge==1) @@ -1081,20 +1132,15 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para hit_region_cnt+=region_ret; } } + if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS) + { + hit_region_cnt=match_district(_mid,region_result,hit_region_cnt); + } if(hit_region_cnt>0) { - sp->feather->p_table_info[sp->table_id]->hit_cnt++; - - if(*mid==NULL) - { - _mid=_Maat_make_status(sp->feather,sp->thread_num); - *mid=_mid; - } - else - { - _mid=(struct _scan_status_t*)(*mid); - } - compile_ret=region_compile(_mid, + p_table->hit_cnt++; + _mid=grab_mid(mid, sp->feather,sp->thread_num, 1); + compile_ret=region_compile(sp->feather,_mid->inner, region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), hit_region_cnt, result,compile_result,rule_num); @@ -1102,14 +1148,14 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para { if(sp->scan_buff!=NULL) { - *detail_ret=fill_region_hit_detail(sp->scan_buff,_mid, + *detail_ret=fill_region_hit_detail(sp->scan_buff,_mid->inner, region_result,hit_region_cnt, compile_result,compile_ret, hit_detail,detail_num); } else { - *detail_ret=fill_region_hit_detail(data,_mid, + *detail_ret=fill_region_hit_detail(data,_mid->inner, region_result,hit_region_cnt, compile_result,compile_ret, hit_detail,detail_num); @@ -1119,7 +1165,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para if(*detail_ret==0) { free(sp->scan_buff); - sp->scan_buff=0; + sp->scan_buff=NULL; } if(sp->feather->perf_on==1) { @@ -1244,7 +1290,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int GIE_handle_t* GIE_handle=sp->feather->scanner->digest_handle[sp->table_id]; unsigned long long digest_len=0; char* digest_buff=NULL; - struct _scan_status_t* _mid=(struct _scan_status_t*)(*mid); + struct _OUTER_scan_status_t* _mid=NULL; pthread_rwlock_t *GIE_rwlock=&(sp->feather->scanner->digest_rwlock[sp->table_id]); struct timeval start,end; if(sp->feather->perf_on==1) @@ -1291,16 +1337,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int if(hit_region_cnt>0) { sp->feather->p_table_info[sp->table_id]->hit_cnt++; - if(*mid==NULL) - { - _mid=_Maat_make_status(sp->feather,sp->thread_num); - *mid=_mid; - } - else - { - _mid=(struct _scan_status_t*)(*mid); - } - compile_ret=region_compile(_mid, + _mid=grab_mid(mid,sp->feather, sp->thread_num,1); + compile_ret=region_compile(sp->feather,_mid->inner, query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag), hit_region_cnt, result,compile_result,rule_num); @@ -1340,17 +1378,52 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para) return; } +int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT type,const void* value,int size) +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct _OUTER_scan_status_t* _mid=NULL; + int ret=0,map_ret=-1; + _mid=grab_mid(mid,_feather, 0, 0); + if(*mid==NULL) + { + _mid=_make_outer_status(_feather); + *mid=_mid; + } + switch(type) + { + case MAAT_SET_SCAN_DISTRICT: + map_ret=map_str2int(_feather->scanner->district_map,(const char*)value,&(_mid->district_id)); + if(map_ret<0) + { + //May be the district have not effect yet. + } + _mid->is_set_district=1; + ret=0; + break; + default: + ret=-1; + break; + } + return ret; +} void Maat_clean_status(scan_status_t* mid) { - struct _scan_status_t* _mid=NULL; + struct _OUTER_scan_status_t* _mid=NULL; if(*mid==NULL) { return; } - _mid=(struct _scan_status_t*)(*mid); - _mid->feather->mid_cnt--; - free(_mid->hitted_group_id); + _mid=(struct _OUTER_scan_status_t*)(*mid); + _mid->feather->outer_mid_cnt--; + if(_mid->inner!=NULL) + { + free(_mid->inner->hitted_group_id); + free(_mid->inner); + _mid->feather->inner_mid_cnt--; + } + _mid->feather=NULL; free(_mid); *mid=NULL; return; } + diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index d47debd..92a5e4c 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -25,7 +25,7 @@ #include "mesa_fuzzy.h" #include "great_index_engine.h" -int MAAT_FRAME_VERSION_1_4_20160210=1; +int MAAT_FRAME_VERSION_1_5_20160211=1; const char *maat_module="MAAT Frame"; const char* CHARSET_STRING[]={"CHARSET_NONE","GBK","BIG5","UNICODE","UTF-8"}; @@ -269,6 +269,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN); map_register(string2int_map,"intval", TABLE_TYPE_INTVAL); map_register(string2int_map,"digest", TABLE_TYPE_DIGEST); + map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS); map_register(string2int_map,"group", TABLE_TYPE_GROUP); map_register(string2int_map,"bin", CHARSET_NONE); map_register(string2int_map,"gbk", CHARSET_GBK); @@ -706,6 +707,8 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread scanner->region_hash=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(scanner->region_hash,0); + + scanner->district_map=map_create(); scanner->version=version; scanner->cfg_num=0; @@ -738,7 +741,7 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner) MESA_htable_destroy(scanner->compile_hash,NULL); MESA_htable_destroy(scanner->group_hash, NULL); MESA_htable_destroy(scanner->region_hash, NULL); - + map_destroy(scanner->district_map); destroy_bool_matcher((void*)scanner->expr_compiler); q_cnt=MESA_lqueue_get_count(scanner->region_update_q); for(i=0;ilast_update_time=time(NULL); return; } -struct _Maat_group_rule_t* add_region_to_group(struct _Maat_group_rule_t* group,int region_id,int expr_id,enum MAAT_TABLE_TYPE region_type) +struct _Maat_group_rule_t* add_region_to_group(struct _Maat_group_rule_t* group,int region_id,int district_id,int expr_id,enum MAAT_TABLE_TYPE region_type) { struct _Maat_region_rule_t* region_rule=(struct _Maat_region_rule_t*)malloc(sizeof(struct _Maat_region_rule_t)); region_rule->region_id=region_id; region_rule->expr_id=expr_id; + region_rule->district_id=district_id; region_rule->region_type=region_type; pthread_mutex_lock(&(group->mutex)); dynamic_array_write(group->region_rules,group->region_boundary,region_rule); @@ -1076,12 +1080,24 @@ int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_n } return 1; } +int get_district_id(_Maat_scanner_t *scanner,const char* district_str) +{ + int map_ret=0,district_id=-1; + map_ret=map_str2int(scanner->district_map, district_str,&district_id); + if(map_ret<0) + { + district_id= scanner->district_num; + map_register(scanner->district_map,district_str, district_id); + scanner->district_num++; + } + return district_id; +} int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger) { unsigned int i=0,j=0; char* p=NULL,*saveptr=NULL,*region_string=NULL; int region_str_len=0,ret=0,k=0; - int expr_id=0; + int expr_id=0,district_id=-1; scan_rule_t*p_rule=NULL; struct _Maat_group_rule_t* group_rule=NULL; @@ -1096,7 +1112,12 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule int sub_expr_cnt=0; struct op_expr_t *op_expr=NULL; struct _Maat_group_rule_t* u_para=NULL; - + + if(table->table_type==TABLE_TYPE_EXPR_PLUS) + { + assert(strlen(db_rule->district)>0); + district_id=get_district_id(scanner, db_rule->district); + } group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id); if(group_rule==NULL) { @@ -1168,7 +1189,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule break; case EXPR_TYPE_REGEX://it's easy,no need to charset convert expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule,db_rule->region_id,expr_id,TABLE_TYPE_EXPR); + u_para=add_region_to_group(group_rule,db_rule->region_id,district_id,expr_id,TABLE_TYPE_EXPR); if(u_para==NULL) { return -1; @@ -1234,7 +1255,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule break; } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule, db_rule->region_id,expr_id, TABLE_TYPE_EXPR); + u_para=add_region_to_group(group_rule, db_rule->region_id,district_id,expr_id, table->table_type); if(u_para==NULL)//duplicate { return -1; @@ -1299,7 +1320,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule else { expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule, db_rule->region_id,expr_id, TABLE_TYPE_EXPR); + u_para=add_region_to_group(group_rule, db_rule->region_id,district_id,expr_id, table->table_type); if(u_para==NULL) { return -1; @@ -1335,7 +1356,7 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule scan_rule_t* p_rule=NULL; struct op_expr_t* op_expr=NULL; struct _Maat_group_rule_t* u_para=NULL; - int expr_id=0; + int expr_id=0,district_id=-1; group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id); if(group_rule==NULL) @@ -1345,7 +1366,7 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule,db_ip_rule->region_id,expr_id,TABLE_TYPE_IP); + u_para=add_region_to_group(group_rule,db_ip_rule->region_id,district_id,expr_id,TABLE_TYPE_IP); if(u_para==NULL) { return -1; @@ -1366,8 +1387,8 @@ int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* in scan_rule_t* p_rule=NULL; struct op_expr_t* op_expr=NULL; struct _Maat_group_rule_t* u_para=NULL; - int expr_id=0; - + int expr_id=0,district_id=-1; + group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id); if(group_rule==NULL) { @@ -1375,7 +1396,7 @@ int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* in HASH_add_by_id(scanner->group_hash, intval_rule->group_id, group_rule); } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule,intval_rule->region_id,expr_id,TABLE_TYPE_INTVAL); + u_para=add_region_to_group(group_rule,intval_rule->region_id,district_id,expr_id,TABLE_TYPE_INTVAL); if(u_para==NULL) { return -1; @@ -1395,7 +1416,7 @@ int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db struct _Maat_group_rule_t* group_rule=NULL; GIE_digest_t* digest_rule=NULL; struct _Maat_group_rule_t* u_para=NULL; - int expr_id=0; + int expr_id=0,district_id=-1; group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); if(group_rule==NULL) @@ -1404,7 +1425,7 @@ int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db HASH_add_by_id(scanner->group_hash, db_digest_rule->group_id, group_rule); } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule,db_digest_rule->region_id,expr_id,TABLE_TYPE_DIGEST); + u_para=add_region_to_group(group_rule,db_digest_rule->region_id,expr_id,district_id,TABLE_TYPE_DIGEST); if(u_para==NULL) { return -1; @@ -1643,20 +1664,46 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st { struct db_str_rule_t* maat_str_rule=(struct db_str_rule_t*)malloc(sizeof(struct db_str_rule_t)); int ret=0,db_hexbin=0; - ret=sscanf(table_line,"%d\t%d\t%s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id) - ,&(maat_str_rule->group_id) - ,maat_str_rule->keywords - ,(int*)&(maat_str_rule->expr_type) - ,(int*)&(maat_str_rule->match_method) - ,&db_hexbin - ,&(maat_str_rule->is_valid)); - if(ret!=7) + switch(table->table_type) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,invalid format of expr table %s:%s",table->table_name,table_line); - free(maat_str_rule); - maat_str_rule=NULL; - return; + case TABLE_TYPE_EXPR: + ret=sscanf(table_line,"%d\t%d\t%s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id) + ,&(maat_str_rule->group_id) + ,maat_str_rule->keywords + ,(int*)&(maat_str_rule->expr_type) + ,(int*)&(maat_str_rule->match_method) + ,&db_hexbin + ,&(maat_str_rule->is_valid)); + if(ret!=7) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,invalid format of expr table %s:%s",table->table_name,table_line); + free(maat_str_rule); + maat_str_rule=NULL; + return; + } + break; + case TABLE_TYPE_EXPR_PLUS: + ret=sscanf(table_line,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id) + ,&(maat_str_rule->group_id) + ,maat_str_rule->district + ,maat_str_rule->keywords + ,(int*)&(maat_str_rule->expr_type) + ,(int*)&(maat_str_rule->match_method) + ,&db_hexbin + ,&(maat_str_rule->is_valid)); + if(ret!=8) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,invalid format of expr_plus table %s:%s",table->table_name,table_line); + free(maat_str_rule); + maat_str_rule=NULL; + return; + } + break; + default: + assert(0); + break; } switch(db_hexbin) { @@ -2305,6 +2352,7 @@ void maat_update_cb(const char* table_name,const char* line,void *u_para) switch(feather->p_table_info[table_id]->table_type) { case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: update_expr_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); break; case TABLE_TYPE_IP: diff --git a/src/entry/Maat_rule_internal.h b/src/entry/Maat_rule_internal.h index 89dd64f..a2b223b 100644 --- a/src/entry/Maat_rule_internal.h +++ b/src/entry/Maat_rule_internal.h @@ -40,6 +40,7 @@ typedef int atomic_t; #define MAX_TABLE_NAME_LEN 256 #define MAX_TABLE_LINE_SIZE (1024*4) #define MAX_EXPR_KEYLEN 1024 +#define MAX_DISTRICT_LEN 64 #define MAX_PLUGING_NUM 32 #define MAX_SCANNER_HIT_NUM 64 @@ -76,6 +77,7 @@ enum MAAT_TABLE_TYPE TABLE_TYPE_PLUGIN, TABLE_TYPE_INTVAL, TABLE_TYPE_DIGEST, + TABLE_TYPE_EXPR_PLUS, TABLE_TYPE_GROUP }; @@ -99,6 +101,7 @@ struct db_str_rule_t int region_id; int group_id; char keywords[MAX_EXPR_KEYLEN]; + char district[MAX_DISTRICT_LEN]; enum MAAT_EXPR_TYPE expr_type; enum MAAT_MATCH_METHOD match_method; int is_hexbin; @@ -171,6 +174,7 @@ struct _Maat_region_rule_t { int region_id; int expr_id; + int district_id; enum MAAT_TABLE_TYPE region_type; }; struct _Maat_group_rule_t @@ -235,9 +239,9 @@ struct _Maat_table_info_t long long stream_num; long long hit_cnt; }; -struct _scan_status_t + +struct _INNER_scan_status_t { - struct _Maat_feather_t* feather; int thread_num; int cur_hit_cnt; int hit_group_cnt; @@ -245,6 +249,13 @@ struct _scan_status_t unsigned int cur_hit_id[MAX_SCANNER_HIT_NUM]; unsigned int *hitted_group_id; }; +struct _OUTER_scan_status_t +{ + struct _Maat_feather_t* feather; + int is_set_district; + int district_id; + struct _INNER_scan_status_t* inner; +}; enum maat_garbage_type { GARBAGE_SCANNER=0, @@ -288,6 +299,8 @@ struct _Maat_scanner_t MESA_htable_handle region_hash; MESA_htable_handle group_hash; MESA_htable_handle compile_hash; + MESA_htable_handle district_map; + unsigned int district_num; unsigned int cfg_num; unsigned int exprid_generator; MESA_lqueue_head region_update_q; @@ -324,7 +337,8 @@ struct _Maat_feather_t int total_stat_id; int fs_status_id[MAX_MAAT_STAT_NUM]; int fs_column_id[MAX_MAAT_STAT_NUM]; - long long mid_cnt; + long long outer_mid_cnt; + long long inner_mid_cnt; long long hit_cnt; }; struct _maat_garbage_t diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index cb25e3d..2b05e77 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -3,7 +3,8 @@ enum MAAT_FS_STATUS{ STATUS_VERSION=0, STATUS_TABLE_NUM, - STATUS_MID_NUM, + STATUS_OUTER_MID_NUM, + STATUS_INNER_MID_NUM, STATUS_GARBAGE_QSIZE, }; @@ -32,7 +33,8 @@ void maat_stat_init(struct _Maat_feather_t* feather) feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"version"); feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"table_num"); - feather->fs_status_id[STATUS_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"mid_num"); + feather->fs_status_id[STATUS_OUTER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"outer_mid"); + feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid"); feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"garbage_num"); feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule"); @@ -93,7 +95,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_MID_NUM], 0,FS_OP_SET,feather->mid_cnt); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,feather->outer_mid_cnt); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,feather->inner_mid_cnt); value=MESA_lqueue_get_count(feather->garbage_q); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value); diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index cdf5a98..397aa85 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -110,8 +110,11 @@ int set_iris_descriptor(const char* json_file,cJSON *json,struct iris_descriptio map_register(iris_cfg->str2int_map, "ip",TABLE_TYPE_IP); map_register(iris_cfg->str2int_map, "string",TABLE_TYPE_EXPR); + map_register(iris_cfg->str2int_map, "expr",TABLE_TYPE_EXPR); + map_register(iris_cfg->str2int_map, "expr_plus",TABLE_TYPE_EXPR_PLUS); map_register(iris_cfg->str2int_map, "intval",TABLE_TYPE_INTVAL); map_register(iris_cfg->str2int_map, "digest",TABLE_TYPE_DIGEST); + map_register(iris_cfg->str2int_map, "ipv4",4); map_register(iris_cfg->str2int_map, "ipv6",6); @@ -364,7 +367,7 @@ int write_ip_rule(cJSON *region_json,struct iris_description_t *p_iris,const cha return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); } -int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,enum MAAT_TABLE_TYPE table_type,void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -378,6 +381,13 @@ int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const c json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; + if(table_type==TABLE_TYPE_EXPR_PLUS) + { + json_cmd[cmd_cnt].json_string="district"; + json_cmd[cmd_cnt].json_type=cJSON_String; + cmd_cnt++; + } + json_cmd[cmd_cnt].json_string="keywords"; json_cmd[cmd_cnt].json_type=cJSON_String; cmd_cnt++; @@ -598,7 +608,8 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri switch(table_type) { case TABLE_TYPE_EXPR: - ret=write_expr_rule(table_content, p_iris, table_info->table_path, logger); + case TABLE_TYPE_EXPR_PLUS: + ret=write_expr_rule(table_content, p_iris, table_info->table_path,table_type, logger); break; case TABLE_TYPE_IP: ret=write_ip_rule(table_content, p_iris, table_info->table_path, logger); diff --git a/src/entry/map_str2int.cpp b/src/entry/map_str2int.cpp index 07f4b38..8cad88d 100644 --- a/src/entry/map_str2int.cpp +++ b/src/entry/map_str2int.cpp @@ -1,21 +1,35 @@ #include + void map_tmp_free(void* ptr) { free(ptr); } + +long read_map_val(void *data, const uchar *key, uint size, void *user_arg) +{ + if(data!=NULL) + { + *(int*)user_arg=*(int*)data; + return 1; + } + else + { + return 0; + } +} MESA_htable_handle map_create(void) { MESA_htable_handle string2int_map; MESA_htable_create_args_t hargs; memset(&hargs,0,sizeof(hargs)); hargs.thread_safe=1; - hargs.hash_slot_size = 1024*1024; + hargs.hash_slot_size = 4*1024; hargs.max_elem_num = 0; hargs.eliminate_type = HASH_ELIMINATE_ALGO_LRU; hargs.expire_time = 0; hargs.key_comp = NULL; hargs.key2index = NULL; - hargs.recursive = 0; + hargs.recursive = 1; hargs.data_free = map_tmp_free; hargs.data_expire_with_condition = NULL; string2int_map=MESA_htable_create(&hargs, sizeof(hargs)); @@ -31,20 +45,28 @@ int map_register(MESA_htable_handle handle,const char* string,int value) { unsigned int size=strlen(string); unsigned char *key=(unsigned char *)string; - int *data=(int*)malloc(sizeof(int)); int ret=0; + int * data= (int*)malloc(sizeof(int)); *data=value; ret=MESA_htable_add(handle,key,size,data); + if(ret<0) + { + free(data); + } return ret; } int map_str2int(MESA_htable_handle handle,const char* string,int* value) { int *data=NULL; + long cb_ret=0; unsigned int size=strlen(string); - data=(int*)MESA_htable_search(handle,(unsigned char*)string,size); - if(data!=NULL) + + data=(int*)MESA_htable_search_cb(handle,(unsigned char*)string,size, + read_map_val,value,&cb_ret); + +// data=(int*)MESA_htable_search(handle,(unsigned char*)string,size); + if(cb_ret>0) { - *value=*data; return 1; } else diff --git a/test/maat_json.json b/test/maat_json.json index 80bf5c2..4ee7c21 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -188,6 +188,34 @@ ] } ] + }, + { + "compile_id": 128, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_rage": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "group_8", + "regions": [ + { + "table_name": "HTTP_REGION", + "table_type": "expr_plus", + "table_content": { + "district": "URL", + "keywords": "abckkk&123", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] } ], "plugin_table": [ diff --git a/test/maat_test.cpp b/test/maat_test.cpp index f52d2ad..ef1b04e 100644 --- a/test/maat_test.cpp +++ b/test/maat_test.cpp @@ -37,26 +37,273 @@ void Maat_read_entry_finish_cb(void* u_para) { return; } +void print_maat_ret(int ret) +{ + switch(ret) + { + case -1: + printf("scan error.\n"); + break; + case -2: + printf("hit current region,but not hit compile rule.\n"); + break; + case 0: + printf("nothing hit\n"); + break; + default://>0 + printf("hit %d rules\n",ret); + break; + } + return; +} +int test_string_full_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid) +{ + int ret=0; + int table_id=0; + struct Maat_rule_t result[4]; + int found_pos[4]; + const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; + table_id=Maat_table_register(feather,table_name); + if(table_id==-1) + { + printf("Database table %s register failed.\n",table_name); + return -1; + } + + ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), + result,found_pos, 4, + mid, 0); + print_maat_ret(ret); + return ret; +} +int test_intval_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid) +{ + int table_id=0,ret=0; + int scan_val=2015; + struct Maat_rule_t result[4]; + table_id=Maat_table_register(feather,table_name); + if(table_id==-1) + { + printf("Database table %s register failed.",table_name); + + } + else + { + ret=Maat_scan_intval(feather, table_id, scan_val, result,4,mid, 0); + print_maat_ret(ret); + } + return ret; +} +int test_str_stream_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid) +{ + int table_id=0,ret=0; + struct Maat_rule_t result[4]; + const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; + table_id=Maat_table_register(feather,table_name); + if(table_id==-1) + { + printf("Database table %s register failed.\n",table_name); + return -1; + } + struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10); + stream_para_t sp=Maat_stream_scan_string_start(feather,table_id,0); + int detail_ret=0; + if(sp==NULL) + { + printf("stream scan start failed.\n"); + return -1; + } + ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,"www.cyberessays.com", strlen("www.cyberessays.com") + ,result,4,hit_detail,10 + ,&detail_ret,mid); + ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data, strlen(scan_data) + ,result,4,hit_detail,10 + ,&detail_ret,mid); + Maat_stream_scan_string_end(&sp); + free(hit_detail); + print_maat_ret(ret); + return ret; +} +int test_ipv4_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid) +{ + int table_id=0,ret=0; + struct Maat_rule_t result[4]; + struct ipaddr ipv4_addr; + struct stream_tuple4_v4 v4_addr; + ipv4_addr.addrtype=ADDR_TYPE_IPV4; + inet_pton(AF_INET,"10.0.6.205",&(v4_addr.saddr)); + v4_addr.source=htons(50001); + inet_pton(AF_INET,"10.0.6.201",&(v4_addr.daddr)); + v4_addr.dest=htons(80); + ipv4_addr.v4=&v4_addr; + + table_id=Maat_table_register(feather,table_name); + if(table_id==-1) + { + printf("Database table %s register failed.\n",table_name); + + } + else + { + ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,result,4, mid,0); + if(ret>0) + { + printf("ipv4 scan hit compile rule id %d.\n",result[0].config_id); + } + } + return ret; +} +int test_ipv6_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid) +{ + int table_id=0,ret=0; + struct Maat_rule_t result[4]; + struct ipaddr ipv6_addr; + struct stream_tuple4_v6 v6_addr; + + ipv6_addr.addrtype=ADDR_TYPE_IPV6; + inet_pton(AF_INET6,"2001:da8:205:1::101",&(v6_addr.saddr)); + v6_addr.source=htons(50001); + inet_pton(AF_INET6,"2001:da8:205:1::102",&(v6_addr.daddr)); + v6_addr.dest=htons(80); + ipv6_addr.v6=&v6_addr; + + table_id=Maat_table_register(feather,table_name); + if(table_id==-1) + { + printf("Database table %s register failed.\n",table_name); + + } + else + { + ret=Maat_scan_proto_addr(feather,table_id,&ipv6_addr,6,result,4, mid,0); + if(ret==-2) + { + printf("ipv6 scan hit region.\n"); + } + else + { + printf("ipv6 scan result:%d ,shoulde be -2.\n",ret); + + } + } + return ret; +} +int test_digest_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid) +{ + int table_id=0,ret=0; + const char* digest_test_file="./digest_test.data"; + struct stat digest_fstat; + unsigned long long read_size=0,scan_offset=0; + char digest_test_buff[4096]={0}; + + struct Maat_rule_t result[4]; + stream_para_t sp=NULL; + table_id=Maat_table_register(feather, table_name); + if(table_id<0) + { + printf("registe table %s error.\n",table_name); + return 0; + } + ret=stat(digest_test_file,&digest_fstat); + if(ret!=0) + { + printf("fstat %s error.\n",digest_test_file); + return 0; + } + FILE* fp=fopen(digest_test_file,"r"); + if(fp!=NULL) + { + sp=Maat_stream_scan_digest_start(feather, table_id, digest_fstat.st_size, 0); + while(0==feof(fp)) + { + read_size=fread(digest_test_buff,1,sizeof(digest_test_buff),fp); + ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,mid); + scan_offset+=read_size; + if(ret>0) + { + printf("digest scan hit %d.\n",result[0].config_id); + + } + } + fclose(fp); + } + else + { + printf("fopen %s error.\n",digest_test_file); + } + Maat_stream_scan_string_end(&sp); + return ret; +} +int test_plugin_table(Maat_feather_t feather,const char* table_name,void* logger) +{ + int table_id=0,ret=0; + table_id=Maat_table_register(feather,table_name); + if(table_id==-1) + { + printf("Database table %s register failed.\n",table_name); + } + else + { + ret=Maat_table_callback_register(feather, table_id, + Maat_read_entry_start_cb, + Maat_read_entry_cb, + Maat_read_entry_finish_cb, + logger); + if(ret<0) + { + printf("Maat callback register table %s error.\n",table_name); + } + } + return ret; +} +int test_expr_plus(Maat_feather_t feather,const char* table_name,scan_status_t* mid) +{ + int ret=0; + int table_id=0; + struct Maat_rule_t result[4]; + int found_pos[4]; + const char* region_name="URL"; + const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; + table_id=Maat_table_register(feather,table_name); + if(table_id==-1) + { + printf("Database table %s register failed.\n",table_name); + return -1; + } + ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), + result,found_pos, 4, + mid, 0); + if(ret>0) + { + printf("Should not hit without set district.\n"); + return -1; + } + ret=Maat_set_scan_status(feather, mid, MAAT_SET_SCAN_DISTRICT,region_name,strlen(region_name)+1); + if(ret<0) + { + printf("set MAAT_SET_SCAN_DISTRICT failed.\n"); + return -1; + } + ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), + result,found_pos, 4, + mid, 0); + if(ret>0) + { + printf("Hit expr_puls rule %d.\n",result[0].config_id); + } + return ret; + +} int main(int argc,char* argv[]) { Maat_feather_t feather=NULL; - int cb_table_id=-1,url_scan_table_id=-1,size_scan_table_id=-1,ip_scan_table_id=-1,digest_scan_table_id=-1; - int ret=-1; int g_iThreadNum=4; const char* table_info_path="./table_info.conf"; const char* json_path="./maat_json.json"; // const char* ful_cfg_dir="./maat_json.json_iris_tmp/index"; // const char* inc_cfg_dir="./rules/inc/index"; const char* log_file="./test.log"; - const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; - const char* digest_test_file="./digest_test.data"; const char* stat_file="./scan_staus.log"; - struct stat digest_fstat; - unsigned long long read_size=0,scan_offset=0; - char digest_test_buff[4096]={0}; - int scan_val=2015; - struct Maat_rule_t result[4]; - int found_pos[4]; int scan_detail=0; scan_status_t mid=NULL; void *logger=MESA_create_runtime_log_handle(log_file,0); @@ -80,178 +327,30 @@ int main(int argc,char* argv[]) printf("Maat initial error, see %s\n",log_file); return -1; } + test_plugin_table(feather, "QD_ENTRY_INFO",logger); - cb_table_id=Maat_table_register(feather,"QD_ENTRY_INFO"); - if(cb_table_id==-1) - { - printf("Database table QD_ENTRY_INFO register failed."); - } - else - { - ret=Maat_table_callback_register(feather, cb_table_id, - Maat_read_entry_start_cb, - Maat_read_entry_cb, - Maat_read_entry_finish_cb, - logger); - if(ret<0) - { - printf("Maat callback register table QD_ENTRY_INFO error."); - } - } - - url_scan_table_id=Maat_table_register(feather,"HTTP_URL"); - if(url_scan_table_id==-1) - { - printf("Database table HTTP_URL register failed."); - - } - else - { - ret=Maat_full_scan_string(feather, url_scan_table_id,CHARSET_GBK, scan_data, strlen(scan_data), - result,found_pos, 4, - &mid, 0); - switch(ret) - { - case -1: - printf("scan error.\n"); - break; - case -2: - printf("hit current region,but not hit compile rule.\n"); - break; - case 0: - printf("nothing hit\n"); - break; - default://>0 - printf("hit %d rules\n",ret); - break; - } - } + test_string_full_scan(feather, "HTTP_URL", &mid); + //not clean status here, to test_ipv4_scan make hit compile rule. + test_ipv4_scan(feather, "IP_CONFIG", &mid); Maat_clean_status(&mid); - size_scan_table_id=Maat_table_register(feather,"CONTENT_SIZE"); - if(size_scan_table_id==-1) - { - printf("Database table CONTENT_SIZE register failed."); - - } - else - { - ret=Maat_scan_intval(feather, size_scan_table_id, scan_val, result,4,&mid, 0); - switch(ret) - { - case -1: - printf("scan error.\n"); - break; - case -2: - printf("hit current region,but not hit compile rule.\n"); - break; - case 0: - printf("nothing hit\n"); - break; - default://>0 - printf("hit %d rules\n",ret); - break; - } - } - sleep(2); - Maat_clean_status(&mid); - struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10); - stream_para_t sp=Maat_stream_scan_string_start(feather,url_scan_table_id,0); - int detail_ret=0; - if(sp==NULL) - { - printf("stream scan start failed.\n"); - return -1; - } - ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,"www.cyberessays.com", strlen("www.cyberessays.com") - ,result,4,hit_detail,10 - ,&detail_ret,&mid); - ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data, strlen(scan_data) - ,result,4,hit_detail,10 - ,&detail_ret,&mid); - Maat_stream_scan_string_end(&sp); - struct ipaddr ipv4_addr,ipv6_addr; - struct stream_tuple4_v4 v4_addr; - struct stream_tuple4_v6 v6_addr; - ipv4_addr.addrtype=ADDR_TYPE_IPV4; - inet_pton(AF_INET,"10.0.6.205",&(v4_addr.saddr)); - v4_addr.source=htons(50001); - inet_pton(AF_INET,"10.0.6.201",&(v4_addr.daddr)); - v4_addr.dest=htons(80); - ipv4_addr.v4=&v4_addr; - ip_scan_table_id=Maat_table_register(feather,"IP_CONFIG"); - if(ip_scan_table_id==-1) - { - printf("Database table IP_CONFIG register failed."); - - } - else - { - ret=Maat_scan_proto_addr(feather,ip_scan_table_id,&ipv4_addr,6,result,4, &mid,0); - if(ret>0) - { - printf("ipv4 scan hit compile rule id %d.\n",result[0].config_id); - } - } + test_intval_scan(feather,"CONTENT_SIZE" , &mid); Maat_clean_status(&mid); - ipv6_addr.addrtype=ADDR_TYPE_IPV6; - inet_pton(AF_INET6,"2001:da8:205:1::101",&(v6_addr.saddr)); - v6_addr.source=htons(50001); - inet_pton(AF_INET6,"2001:da8:205:1::102",&(v6_addr.daddr)); - v6_addr.dest=htons(80); - ipv6_addr.v6=&v6_addr; - if(ip_scan_table_id>=0) - { - ret=Maat_scan_proto_addr(feather,ip_scan_table_id,&ipv6_addr,6,result,4, &mid,0); - if(ret==-2) - { - printf("ipv6 scan hit region.\n"); - } - else - { - printf("ipv6 scan result:%d ,shoulde be -2.\n",ret); - } - } - Maat_clean_status(&mid); - digest_scan_table_id=Maat_table_register(feather, "FILE_DIGEST"); - if(digest_scan_table_id<0) - { - printf("registe table FILE_DIGEST error.\n"); - return 0; - } - ret=stat(digest_test_file,&digest_fstat); - if(ret!=0) - { - printf("fstat %s error.\n",digest_test_file); - return 0; - } - FILE* fp=fopen(digest_test_file,"r"); - if(fp!=NULL) - { - sp=Maat_stream_scan_digest_start(feather, digest_scan_table_id, digest_fstat.st_size, 0); - while(0==feof(fp)) - { - read_size=fread(digest_test_buff,1,sizeof(digest_test_buff),fp); - ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,&mid); - scan_offset+=read_size; - if(ret>0) - { - printf("digest scan hit %d.\n",result[0].config_id); - } - } - fclose(fp); - } - else - { - printf("fopen %s error.\n",digest_test_file); - } - Maat_stream_scan_string_end(&sp); + test_ipv6_scan(feather, "IP_CONFIG", &mid); Maat_clean_status(&mid); + + test_digest_scan(feather,"FILE_DIGEST", &mid); + Maat_clean_status(&mid); + + test_expr_plus(feather, "HTTP_REGION", &mid); + Maat_clean_status(&mid); + sleep(4); + Maat_burn_feather(feather); - free(hit_detail); + return 0; } diff --git a/test/table_info.conf b/test/table_info.conf index 7ce9a2e..0facdd5 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -1,7 +1,7 @@ #each collumn seperate with '\t' #id (0~65535) #name string -#type one of ip,expr,digest,intval,compile or plugin +#type one of ip,expr,expr_plus,digest,intval,compile or plugin #src_charset one of GBK,BIG5,UNICODE,UTF8 #dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/' #do_merege yes or no @@ -14,3 +14,4 @@ 5 CONTENT_SIZE intval GBK GBK no 0 6 QD_ENTRY_INFO plugin GBK GBK no 0 7 FILE_DIGEST digest GBK GBK no 0 +8 HTTP_REGION expr_plus GBK GBK no 0