diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index ca0de2e..b8d3e66 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -108,17 +108,36 @@ struct Maat_hit_detail_t struct Maat_region_pos_t region_pos[MAAT_MAX_HIT_RULE_NUM]; }; //--------------------HITTING DETAIL DESCRIPTION END + +//Abondon interface ,left for compatible. Maat_feather_t Maat_summon_feather(int max_thread_num, const char* table_info_path, const char* ful_cfg_dir, const char* inc_cfg_dir, - void*logger); - + void*logger);//MESA_handle_logger +//Abondon interface ,left for compatible. Maat_feather_t Maat_summon_feather_json(int max_thread_num, const char* table_info_path, const char* json_rule, void* logger); +Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger); +int Maat_initiate_feather(Maat_feather_t feather); + +enum MAAT_INIT_OPT{ + MAAT_OPT_SCANDIR_INTERVAL_MS=1, //VALUE is interger,SIZE=sizeof(int). DEFAULT:1,000 milliseconds. + MAAT_OPT_EFFECT_INVERVAL_MS, //VALUE is interger,SIZE=sizeof(int). DEFAULT:60,000 milliseconds. + MAAT_OPT_FULL_CFG_DIR, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. + MAAT_OPT_INC_CFG_DIR, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. + MAAT_OPT_JSON_FILE_PATH, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. + MAAT_OPT_STAT_ON, //VALUE is indifferent,SIZE is indifferent.MAAT_OPT_STAT_FILE_PATH must be set.Default: stat OFF. + MAAT_OPT_PERF_ON, //VALUE is indifferent,SIZE is indifferent.MAAT_OPT_STAT_FILE_PATH must be set.Default: stat OFF. + MAAT_OPT_STAT_FILE_PATH, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. + MAAT_OPT_SCAN_DETAIL //VALUE is interger,SIZE=sizeof(int). 0: not return any detail;1: return hit pos, not include regex grouping; + // 2 return hit pos and regex grouping pos;DEFAULT:2 +}; +//return -1 if failed, return 0 on success; +int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size); void Maat_burn_feather(Maat_feather_t feather); //return table_id(>=0) if success,otherwise return -1; @@ -179,10 +198,5 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para); void Maat_clean_status(scan_status_t* mid); -#define MAAT_OPT_SCANDIR_INTERVAL_MS 1 //opt value is interger,size=sizeof(int),1,000 milliseconds as default -#define MAAT_OPT_EFFECT_INVERVAL_MS 2 //opt value is interger,size=sizeof(int),60,000 milliseconds as default -//return -1 if failed, return 0 on success; -int Maat_set_feather_opt(Maat_feather_t feather,int type,void* value,int size); - #endif // H_MAAT_RULE_H_INCLUDE diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 1822130..d87cc88 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -1,6 +1,7 @@ #include #include #include + #include #include #include "rulescan.h" @@ -101,7 +102,6 @@ int region_compile(struct _scan_status_t *_mid,void* region_hit,int region_type_ struct _Maat_compile_rule_t* array_mi_rule[MAX_SCANNER_HIT_NUM]; struct _Maat_compile_rule_t* _mi_rule=NULL; int region_pos[MAX_SCANNER_HIT_NUM]; - _mid->cur_hit_cnt=0; for(i=0;irwlock)); - - } - + } } + if(result_cnt>0) + { + _mid->feather->hit_cnt++; + } return result_cnt; } @@ -320,6 +322,7 @@ struct _scan_status_t* _Maat_make_status(struct _Maat_feather_t* feather,int thr _mid->hit_group_cnt=0; _mid->hit_group_size=4; _mid->hitted_group_id=(unsigned int*)malloc(sizeof(unsigned int)*_mid->hit_group_size); + feather->mid_cnt++; return _mid; } @@ -350,14 +353,8 @@ int detain_last_data(char* buff,int buff_size,int detained_len,const char* data, return ret_len; } - -Maat_feather_t Maat_summon_feather(int max_thread_num, - const char* table_info_path, - const char* ful_cfg_dir, - const char* inc_cfg_dir, - void* logger) +Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger) { - _Maat_feather_t* feather=(_Maat_feather_t*)calloc(sizeof(struct _Maat_feather_t),1); feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path,logger); feather->map_tablename2id=map_create(); @@ -373,65 +370,25 @@ Maat_feather_t Maat_summon_feather(int max_thread_num, map_register(feather->map_tablename2id,feather->p_table_info[i]->table_name,feather->p_table_info[i]->table_id); } } - memcpy(feather->inc_dir,inc_cfg_dir,strlen(inc_cfg_dir)); - memcpy(feather->full_dir,ful_cfg_dir,strlen(ful_cfg_dir)); feather->logger=logger; feather->scan_thread_num=max_thread_num; feather->garbage_q=MESA_lqueue_create(0,0); - config_monitor_traverse(feather->maat_version, - ful_cfg_dir, - maat_start_cb, - maat_update_cb, - maat_finish_cb, - feather, - logger); - if(feather->update_tmp_scanner==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "At initiation: no valid index file in %s",ful_cfg_dir); - } - feather->scanner=feather->update_tmp_scanner; - feather->update_tmp_scanner=NULL; - feather->still_working=1; - if(feather->scanner!=NULL) - { - feather->maat_version=feather->scanner->version; - } feather->effect_interval_ms=60*1000; feather->scan_interval_ms=1*1000; - pthread_t cfg_mon_t; - pthread_create(&cfg_mon_t, NULL, thread_rule_monitor, (void*)feather); return feather; } -Maat_feather_t Maat_summon_feather_json(int max_thread_num, - const char* table_info_path, - const char* json_rule, - void* logger) -{ - Maat_feather_t feather; - char full_index_dir[256]={0}; - int ret=-1; - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "Maat initial with JSON file %s.",json_rule); - - ret=json2iris(json_rule, full_index_dir,sizeof(full_index_dir),logger); - if(ret<0) - { - return NULL; - } - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "generate index file %s OK.",full_index_dir); - feather=Maat_summon_feather(max_thread_num,table_info_path, full_index_dir, full_index_dir,logger); - return feather; -} -int Maat_set_feather_opt(Maat_feather_t feather,int type,void* value,int size) +int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size) { _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - int intval=0; + int intval=0,ret=-1; + if(_feather->still_working==1)// not allowed set after Maat_initiate_feather; + { + return -2; + } switch(type) { case MAAT_OPT_EFFECT_INVERVAL_MS: - intval=*(int*)value; + intval=*(const int*)value; if(size!=sizeof(int)||intval<=0) { return -1; @@ -439,18 +396,168 @@ int Maat_set_feather_opt(Maat_feather_t feather,int type,void* value,int size) _feather->effect_interval_ms=intval; break; case MAAT_OPT_SCANDIR_INTERVAL_MS: - intval=*(int*)value; + intval=*(const int*)value; if(size!=sizeof(int)||intval<0) { return -1; } _feather->scan_interval_ms=intval; break; + case MAAT_OPT_FULL_CFG_DIR: + if(size>(int)sizeof(_feather->full_dir)) + { + return -1; + } + memcpy(_feather->full_dir,(const char*)value,size); + break; + case MAAT_OPT_INC_CFG_DIR: + if(size>(int)sizeof(_feather->inc_dir)) + { + return -1; + } + memcpy(_feather->inc_dir,(const char*)value,size); + break; + case MAAT_OPT_JSON_FILE_PATH: + ret=json2iris((const char*)value, _feather->full_dir,sizeof(_feather->full_dir),_feather->logger); + if(ret<0) + { + return -1; + } + memcpy(_feather->inc_dir,_feather->full_dir,sizeof(_feather->inc_dir)); + MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , + "Maat initial with JSON file %s,generate index file %s OK." + ,(const char*)value + ,_feather->full_dir); + break; + case MAAT_OPT_STAT_ON: + _feather->stat_on=1; + _feather->stat_handle=FS_create_handle(); + break; + case MAAT_OPT_PERF_ON: + _feather->perf_on=1; + break; + case MAAT_OPT_STAT_FILE_PATH: + if(size>(int)sizeof(_feather->stat_file)) + { + return -1; + } + memcpy(_feather->stat_file,(const char*)value,size); + MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , + "Maat performance statistic output to %s." + ,(const char*)value); + _feather->stat_on=1; + break; + case MAAT_OPT_SCAN_DETAIL: + intval=*(const int*)value; + _feather->rule_scan_type=intval; + break; default: return -1; } return 0; } +int Maat_initiate_feather(Maat_feather_t feather) +{ + _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + config_monitor_traverse(_feather->maat_version, + _feather->full_dir, + maat_start_cb, + maat_update_cb, + maat_finish_cb, + _feather, + _feather->logger); + if(_feather->update_tmp_scanner==NULL) + { + MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module , + "At initiation: no valid index file in %s",_feather->full_dir); + } + _feather->scanner=_feather->update_tmp_scanner; + _feather->update_tmp_scanner=NULL; + _feather->still_working=1; + _feather->rule_scan_type=2; + if(_feather->scanner!=NULL) + { + _feather->maat_version=_feather->scanner->version; + } + if(strlen(_feather->stat_file)==0) + { + _feather->perf_on=0; + _feather->stat_on=0; + } + maat_stat_init(_feather); + + pthread_t cfg_mon_t; + pthread_create(&cfg_mon_t, NULL, thread_rule_monitor, (void*)_feather); + return 0; + +} + +Maat_feather_t Maat_summon_feather(int max_thread_num, + const char* table_info_path, + const char* ful_cfg_dir, + const char* inc_cfg_dir, + void* logger) +{ + int ret=-1; + Maat_feather_t feather=NULL; + feather=Maat_feather(max_thread_num,table_info_path,logger); + if(feather==NULL) + { + return NULL; + } + ret=Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, ful_cfg_dir, strlen(ful_cfg_dir)+1); + if(ret<0) + { + goto error_out; + } + ret=Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, inc_cfg_dir, strlen(inc_cfg_dir)+1); + if(ret<0) + { + goto error_out; + } + ret=Maat_initiate_feather(feather); + if(ret<0) + { + goto error_out; + } + return feather; + +error_out: + Maat_burn_feather(feather); + return NULL; + + +} +Maat_feather_t Maat_summon_feather_json(int max_thread_num, + const char* table_info_path, + const char* json_rule, + void* logger) +{ + int ret=-1; + Maat_feather_t feather=NULL; + feather=Maat_feather(max_thread_num,table_info_path,logger); + if(feather==NULL) + { + return NULL; + } + ret=Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, json_rule, strlen(json_rule)+1); + if(ret<0) + { + goto error_out; + } + ret=Maat_initiate_feather(feather); + if(ret<0) + { + goto error_out; + } return feather; + +error_out: + + Maat_burn_feather(feather); + return NULL; + +} + void Maat_burn_feather(Maat_feather_t feather) { _Maat_feather_t* _feather=(_Maat_feather_t*)feather; @@ -530,8 +637,12 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id scan_result_t *region_result=NULL; _compile_result_t compile_result[rule_num];//dynamic array - struct _Maat_table_info_t *p_table=NULL; + struct timeval start,end; + if(_feather->perf_on==1) + { + gettimeofday(&start,NULL); + } p_table=acqurie_table(_feather, table_id,TABLE_TYPE_EXPR); if(p_table==NULL) { @@ -587,6 +698,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } if(hit_region_cnt>0) { + p_table->hit_cnt++; if(*mid==NULL) { _mid=_Maat_make_status(_feather,thread_num); @@ -609,6 +721,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } } DEC_SCANNER_REF(my_scanner, thread_num); + if(_feather->perf_on==1) + { + gettimeofday(&end,NULL); + maat_stat_table(p_table,data_len,&start, &end); + } if(compile_ret==0&&hit_region_cnt>0) { return -2; @@ -643,6 +760,11 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); intval_scan_data.int_data=intval; _Maat_table_info_t* p_table=NULL; + struct timeval start,end; + if(_feather->perf_on==1) + { + gettimeofday(&start,NULL); + } p_table=acqurie_table(_feather,table_id,TABLE_TYPE_INTVAL); if(p_table==NULL) { @@ -669,6 +791,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id } else if(region_ret>0) { + p_table->hit_cnt++; if(*mid==NULL) { _mid=_Maat_make_status(_feather,thread_num); @@ -685,6 +808,11 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id } DEC_SCANNER_REF(my_scanner,thread_num); + if(_feather->perf_on==1) + { + gettimeofday(&end,NULL); + maat_stat_table(p_table,0,&start, &end); + } if(compile_ret==0&®ion_ret>0) { return -2; @@ -707,6 +835,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_scanner_t* my_scanner=NULL; + struct timeval start,end; + if(_feather->perf_on==1) + { + gettimeofday(&start,NULL); + } p_table=acqurie_table(_feather, table_id, TABLE_TYPE_IP); if(p_table==NULL) { @@ -758,6 +891,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id } else if(region_ret>0) { + p_table->hit_cnt++; if(*mid==NULL) { _mid=_Maat_make_status(_feather,thread_num); @@ -773,7 +907,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id result,compile_result,rule_num); } DEC_SCANNER_REF(my_scanner,thread_num); - + if(_feather->perf_on==1) + { + gettimeofday(&end,NULL); + maat_stat_table(p_table,0,&start, &end); + } if(compile_ret==0&®ion_ret>0) { return -2; @@ -834,6 +972,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, { sp->do_regex=1; } + p_table->stream_num++; sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num); return sp; } @@ -852,6 +991,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para scan_result_t *region_result; _compile_result_t compile_result[rule_num];//dynamic array scan_data_t region_scan_data; + struct timeval start,end; + if(sp->feather->perf_on==1) + { + gettimeofday(&start,NULL); + } if(data==NULL||data_len==0) { return 0; @@ -939,6 +1083,8 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para } if(hit_region_cnt>0) { + sp->feather->p_table_info[sp->table_id]->hit_cnt++; + if(*mid==NULL) { _mid=_Maat_make_status(sp->feather,sp->thread_num); @@ -975,6 +1121,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para free(sp->scan_buff); sp->scan_buff=0; } + if(sp->feather->perf_on==1) + { + gettimeofday(&end,NULL); + maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end); + } if(compile_ret==0&&hit_region_cnt>0) { return -2; @@ -998,7 +1149,8 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _Maat_scanner_t* scanner=sp->feather->scanner; - + struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; + p_table->stream_num--; if(scanner!=NULL) { if(sp->version==sp->feather->maat_version) @@ -1056,6 +1208,8 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, sp->total_len=total_len; sp->fuzzy_hash_handle=tmp_fuzzy_handle; pthread_mutex_init(&(sp->fuzzy_mutex),NULL); + p_table->stream_num++; + return sp; } @@ -1092,6 +1246,11 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int char* digest_buff=NULL; struct _scan_status_t* _mid=(struct _scan_status_t*)(*mid); pthread_rwlock_t *GIE_rwlock=&(sp->feather->scanner->digest_rwlock[sp->table_id]); + struct timeval start,end; + if(sp->feather->perf_on==1) + { + gettimeofday(&start,NULL); + } if(sp->acc_scan_len+(unsigned long long)data_len > sp->total_len) { return 0; @@ -1129,24 +1288,30 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { return -1; } - if(hit_region_cnt==0) + if(hit_region_cnt>0) { - return 0; + sp->feather->p_table_info[sp->table_id]->hit_cnt++; + if(*mid==NULL) + { + _mid=_Maat_make_status(sp->feather,sp->thread_num); + *mid=_mid; + } + else + { + _mid=(struct _scan_status_t*)(*mid); + } + compile_ret=region_compile(_mid, + query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag), + hit_region_cnt, + result,compile_result,rule_num); + + } + if(sp->feather->perf_on==1) + { + gettimeofday(&end,NULL); + maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end); } - if(*mid==NULL) - { - _mid=_Maat_make_status(sp->feather,sp->thread_num); - *mid=_mid; - } - else - { - _mid=(struct _scan_status_t*)(*mid); - } - compile_ret=region_compile(_mid, - query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag), - hit_region_cnt, - result,compile_result,rule_num); if(compile_ret==0&&hit_region_cnt>0) { return -2; @@ -1157,7 +1322,8 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _Maat_scanner_t* scanner=sp->feather->scanner; - + struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; + p_table->stream_num--; if(scanner!=NULL) { if(sp->version==sp->feather->maat_version) @@ -1171,6 +1337,7 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para) assert(sp->scan_buff==NULL); free(sp); *stream_para=NULL; + return; } void Maat_clean_status(scan_status_t* mid) @@ -1181,6 +1348,7 @@ void Maat_clean_status(scan_status_t* mid) return; } _mid=(struct _scan_status_t*)(*mid); + _mid->feather->mid_cnt--; free(_mid->hitted_group_id); free(_mid); *mid=NULL; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 5a5585f..d47debd 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "Maat_rule.h" #include "Maat_rule_internal.h" @@ -24,11 +25,10 @@ #include "mesa_fuzzy.h" #include "great_index_engine.h" -int MAAT_FRAME_VERSION_1_3_20151224=1; +int MAAT_FRAME_VERSION_1_4_20160210=1; const char *maat_module="MAAT Frame"; const char* CHARSET_STRING[]={"CHARSET_NONE","GBK","BIG5","UNICODE","UTF-8"}; - int converHextoint(char srctmp) { if(isdigit(srctmp)) @@ -673,7 +673,7 @@ void destroy_digest_rule(GIE_digest_t*rule) rule=NULL; return; } -struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread_num,MESA_lqueue_head tomb) +struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread_num,MESA_lqueue_head tomb,int rs_scan_type) { int i=0; @@ -714,6 +714,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread scanner->ref_cnt=(int*)calloc(CPU_CACHE_ALIGMENT,scan_thread_num); scanner->region_update_q=MESA_lqueue_create(0,0); scanner->region=rulescan_initialize(scan_thread_num); + rulescan_set_param(scanner->region,rs_scan_type); scanner->tomb_ref=tomb; scanner->region_rslt_buff=(scan_result_t*)malloc(sizeof(scan_result_t)*MAX_SCANNER_HIT_NUM*scan_thread_num); @@ -2191,7 +2192,7 @@ void maat_start_cb(unsigned int new_version,int update_type,void*u_para) int i=0,j=0; if(update_type==CM_UPDATE_TYPE_FULL) { - feather->update_tmp_scanner=create_maat_scanner(new_version,feather->scan_thread_num,feather->garbage_q); + feather->update_tmp_scanner=create_maat_scanner(new_version,feather->scan_thread_num,feather->garbage_q,feather->rule_scan_type); MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, "Full config version %u -> %u update start", feather->maat_version,new_version); @@ -2375,12 +2376,17 @@ void *thread_rule_monitor(void *arg) } } garbage_bury(feather->garbage_q,feather->logger); + if(feather->stat_on==1) + { + maat_stat_output(feather); + } } MESA_htable_destroy(feather->map_tablename2id,free); destroy_maat_scanner(feather->scanner); garbage_bury(feather->garbage_q,feather->logger); MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL); + FS_stop(&(feather->stat_handle)); int i=0,j=0; struct dynamic_array_t* d_array=NULL; @@ -2406,3 +2412,4 @@ void *thread_rule_monitor(void *arg) free(feather); return NULL; } + diff --git a/src/entry/Maat_rule_internal.h b/src/entry/Maat_rule_internal.h index 0204cec..89dd64f 100644 --- a/src/entry/Maat_rule_internal.h +++ b/src/entry/Maat_rule_internal.h @@ -5,6 +5,7 @@ #include #include +#include #include "dynamic_array.h" #include "UniversalBoolMatch.h" #include "rulescan.h" @@ -47,6 +48,7 @@ typedef int atomic_t; #define MAX_FAILED_NUM 128 +#define MAX_MAAT_STAT_NUM 64 #ifndef MAX #define MAX(a, b) (((a) > (b)) ? (a) : (b)) #endif @@ -91,6 +93,7 @@ enum MAAT_MATCH_METHOD MATCH_METHOD_LEFT, MATCH_METHOD_FULL }; + struct db_str_rule_t { int region_id; @@ -224,6 +227,13 @@ struct _Maat_table_info_t int expr_rule_cnt; //expr_type=0,1,3 int regex_rule_cnt; //expr_type=2 struct _plugin_table_info *cb_info; +//for stat>>>>>>>> + int stat_line_id; + long long scan_cnt; + long long scan_cpu_time; //microseconds + long long input_bytes; + long long stream_num; + long long hit_cnt; }; struct _scan_status_t { @@ -294,17 +304,28 @@ struct _Maat_feather_t struct _Maat_scanner_t *update_tmp_scanner; MESA_lqueue_head garbage_q; int table_cnt; + int GROUP_MODE_ON; + int still_working; + int scan_interval_ms; + int effect_interval_ms; + int stat_on; + int perf_on; struct _Maat_table_info_t *p_table_info[MAX_TABLE_NUM]; MESA_htable_handle map_tablename2id; void* logger; int maat_version; int scan_thread_num; + int rule_scan_type; char inc_dir[MAX_TABLE_NAME_LEN]; char full_dir[MAX_TABLE_NAME_LEN]; - int GROUP_MODE_ON; - int still_working; - int scan_interval_ms; - int effect_interval_ms; + char stat_file[MAX_TABLE_NAME_LEN]; +//for stat>>>> + screen_stat_handle_t stat_handle; + int total_stat_id; + int fs_status_id[MAX_MAAT_STAT_NUM]; + int fs_column_id[MAX_MAAT_STAT_NUM]; + long long mid_cnt; + long long hit_cnt; }; struct _maat_garbage_t { @@ -338,5 +359,11 @@ inline void ipv6_ntoh(unsigned int *v6_addr) } return; } +void maat_stat_init(struct _Maat_feather_t* feather); +void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timeval* start, struct timeval* end); +void maat_stat_output(struct _Maat_feather_t* feather); + + + #endif diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp new file mode 100644 index 0000000..cb25e3d --- /dev/null +++ b/src/entry/Maat_stat.cpp @@ -0,0 +1,206 @@ +#include "Maat_rule_internal.h" +#include "field_stat.h" +enum MAAT_FS_STATUS{ + STATUS_VERSION=0, + STATUS_TABLE_NUM, + STATUS_MID_NUM, + STATUS_GARBAGE_QSIZE, +}; + +enum MAAT_FS_COLUMN +{ + COLUMN_TABLE_RULE_NUM=0, + COLUMN_TABLE_REGEX_NUM, + COLUMN_TABLE_STREAM_NUM, + COLUMN_TABLE_SCAN_CNT, + COLUMN_TABLE_SCAN_BYTES, + COLUMN_TABLE_CPU_TIME,//microseconds + COLUMN_TABLE_HIT_CNT +}; +void maat_stat_init(struct _Maat_feather_t* feather) +{ + int value=0; + int i=0; + struct _Maat_table_info_t* p_table=NULL; + + feather->stat_handle=FS_create_handle(); + FS_set_para(feather->stat_handle, OUTPUT_DEVICE, feather->stat_file, strlen(feather->stat_file)+1); + value=1; + FS_set_para(feather->stat_handle, PRINT_MODE, &value, sizeof(value)); + value=0; + FS_set_para(feather->stat_handle, CREATE_THREAD, &value, sizeof(value)); + + feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"version"); + feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"table_num"); + feather->fs_status_id[STATUS_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"mid_num"); + feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"garbage_num"); + + feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule"); + feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"regex"); + feather->fs_column_id[COLUMN_TABLE_STREAM_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"stream"); + feather->fs_column_id[COLUMN_TABLE_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"scan_cps"); + feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"IN_Bps"); + if(feather->perf_on==1) + { + feather->fs_column_id[COLUMN_TABLE_CPU_TIME]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"cpu_us"); + value=feather->fs_column_id[COLUMN_TABLE_CPU_TIME]; + FS_set_para(feather->stat_handle, ID_INVISBLE, &value, sizeof(value)); + FS_register_ratio(feather->stat_handle, + feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], + feather->fs_column_id[COLUMN_TABLE_CPU_TIME], + FS_STYLE_COLUMN, + FS_CALC_SPEED, + "PROC_Bps"); + } + feather->fs_column_id[COLUMN_TABLE_HIT_CNT]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"hit_cnt"); + value=feather->fs_column_id[COLUMN_TABLE_HIT_CNT]; + FS_set_para(feather->stat_handle, ID_INVISBLE, &value, sizeof(value)); + FS_register_ratio(feather->stat_handle, + feather->fs_column_id[COLUMN_TABLE_HIT_CNT], + feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], + FS_STYLE_COLUMN, + FS_CALC_SPEED, + "hit_rate"); + feather->total_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT,"Sum"); + for(i=0;ip_table_info[i]; + if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN + ||p_table->table_type==TABLE_TYPE_GROUP + ||p_table->table_type==TABLE_TYPE_COMPILE) + { + continue; + } + p_table->stat_line_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT,p_table->table_name); + } + FS_start(feather->stat_handle); + return; +} +void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timeval* start, struct timeval* end) +{ + p_table->scan_cnt++; + p_table->input_bytes+=scan_len; + p_table->scan_cpu_time+=(end->tv_sec-start->tv_sec)*1000000+end->tv_usec-start->tv_usec; + return; +} +void maat_stat_output(struct _Maat_feather_t* feather) +{ + long value=0; + long long total_cfg_num=0, total_input_bytes=0, total_regex_num=0; + long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0; + int i=0; + struct _Maat_table_info_t* p_table=NULL; + + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_MID_NUM], 0,FS_OP_SET,feather->mid_cnt); + + value=MESA_lqueue_get_count(feather->garbage_q); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value); + + for(i=0;ip_table_info[i]; + if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN + ||p_table->table_type==TABLE_TYPE_GROUP + ||p_table->table_type==TABLE_TYPE_COMPILE) + { + continue; + } + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_RULE_NUM], + FS_OP_SET, + p_table->cfg_num); + total_cfg_num+=p_table->cfg_num; + + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_REGEX_NUM], + FS_OP_SET, + p_table->regex_rule_cnt); + total_regex_num+= p_table->regex_rule_cnt; + + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], + FS_OP_SET, + p_table->stream_num); + total_stream_cnt+= p_table->stream_num; + + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], + FS_OP_ADD, + p_table->scan_cnt); + total_scan_cnt+=p_table->scan_cnt; + p_table->scan_cnt=0; + + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], + FS_OP_ADD, + p_table->input_bytes); + total_input_bytes+=p_table->input_bytes; + p_table->input_bytes=0; + if(feather->perf_on==1) + { + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_CPU_TIME], + FS_OP_ADD, + p_table->scan_cpu_time); + total_cpu_time+=p_table->scan_cpu_time; + p_table->scan_cpu_time=0; + } + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_HIT_CNT], + FS_OP_ADD, + p_table->hit_cnt); + //total hit count stat in region_compile + p_table->hit_cnt=0; + } + FS_operate(feather->stat_handle, + feather->total_stat_id, + feather->fs_column_id[COLUMN_TABLE_RULE_NUM], + FS_OP_SET, + total_cfg_num); + FS_operate(feather->stat_handle, + feather->total_stat_id, + feather->fs_column_id[COLUMN_TABLE_REGEX_NUM], + FS_OP_SET, + total_regex_num); + FS_operate(feather->stat_handle, + feather->total_stat_id, + feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], + FS_OP_SET, + total_stream_cnt); + FS_operate(feather->stat_handle, + feather->total_stat_id, + feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], + FS_OP_ADD, + total_scan_cnt); + FS_operate(feather->stat_handle, + feather->total_stat_id, + feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], + FS_OP_ADD, + total_input_bytes); + if(feather->perf_on==1) + { + FS_operate(feather->stat_handle, + feather->total_stat_id, + feather->fs_column_id[COLUMN_TABLE_CPU_TIME], + FS_OP_ADD, + total_cpu_time); + } + FS_operate(feather->stat_handle, + feather->total_stat_id, + feather->fs_column_id[COLUMN_TABLE_HIT_CNT], + FS_OP_ADD, + feather->hit_cnt); + feather->hit_cnt=0; + FS_passive_output(feather->stat_handle); + return; +} + diff --git a/src/entry/Makefile b/src/entry/Makefile index 539f22a..8a8fcd7 100644 --- a/src/entry/Makefile +++ b/src/entry/Makefile @@ -6,7 +6,7 @@ CCC = g++ CFLAGS = -Wall -g -fPIC CFLAGS += $(OPTFLAGS) LDDICTATOR = -Wl,-wrap,malloc -Wl,-wrap,calloc -Wl,-wrap,free -Wl,-wrap,realloc -LDFLAGS = -lMESA_handle_logger -lMESA_htable -lpthread -lm -lrulescan -lpcre +LDFLAGS = -lMESA_handle_logger -lMESA_htable -lpthread -lm -lrulescan -lpcre -lMESA_field_stat #LDFLAGS += $(LDDICTATOR) MAILLIB = ../lib @@ -15,8 +15,8 @@ H_DIR =-I$(G_H_DIR) -I../../inc LIBMAAT = libmaatframe.a LIBMAAT_SO = libmaatframe.so -OBJS=config_monitor.o Maat_rule.o Maat_api.o UniversalBoolMatch.o dynamic_array.o cJSON.o json2iris.o map_str2int.o\ - interval_index.o great_index_engine.o mesa_fuzzy.o +OBJS=config_monitor.o Maat_rule.o Maat_api.o Maat_stat.o UniversalBoolMatch.o dynamic_array.o cJSON.o\ + json2iris.o map_str2int.o interval_index.o great_index_engine.o mesa_fuzzy.o .c.o: $(CC) -c $(CFLAGS) -I. $(H_DIR) $< diff --git a/src/inc_internal/field_stat.h b/src/inc_internal/field_stat.h new file mode 100644 index 0000000..07e8a5f --- /dev/null +++ b/src/inc_internal/field_stat.h @@ -0,0 +1,70 @@ +#ifndef H_SCREEN_STAT_H_INCLUDE +#define H_SCREEN_STAT_H_INCLUDE +#include + +#ifndef __cplusplus +#error("This file should be compiled with C++ compiler") +#endif + +enum field_dsp_style_t +{ + FS_STYLE_FIELD=0, + FS_STYLE_COLUMN, + FS_STYLE_LINE, + FS_STYLE_STATUS +}; +enum field_calc_algo +{ + FS_CALC_CURRENT=0, + FS_CALC_SPEED +}; +enum field_op +{ + FS_OP_ADD=1, + FS_OP_SET +}; + + +typedef void* screen_stat_handle_t; + +enum FS_option +{ + OUTPUT_DEVICE, //VALUE is a const char*, indicate a file path string, SIZE = strlen(string+'\0')+1.DEFAULT:output to stdout. + PRINT_MODE, //VALUE is an interger,1:Rewrite ,2: Append. SIZE=4,DEFALUT:REWRITE. + STAT_CYCLE, //VALUE is an interger idicate interval seconds of every output, SIZE=4 ,DEFUALT:2 seconds. + PRINT_TRIGGER, //VALUE is an interger,1:Do print,0: Don't print.SIZE=4.DEFAULT:1. + CREATE_THREAD,//VALUE is an interger,1: Create a print thread,0:not create,output by call passive_output function, + //and the STAT_CYCLE is meaningless.SIZE=4,DEFAULT:0. + ID_INVISBLE,//value is field_id/status_id/column_id, not output this string, SIZE=4,DEFAULT: shutdown NO one. +}; + +//Always success. +screen_stat_handle_t FS_create_handle(void); + +int FS_set_para(screen_stat_handle_t handle, enum FS_option type,const void* value,int size); +void FS_start(screen_stat_handle_t handle); +void FS_stop(screen_stat_handle_t* handle); + +//return field_id/line_id/column_id greater than zero if success,return an interger less than zero if failed. +int FS_register(screen_stat_handle_t handle,enum field_dsp_style_t style,enum field_calc_algo calc_type,const char* name); + +//numerator_id and denominator_id must be column/field/status style. +int FS_register_ratio(screen_stat_handle_t handle,int numerator_id,int denominator_id,enum field_dsp_style_t style,enum field_calc_algo calc_type,const char* name); + +//id: when id's type is FIELD , column_id is ignore. +int FS_operate(screen_stat_handle_t handle,int id,int column_id,enum field_op op,long long value); + +void FS_passive_output(screen_stat_handle_t handle); + +screen_stat_handle_t init_screen_stat(FILE* output_fp,int stat_cycle,int screen_print_trigger); + +//return field_id >=0 when success, return -1 when failed. +int stat_field_register(screen_stat_handle_t handle,const char* field_name); + +//return >=0 when success, return -1 when failed. +#define FS_OP_TYPE_ADD (FS_OP_ADD) +#define FS_OP_TYPE_SET (FS_OP_SET) +int stat_field_operation(screen_stat_handle_t handle,int field_id,int operation,long long value); + +#endif + diff --git a/src/inc_internal/rulescan.h b/src/inc_internal/rulescan.h index 9230e64..8ee4b80 100644 --- a/src/inc_internal/rulescan.h +++ b/src/inc_internal/rulescan.h @@ -23,10 +23,21 @@ extern "C" { #endif + /* 定义不同的扫描类型 */ + enum ScanType + { + SCANTYPE_DEFAULT = 0, + SCANTYPE_DETAIL_RESULT = 1, + SCANTYPE_REGEX_GROUP = 2 + }; + + #define MAX_REGEX_GROUP_NUM 5 /* 对于正则表达式,所支持的最大分组的个数 */ + #define MAX_EXPR_ITEM_NUM (1U<<3) /* 每条与表达式最多由MAX_EXPR_ITEM_NUM个规则组成 */ #define MAX_MATCH_POS_NUM 1024 /* 每条规则最多允许返回的命中位置的个数 */ + #define MATCH_POS_NUM_INC 64 /* 每条规则允许返回的命中位置的个数初始值与增量值 */ - /* 定义不同的规则类型 */ + /* 定义不同的规则类型 */ const unsigned int RULETYPE_STR = 0; /* 字符串或二进制规则 */ const unsigned int RULETYPE_REG = 1; /* 正则表达式规则 */ const unsigned int RULETYPE_INT = 2; /* 数值区间规则 */ @@ -195,6 +206,17 @@ extern "C" */ void * rulescan_initialize(unsigned int max_thread_num); + /* + 功能: + 设置扫描参数,本函数在rulescan_update之前可多次调用,每次设置一种扫描类型 + 参数: + instance[in]: 扫描器对象指针; + scan_type_flag[in]: 扫描类型的标志信息,0代表不返回位置等信息;1代表返回位置等信息,但是不返回正则分组信息;2代表返回位置信息和正则分组信息 + 返回值: + 1:正确设置,-1:设置失败。 + */ + int rulescan_set_param(void * instance, int scan_type_flag); + /* 功能:动态注册一组与表达式,更新扫描器对象。对于同一个instance句柄,不允许同时有多个更新线程。 参数: diff --git a/test/maat_test.cpp b/test/maat_test.cpp index 4be5a90..f52d2ad 100644 --- a/test/maat_test.cpp +++ b/test/maat_test.cpp @@ -13,6 +13,7 @@ #include #include //fstat #include //fstat +#include void Maat_read_entry_start_cb(int update_type,void* u_para) { @@ -36,7 +37,7 @@ void Maat_read_entry_finish_cb(void* u_para) { return; } -int main() +int main(int argc,char* argv[]) { Maat_feather_t feather=NULL; int cb_table_id=-1,url_scan_table_id=-1,size_scan_table_id=-1,ip_scan_table_id=-1,digest_scan_table_id=-1; @@ -49,20 +50,31 @@ int main() const char* log_file="./test.log"; const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; const char* digest_test_file="./digest_test.data"; + const char* stat_file="./scan_staus.log"; struct stat digest_fstat; unsigned long long read_size=0,scan_offset=0; char digest_test_buff[4096]={0}; int scan_val=2015; struct Maat_rule_t result[4]; int found_pos[4]; + int scan_detail=0; scan_status_t mid=NULL; - void *logger=MESA_create_runtime_log_handle(log_file,0); - feather=Maat_summon_feather_json(g_iThreadNum, + +/* feather=Maat_summon_feather_json(g_iThreadNum, table_info_path, json_path, logger); +*/ // feather=Maat_summon_feather(g_iThreadNum,table_info_path,ful_cfg_dir,inc_cfg_dir,logger); + feather=Maat_feather(g_iThreadNum, table_info_path, logger); + Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, json_path, sizeof(json_path)+1); + Maat_set_feather_opt(feather, MAAT_OPT_STAT_FILE_PATH, stat_file, strlen(stat_file)); + Maat_set_feather_opt(feather, MAAT_OPT_STAT_ON, NULL, 0); + Maat_set_feather_opt(feather, MAAT_OPT_PERF_ON, NULL, 0); + Maat_set_feather_opt(feather, MAAT_OPT_SCAN_DETAIL, &scan_detail, sizeof(scan_detail)); + Maat_initiate_feather(feather); + if(feather==NULL) { printf("Maat initial error, see %s\n",log_file); @@ -140,6 +152,7 @@ int main() break; } } + sleep(2); Maat_clean_status(&mid); struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10); stream_para_t sp=Maat_stream_scan_string_start(feather,url_scan_table_id,0); @@ -202,7 +215,6 @@ int main() } } Maat_clean_status(&mid); - digest_scan_table_id=Maat_table_register(feather, "FILE_DIGEST"); if(digest_scan_table_id<0) { @@ -238,6 +250,7 @@ int main() } Maat_stream_scan_string_end(&sp); Maat_clean_status(&mid); + sleep(4); Maat_burn_feather(feather); free(hit_detail); return 0;