diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index fe5732e..090c194 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -1,9 +1,17 @@ #include #include +#include +#include + #include "rulescan.h" #include "UniversalBoolMatch.h" #include "Maat_rule.h" #include "Maat_rule_internal.h" +#include "dynamic_array.h" +#include "config_monitor.h" +#include "map_str2int.h" +#include "rulescan.h" +#include "json2iris.h" struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type) { @@ -35,6 +43,7 @@ inline void DEC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num) scanner->ref_cnt[offset]--; return; } + //return 1 if insert a unique id //return 0 if id is duplicated //return -1 if set is full @@ -648,11 +657,11 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; - INC_SCANNER_REF(my_scanner, int thread_num); + INC_SCANNER_REF(my_scanner,thread_num); region_ret=rulescan_search(my_scanner->region, thread_num, &intval_scan_data, region_result, MAX_SCANNER_HIT_NUM); if(region_ret<0) { - DEC_SCANNER_REF(my_scanner, int thread_num); + DEC_SCANNER_REF(my_scanner, thread_num); return -1; } else if(region_ret>0) @@ -672,7 +681,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id result,compile_result,rule_num); } - DEC_SCANNER_REF(my_scanner, int thread_num); + DEC_SCANNER_REF(my_scanner,thread_num); if(compile_ret==0&®ion_ret>0) { return -2; @@ -691,7 +700,6 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id scan_data_t ip_scan_data; scan_result_t *region_result=NULL; _compile_result_t compile_result[rule_num]; - int table_cfg_num=0; _Maat_table_info_t* p_table=NULL; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; @@ -946,8 +954,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para if(sp->scan_buff!=NULL) { *detail_ret=fill_region_hit_detail(sp->scan_buff,_mid, - region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), - hit_region_cnt, + region_result,hit_region_cnt, compile_result,compile_ret, hit_detail,detail_num); } @@ -1021,10 +1028,10 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, struct _Maat_scanner_t* scanner=NULL; struct _Maat_table_info_t *p_table=NULL; - p_table=acqurie_table(_feather, table_id, thread_num, TABLE_TYPE_EXPR); + p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR); if(p_table==NULL) { - return -1; + return NULL; } struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1); scanner=_feather->scanner; @@ -1045,15 +1052,15 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, inline int REACH_QUERY_THRESH(unsigned long long total_len,unsigned long long acc_len,unsigned char* query_point,int point_size) { - const int QUERY_MIN_RATE=(3); //30% - const int QUERY_MIN_LEN=(1024*1024*4); + const unsigned long long QUERY_MIN_RATE=(3); //30% +// const unsigned long long QUERY_MIN_LEN=(1024*1024*4); //do query every 10 percent since 30%, e.g. 0.3/0.4/0.5/.../1.0 unsigned long long rate=(acc_len*10)/total_len; // if(acc_len>QUERY_MIN_LEN) // { // return 1; // } - assert(rate=QUERY_MIN_RATE&&query_point[rate-QUERY_MIN_RATE]==0) { query_point[rate-QUERY_MIN_RATE]=1; @@ -1072,9 +1079,9 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int unsigned long long digest_len=0; char* digest_buff=NULL; struct _scan_status_t* _mid=(struct _scan_status_t*)(*mid); - pthread_rwlock_t GIE_rwlock=&(sp->feather->scanner->digest_rwlock[sp->table_id]); + pthread_rwlock_t *GIE_rwlock=&(sp->feather->scanner->digest_rwlock[sp->table_id]); sp->acc_scan_len+=fuzzy_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset); - do_query=REACH_QUERY_THRESH(sp->total_len, sp->acc_len, sp->query_point,8); + do_query=REACH_QUERY_THRESH(sp->total_len, sp->acc_scan_len, sp->query_point,8); if(do_query==0) { return 0; @@ -1088,7 +1095,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int fuzzy_digest(sp->fuzzy_hash_handle,digest_buff, digest_len); if(0==pthread_rwlock_tryrdlock(GIE_rwlock)) { - hit_region_cnt=GIE_query(GIE_handle, sp->origin_len, digest_buff, query_result, MAX_SCANNER_HIT_NUM); + hit_region_cnt=GIE_query(GIE_handle, sp->total_len, digest_buff, query_result, MAX_SCANNER_HIT_NUM); pthread_rwlock_unlock(GIE_rwlock); } free(digest_buff); diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index d242208..3c3dbed 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -17,12 +17,16 @@ #include "json2iris.h" #include "dynamic_array.h" #include "config_monitor.h" + #include "map_str2int.h" #include "rulescan.h" #include "UniversalBoolMatch.h" +#include "mesa_fuzzy.h" +#include "great_index_engine.h" int MAAT_FRAME_VERSION_1_2_20151103=1; const char *maat_module="MAAT Frame"; + const char* CHARSET_STRING[]={"CHARSET_NONE","GBK","BIG5","UNICODE","UTF-8"}; int converHextoint(char srctmp) @@ -214,15 +218,6 @@ int cnt_maskbits(struct in6_addr mask) } return bits_cnt; } -inline void ipv6_ntoh(unsigned int *v6_addr) -{ - unsigned int i=0; - for(i=0;i<4;i++) - { - v6_addr[i]=ntohl(v6_addr[i]); - } - return; -} int aligment_int_array_sum(int * array,int size) { int sum=0,i=0; @@ -635,7 +630,35 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule) op_expr->p_expr->rnum++; return; } +GIE_digest_t* create_digest_rule(int id,short op,unsigned long long origin_len,const char* digest, + short cfds_lvl,struct _Maat_group_rule_t* tag) +{ + GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1); + int digest_len=strlen(digest); + rule->id=id; + rule->operation=op; + rule->origin_len=origin_len; + if(digest!=NULL) + { + rule->fuzzy_hash=(char*)calloc(sizeof(char),digest_len+1); + memcpy(rule->fuzzy_hash,digest,digest_len); + } + rule->cfds_lvl=cfds_lvl; + rule->tag=(void*)tag; + return rule; +} +void destroy_digest_rule(GIE_digest_t*rule) +{ + if(rule->fuzzy_hash!=NULL) + { + free(rule->fuzzy_hash); + rule->fuzzy_hash=NULL; + } + free(rule); + rule=NULL; + return; +} struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread_num,MESA_lqueue_head tomb) { int i=0; @@ -683,7 +706,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread for(i=0;idigest_update_q[i]=MESA_lqueue_create(0,0); - pthread_rwlock_init(&(scanner->digest_rwlock),NULL); + pthread_rwlock_init(&(scanner->digest_rwlock[i]),NULL); } return scanner; } @@ -740,14 +763,14 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner) destroy_digest_rule(digest_rule); } MESA_lqueue_destroy(scanner->digest_update_q[i], lqueue_destroy_cb, NULL); - pthread_rwlock_destroy(&(scanner->digest_rwlock)); + pthread_rwlock_destroy(&(scanner->digest_rwlock[i])); } free(scanner); return; } -inline unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge) +unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge) { unsigned int sub_type=0; if(do_charset_merge==TRUE) @@ -1938,35 +1961,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line return; } -GIE_digest_t* create_digest_rule(int id,short op,unsigned long long origin_len,const char* digest, - short cfds_lvl,struct _Maat_group_rule_t* tag) -{ - GIE_digest_t* rule=calloc(sizeof(struct GIE_digest_t),1); - int digest_len=strlen(digest); - rule->id=id; - rule->operation=op; - rule->origin_len=origin_len; - if(digest!=NULL) - { - rule->fuzzy_hash=calloc(sizeof(char),digest_len+1); - memcpy(rule->fuzzy_hash,digest,digest_len); - } - rule->cfds_lvl=cfds_lvl; - rule->tag=(void*)tag; - return rule; -} -void destroy_digest_rule(GIE_digest_t*rule) -{ - if(rule->fuzzy_hash!=NULL) - { - free(rule->fuzzy_hash); - rule->fuzzy_hash=NULL; - } - free(rule); - rule=NULL; - return; -} void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { struct db_digest_rule_t* digest_rule=(struct db_digest_rule_t*)calloc(sizeof(struct db_digest_rule_t),1); @@ -2018,7 +2013,7 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line, if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "duplicate config of intval table %s config_id=%d",table->table_name,intval_rule->region_id); + "duplicate config of intval table %s config_id=%d",table->table_name,digest_rule->region_id); } else { @@ -2129,8 +2124,8 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_ int i=0; long q_cnt; GIE_create_para_t para; - para->index_interval=100; - para->query_accuracy=0.1; + para.index_interval=100; + para.query_accuracy=0.1; tmp1=create_bool_matcher(scanner->compile_hash, scan_thread_num, logger); @@ -2155,9 +2150,9 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_ pthread_rwlock_wrlock(&(scanner->digest_rwlock[i])); if(scanner->digest_handle[i]==NULL) { - scanner->digest_handle[i]=GIE_create(¶) + scanner->digest_handle[i]=GIE_create(¶); } - digest_batch_update(scanner->digest_handle[i], scanner->digest_update_q[i],logger) + digest_batch_update(scanner->digest_handle[i], scanner->digest_update_q[i],logger); pthread_rwlock_unlock(&(scanner->digest_rwlock[i])); } scanner->last_update_time=time(NULL); diff --git a/src/entry/Maat_rule_internal.h b/src/entry/Maat_rule_internal.h index 28cf258..0496c87 100644 --- a/src/entry/Maat_rule_internal.h +++ b/src/entry/Maat_rule_internal.h @@ -1,3 +1,6 @@ +#ifndef H_MAAT_RULE_INTERNAL_H_INCLUDE +#define H_MAAT_RULE_INTERNAL_H_INCLUDE + #include "Maat_rule.h" #include @@ -6,9 +9,15 @@ #include "UniversalBoolMatch.h" #include "rulescan.h" +#include "mesa_fuzzy.h" +#include "great_index_engine.h" + #include #include +extern const char *maat_module; + + #if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411) #define atomic_inc(x) __sync_add_and_fetch((x),1) #define atomic_dec(x) __sync_sub_and_fetch((x),1) @@ -311,4 +320,21 @@ struct _maat_garbage_t }; void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); +void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bool_expr_t* a_set); +int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path); +void maat_start_cb(unsigned int new_version,int update_type,void*u_para); +void maat_update_cb(const char* table_name,const char* line,void *u_para); +void maat_finish_cb(void* u_para); +void *thread_rule_monitor(void *arg); +unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge); +inline void ipv6_ntoh(unsigned int *v6_addr) +{ + unsigned int i=0; + for(i=0;i<4;i++) + { + v6_addr[i]=ntohl(v6_addr[i]); + } + return; +} +#endif diff --git a/src/entry/Makefile b/src/entry/Makefile index 32f35c8..e3e8521 100644 --- a/src/entry/Makefile +++ b/src/entry/Makefile @@ -13,7 +13,7 @@ H_DIR =-I$(G_H_DIR) -I../../inc LIBMAAT = libmaatframe.a LIBMAAT_SO = libmaatframe.so -OBJS=config_monitor.o Maat_rule.o UniversalBoolMatch.o dynamic_array.o cJSON.o json2iris.o map_str2int.o +OBJS=config_monitor.o Maat_rule.o Maat_api.o UniversalBoolMatch.o dynamic_array.o cJSON.o json2iris.o map_str2int.o .c.o: $(CC) -c $(CFLAGS) -I. $(H_DIR) $< diff --git a/test/maat_test.cpp b/test/maat_test.cpp index a7ac6d9..4b92c1d 100644 --- a/test/maat_test.cpp +++ b/test/maat_test.cpp @@ -6,11 +6,13 @@ #include //inet_addr #include //inet_addr #include -#include +#include //fstat #include #include #include #include +#include //fstat +#include //fstat void Maat_read_entry_start_cb(int update_type,void* u_para) { @@ -207,7 +209,7 @@ int main() printf("registe table FILE_DIGEST error.\n"); return 0; } - ret=fstat(digest_test_file,&digest_fstat); + ret=stat(digest_test_file,&digest_fstat); if(ret!=0) { printf("fstat %s error.\n",digest_test_file); @@ -216,10 +218,10 @@ int main() FILE* fp=fopen(digest_test_file,"r"); if(fp!=NULL) { - sp=Maat_stream_scan_digest_start(feather, digest_scan_table_id, digest_fstat->off_t, 0); + sp=Maat_stream_scan_digest_start(feather, digest_scan_table_id, digest_fstat.st_size, 0); while(0!=feof(fp)) { - read_size=fread(fp,digest_test_buff,sizeof(digest_test_buff)); + read_size=fread(digest_test_buff,1,sizeof(digest_test_buff),fp); ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,&mid); scan_offset+=read_size; if(ret>0)