diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 83c35f8..d5e4621 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -259,8 +259,8 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa //Following functions are similar to Maat_rule_get_ex_data, except they are effective on plugin table. typedef void* MAAT_PLUGIN_EX_DATA; -typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, size_t key_sz, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); -typedef void Maat_plugin_EX_free_func_t(int table_id, const char* key, size_t key_sz, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); +typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); +typedef void Maat_plugin_EX_free_func_t(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); typedef void Maat_plugin_EX_dup_func_t(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp); typedef int Maat_plugin_EX_key2index_func_t(const char* key); @@ -271,7 +271,7 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, Maat_plugin_EX_key2index_func_t* key2index_func, long argl, void *argp); //Data is duplicated by dup_func of Maat_plugin_EX_register, caller is responsible to free the data. -MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key, int thread_num); +MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key); enum MAAT_RULE_OPT { diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 18d22b8..803ee46 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -443,7 +443,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* return NULL; } _Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1); - feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger); + feather->table_cnt=read_table_description(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger); feather->map_tablename2id=map_create(); int i=0,j=0,ret=0; for(i=0;iscanner->table_rt[table_id]->plugin); //plugin table register blocks background update. pthread_mutex_lock(&(_feather->backgroud_update_mutex)); idx=plugin_desc->cb_plug_cnt; @@ -944,6 +943,12 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, plugin_desc->cb_plug[idx].update=update; plugin_desc->cb_plug[idx].finish=finish; plugin_desc->cb_plug[idx].u_para=u_para; + if(_feather->scanner==NULL) + { + pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); + return 1; + } + plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin); if(plugin_aux->cache_line_num>0) { if(start!=NULL) @@ -1051,46 +1056,128 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa pthread_rwlock_unlock(&(compile_inner->rwlock)); return ad; } -MESA_htable_handle plugin_EX_htable_new(const struct plugin_table_desc* plugin_desc, - struct dynamic_array_t* lines, size_t line_cnt) +struct wrap_plugin_EX_data +{ + MAAT_RULE_EX_DATA exdata; + const struct Maat_table_desc* ref_plugin_table; +}; +void wrap_plugin_EX_data_free(void *data) +{ + struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data; + const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc); + ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp); + wrap_data->ref_plugin_table=NULL; + free(wrap_data); + return; +} +MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index) { MESA_htable_handle key2ex_hash=NULL; unsigned int slot_size=1; - const char* line=NULL; - size_t key_offset=0, key_len=0; - long long estimate_size=plugin_desc->estimate_size, i=0; - int ret=0; - MAAT_RULE_EX_DATA exdata=NULL; - while(estimate_size>0) + while(estimate_size!=0) { - estimate_size=estimate_size<<1; + estimate_size=estimate_size>>1; slot_size*=2; } if(slot_size==1) { slot_size=4096; } + MESA_htable_create_args_t hargs; memset(&hargs,0,sizeof(hargs)); - hargs.thread_safe=1; + hargs.thread_safe=0; hargs.hash_slot_size = slot_size; hargs.max_elem_num = 0; hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; hargs.expire_time = 0; hargs.key_comp = NULL; - hargs.key2index = plugin_desc->ex_desc.key2index_func; + hargs.key2index = NULL; //Not supported yet. hargs.recursive = 1; - hargs.data_free = NULL ; + hargs.data_free = wrap_plugin_EX_data_free; hargs.data_expire_with_condition = NULL; key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(key2ex_hash, 0); + return key2ex_hash; +} + +int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, + MESA_htable_handle key2ex_hash, void *logger) +{ + size_t key_offset=0, key_len=0; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + int ret=0; + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data del error: cannot find column %d of %s", + plugin_desc->key_column, line); + return -1; + } + ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data del error: no such key %.*s of %s", + key_len, line+key_offset, line); + return -1; + } + return 0; +} + +int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, + MESA_htable_handle key2ex_hash, void *logger) +{ + char* key=NULL; + size_t key_offset=0, key_len=0; + MAAT_RULE_EX_DATA exdata=NULL; + struct wrap_plugin_EX_data* wrap_data=NULL; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + int ret=0; + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data add error: cannot find column %d of %s", + plugin_desc->key_column, line); + return -1; + } + key=ALLOC(char, key_len+1); + memcpy(key, line+key_offset, key_len); + plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata, + plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); + wrap_data=ALLOC(struct wrap_plugin_EX_data, 1); + wrap_data->exdata=exdata; + wrap_data->ref_plugin_table=plugin_table; + ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data); + free(key); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data add error: duplicated key %.*s of %s", + key_len, line+key_offset, line); + wrap_plugin_EX_data_free(wrap_data); + return -1; + } + return 0; +} +MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table, + struct dynamic_array_t* lines, size_t line_cnt, void* logger) +{ + MESA_htable_handle key2ex_hash=NULL; + size_t i=0; + const char* line=NULL; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + + key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func); + for(i=0; i< line_cnt; i++) { line=(const char*)dynamic_array_read(lines, i); - ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); - exdata=plugin_desc->ex_desc.new_func(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);; - MESA_htable_add(key2ex_hash, line+key_offset, key_len, const void * data) + plugin_EX_data_new(plugin_table, line, key2ex_hash, logger); } + return key2ex_hash; } int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, Maat_plugin_EX_new_func_t* new_func, @@ -1103,26 +1190,71 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; struct plugin_table_desc* plugin_desc=&(table_desc->plugin); - struct Maat_table_runtime* table_rt=_feather->scanner->table_rt[table_id]; - MESA_htable_handle key2ex_hash=NULL; - if(table_desc->table_type!=TABLE_TYPE_PLUGIN || new_func==NULL || free_func==NULL || dup_func==NULL) + struct Maat_table_runtime* table_rt=NULL; + if(new_func==NULL || free_func==NULL || dup_func==NULL ) { assert(0); + MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); + return -1; + } + if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata + || plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0) + { + assert(0); + MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, + "%s failed: key or valid flag column are not specified", __FUNCTION__); return -1; } plugin_desc->ex_desc.new_func=new_func; plugin_desc->ex_desc.free_func=free_func; plugin_desc->ex_desc.dup_func=dup_func; - plugin_desc->ex_desc.key2index_func=key2index_func; + plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used. plugin_desc->ex_desc.argl=argl; plugin_desc->ex_desc.argp=argp; + plugin_desc->have_exdata=1; + if(_feather->scanner==NULL) + { + return 0; + } pthread_mutex_lock(&(_feather->backgroud_update_mutex)); - table_rt->plugin.key2ex_hash=plugin_EX_htable_new(plugin_desc,table_rt->plugin.cache_lines, table_rt->plugin.cache_line_num); + table_rt=_feather->scanner->table_rt[table_id]; + assert(table_rt->plugin.key2ex_hash==NULL); + table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines, + table_rt->plugin.cache_line_num, _feather->logger); pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); return 0; } +MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key) +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; + struct Maat_table_runtime *table_rt= NULL; + struct plugin_table_desc* plugin_desc=&(table_desc->plugin); + struct wrap_plugin_EX_data* wrap_data=NULL; + MAAT_RULE_EX_DATA exdata=NULL; + if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata==0) + { + assert(0); + return NULL; + } + if(_feather->scanner==NULL) + { + return NULL; + } + table_rt= _feather->scanner->table_rt[table_id]; + pthread_rwlock_rdlock(&table_rt->plugin.rwlock); + wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash, + (const unsigned char*)key, strlen(key)); + if(wrap_data!=NULL) + { + plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata), + plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); + } + pthread_rwlock_unlock(&table_rt->plugin.rwlock); + return exdata; +} int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id ,enum MAAT_CHARSET charset,const char* data,int data_len diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index c8e09a4..8b6f188 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -221,7 +221,7 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_ assert(0); } - ret=get_column_pos(const char* line, int column_seq, &offset, &len); + ret=get_column_pos(line, column_seq, &offset, &len); if(ret<0||offset>=strlen(line)||(line[offset]!='1'&&line[offset]!='0'))// 0 is also a valid value for some non-MAAT producer. { return -1; @@ -2131,6 +2131,11 @@ int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value, { struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather; redisContext* ctx=_feather->mr_ctx.write_ctx; + if(ctx==NULL) + { + MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, "%s failed: Redis is not connected.", __FUNCTION__); + return -1; + } const char *arg_vec[3]; size_t len_vec[3]; arg_vec[0] = "SET"; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 1178912..e66a4f1 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -529,7 +529,7 @@ int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_h char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0}; char *token=NULL,*sub_token=NULL,*saveptr; struct expr_table_desc* p=&(table->expr); - sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id) + sscanf(line,"%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id) ,table->table_name[0] ,table_type ,src_charset @@ -602,31 +602,21 @@ int _read_integer_arrary(char* string, int *array, int size) return i; } -int read_plugin_table_info(const char* line, struct Maat_table_desc* p) +int read_plugin_table_description(const char* line, struct Maat_table_desc* p) { int i=0,ret=0; - + size_t offset=0, len=0; cJSON* json=NULL, *tmp=NULL, *array_item=NULL; char* copy_line=NULL, *plug_info=NULL; - char *token=NULL,*sub_token=NULL,*saveptr; struct plugin_table_desc* plugin_desc=&(p->plugin); - plugin_desc->valid_flag_column=-1; copy_line=_maat_strdup(line); - for (token = copy_line, i=0; ; token= NULL, i++) - { - sub_token= strtok_r(token,"\t ", &saveptr); - if (sub_token == NULL) - break; - if(i==3) - { - break; - } - } - if(i<3) + ret=get_column_pos(copy_line, 4, &offset, &len); + if(i<0) { goto error_out; } - plug_info=sub_token; + copy_line[offset+len+1]='\0'; + plug_info=copy_line+offset; if(strlen(plug_info)<4)//For old version compatible. { @@ -693,7 +683,7 @@ error_out: free(copy_line); return -1; } -int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) +int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) { FILE*fp=NULL; char line[MAX_TABLE_LINE_SIZE]; @@ -750,7 +740,7 @@ int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* ta } p=table_info_new(max_thread_num); - ret=sscanf(line,"%hu\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id) + ret=sscanf(line,"%d\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id) ,p->table_name[0] ,table_type_str ,not_care); @@ -781,7 +771,7 @@ int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* ta } break; case TABLE_TYPE_PLUGIN: - ret=read_plugin_table_info(line, p); + ret=read_plugin_table_description(line, p); if(ret<0) { fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i); @@ -1193,29 +1183,36 @@ void destroy_digest_rule(GIE_digest_t*rule) } -struct Maat_table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_thread_num) +struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num) { - struct Maat_table_runtime* p= ALLOC(struct Maat_table_runtime, 1); - p->table_type=type; - switch(type) + + struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1); + table_rt->table_type=table_desc->table_type; + switch(table_desc->table_type) { case TABLE_TYPE_DIGEST: case TABLE_TYPE_SIMILARITY: - p->similar.update_q=MESA_lqueue_create(0,0); + table_rt->similar.update_q=MESA_lqueue_create(0,0); break; case TABLE_TYPE_PLUGIN: - p->plugin.cache_lines=dynamic_array_create(1, 1024); + table_rt->plugin.cache_lines=dynamic_array_create(1, 1024); + if(table_desc->plugin.have_exdata) + { + table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, + table_desc->plugin.ex_desc.key2index_func); + pthread_rwlock_init(&(table_rt->plugin.rwlock), NULL); + } break; default: break; } - p->scan_cnt=alignment_int64_array_alloc(max_thread_num); - p->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); - p->input_bytes=alignment_int64_array_alloc(max_thread_num); - p->stream_num=alignment_int64_array_alloc(max_thread_num); - p->hit_cnt=alignment_int64_array_alloc(max_thread_num); - return p; + table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num); + table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); + table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num); + table_rt->stream_num=alignment_int64_array_alloc(max_thread_num); + table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num); + return table_rt; } void table_runtime_free(struct Maat_table_runtime* p) { @@ -1344,7 +1341,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t { continue; } - table_rt=table_runtime_new(pp_table_desc[i]->table_type, feather->scan_thread_num); + table_rt=table_runtime_new(pp_table_desc[i], feather->scan_thread_num); if(pp_table_desc[i]->table_type==TABLE_TYPE_EXPR||pp_table_desc[i]->table_type==TABLE_TYPE_EXPR_PLUS) { expr_desc=&(pp_table_desc[i]->expr); @@ -3178,7 +3175,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; char *p=NULL; char* copy=NULL; - + size_t is_valid_offset=0, valid_len=0; char *token=NULL,*sub_token=NULL,*saveptr; if(plugin_desc->rule_tag_column>0&&n_tags>0) { @@ -3213,13 +3210,32 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M } table_rt->plugin.acc_line_num++; - - if(plugin_desc->cb_plug_cnt>0) + if(plugin_desc->have_exdata || plugin_desc->cb_plug_cnt>0) { - for(i=0;icb_plug_cnt;i++) + + if(plugin_desc->have_exdata) { - plugin_desc->cb_plug[i].update(table->table_id,table_line,plugin_desc->cb_plug[i].u_para); + ret=get_column_pos(table_line, plugin_desc->valid_flag_column, &is_valid_offset, &valid_len); + + pthread_rwlock_wrlock(&(table_rt->plugin.rwlock)); + if(atoi(table_line+is_valid_offset)==1) + { + plugin_EX_data_new(table, table_line, table_rt->plugin.key2ex_hash, logger); + } + else + { + plugin_EX_data_free(table, table_line, table_rt->plugin.key2ex_hash, logger); + } + pthread_rwlock_unlock(&(table_rt->plugin.rwlock)); } + if(plugin_desc->cb_plug_cnt>0) + { + for(i=0;icb_plug_cnt;i++) + { + plugin_desc->cb_plug[i].update(table->table_id,table_line,plugin_desc->cb_plug[i].u_para); + } + } + } else { @@ -3227,7 +3243,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M memcpy(p,table_line,len); table_rt->plugin.cache_size+=len; dynamic_array_write(table_rt->plugin.cache_lines,table_rt->plugin.cache_line_num,p); - table_rt->plugin.cache_line_num++; + table_rt->plugin.cache_line_num++; } } void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger) diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 81618ac..36411fa 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -179,7 +179,7 @@ int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len { *offset=subtoken-dup_line; *len=strlen(subtoken); - ret=0 + ret=0; break; } i++; diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 60b474c..a81994e 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -12,6 +12,7 @@ #include "cJSON.h" #include "hiredis.h" #include "map_str2int.h" +#include "Maat_table_description.h" #include "Maat_rule_internal.h" #include "Maat_utils.h" diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 2c78e6a..dfe551f 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -3,6 +3,7 @@ #include "Maat_rule.h" #include "Maat_command.h" +#include "Maat_table_description.h" #include #include @@ -22,185 +23,24 @@ extern const char *maat_module; -#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411) -#define atomic_inc(x) __sync_add_and_fetch((x),1) -#define atomic_dec(x) __sync_sub_and_fetch((x),1) -#define atomic_add(x,y) __sync_add_and_fetch((x),(y)) -#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y)) -typedef int atomic_t; -#define ATOMIC_INIT(i) { (i) } -#define atomic_read(x) __sync_add_and_fetch((x),0) -#define atomic_set(x,y) __sync_lock_test_and_set((x),y) -#else -#include -#endif -#define TRUE 1 -#define FALSE 0 + #define MAX_TABLE_NUM 256 -#define MAX_CONJUNCTION_TABLE_NUM 8 -#define MAX_CHARSET_NUM 16 -#define MAX_TABLE_NAME_LEN 256 #define MAX_TABLE_LINE_SIZE (1024*16) #define MAX_EXPR_KEYLEN 1024 #define MAX_DISTRICT_LEN 64 -#define MAX_PLUGIN_PER_TABLE 32 -#define MAX_FOREIGN_CLMN_NUM 8 + #define MAX_SCANNER_HIT_NUM 64 -#define MAX_COMPILE_EX_DATA_NUM 2 #define MAX_GROUP_CACHE 128 #define MAX_FAILED_NUM 128 #define MAX_MAAT_STAT_NUM 64 -#ifndef MAX -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) -#endif -#ifndef MIN -#define MIN(a, b) (((a) < (b)) ? (a) : (b)) -#endif - -#ifndef offsetof -#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) -#endif - -#ifndef container_of -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) -#endif typedef void* rule_scanner_t; -enum MAAT_TABLE_TYPE -{ - TABLE_TYPE_EXPR=0, - TABLE_TYPE_IP, - TABLE_TYPE_INTERVAL, - TABLE_TYPE_DIGEST, - TABLE_TYPE_EXPR_PLUS, - TABLE_TYPE_SIMILARITY, - TABLE_TYPE_GROUP, - TABLE_TYPE_COMPILE, - TABLE_TYPE_PLUGIN -}; -enum USER_REGION_ENCODE -{ - USER_REGION_ENCODE_NONE=0, - USER_REGION_ENCODE_ESCAPE, - USER_REGION_ENCODE_BASE64 -}; -struct compile_ex_data_idx -{ - Maat_rule_EX_new_func_t *new_func; - Maat_rule_EX_free_func_t* free_func; - Maat_rule_EX_dup_func_t* dup_func; - long argl; - void *argp; - int idx; - int table_id; -}; -struct compile_table_desc -{ - enum USER_REGION_ENCODE user_region_encoding; - int ex_data_num; - struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; -}; -struct plugin_table_callback_desc -{ - Maat_start_callback_t *start; - Maat_update_callback_t *update; - Maat_finish_callback_t *finish; - void* u_para; -}; -struct plugin_table_ex_data_desc -{ - Maat_plugin_EX_new_func_t* new_func; - Maat_plugin_EX_free_func_t* free_func; - Maat_plugin_EX_dup_func_t* dup_func; - Maat_plugin_EX_key2index_func_t* key2index_func; - long argl; - void *argp; - MESA_htable_handle key2ex_hash; -}; -struct plugin_table_desc -{ - int key_column; - int valid_flag_column; - int rule_tag_column; - int n_foreign; - int foreign_columns[MAX_FOREIGN_CLMN_NUM]; - int cb_plug_cnt; - long long estimate_size; - struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; - struct plugin_table_ex_data_desc ex_desc; -}; -struct expr_table_desc -{ - enum MAAT_CHARSET src_charset; - enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; - int src_charset_in_dst; - int do_charset_merge; - int cross_cache_size; - int quick_expr_switch; - long long iconv_err_cnt; -}; -struct ip_table_desc -{ - int ipv4_rule_cnt; - int ipv6_rule_cnt; -}; - -struct Maat_table_desc -{ - unsigned short table_id; - unsigned short conj_cnt; - unsigned short updating_name; - char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN]; - enum MAAT_TABLE_TYPE table_type; - union - { - struct compile_table_desc compile; - struct expr_table_desc expr; - struct ip_table_desc ip; - struct plugin_table_desc plugin; - void* others;//group, interval and digest don't have sperate description info. - }; -/* - enum MAAT_CHARSET src_charset; - enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; - int src_charset_in_dst; - int do_charset_merge; - int cross_cache_size; - int quick_expr_switch; - union - { - int expr_rule_cnt; //expr_type=0,1,3 - int ipv4_rule_cnt; - }; - union - { - int regex_rule_cnt; //expr_type=2 - int ipv6_rule_cnt; - }; - struct _plugin_table_info *cb_info; - int valid_flag_column; //for plugin table - int rule_tag_column; //for plugin table; - int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table; - int n_foreign; - - //for compile table - enum USER_REGION_ENCODE user_region_encoding; - int ex_data_num; - struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; -*/ -//for stat>>>>>>>> - unsigned long long udpate_err_cnt; - unsigned long long unmatch_tag_cnt; - int stat_line_id; -}; struct db_str_rule_t { @@ -380,6 +220,7 @@ struct plugin_runtime long long cache_line_num; long long acc_line_num; long long cache_size; + pthread_rwlock_t rwlock; MESA_htable_handle key2ex_hash; }; struct expr_runtime @@ -595,7 +436,7 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set); -int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); +int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); void maat_start_cb(long long new_version,int update_type,void*u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para); void maat_finish_cb(void* u_para); @@ -620,6 +461,9 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p); void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len); MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc); void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc); +MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index); +int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger); +int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger); void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout); diff --git a/src/inc_internal/Maat_table_description.h b/src/inc_internal/Maat_table_description.h new file mode 100644 index 0000000..deb7629 --- /dev/null +++ b/src/inc_internal/Maat_table_description.h @@ -0,0 +1,113 @@ +#pragma once +#include "Maat_rule.h" +#define MAX_COMPILE_EX_DATA_NUM 2 +#define MAX_FOREIGN_CLMN_NUM 8 +#define MAX_PLUGIN_PER_TABLE 32 +#define MAX_CHARSET_NUM 16 +#define MAX_CONJUNCTION_TABLE_NUM 8 +#define MAX_TABLE_NAME_LEN 256 + +enum USER_REGION_ENCODE +{ + USER_REGION_ENCODE_NONE=0, + USER_REGION_ENCODE_ESCAPE, + USER_REGION_ENCODE_BASE64 +}; + +enum MAAT_TABLE_TYPE +{ + TABLE_TYPE_EXPR=0, + TABLE_TYPE_IP, + TABLE_TYPE_INTERVAL, + TABLE_TYPE_DIGEST, + TABLE_TYPE_EXPR_PLUS, + TABLE_TYPE_SIMILARITY, + TABLE_TYPE_GROUP, + TABLE_TYPE_COMPILE, + TABLE_TYPE_PLUGIN + +}; + +struct compile_ex_data_idx +{ + Maat_rule_EX_new_func_t *new_func; + Maat_rule_EX_free_func_t* free_func; + Maat_rule_EX_dup_func_t* dup_func; + long argl; + void *argp; + int idx; + int table_id; +}; +struct compile_table_desc +{ + enum USER_REGION_ENCODE user_region_encoding; + int ex_data_num; + struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; +}; + +struct plugin_table_callback_desc +{ + Maat_start_callback_t *start; + Maat_update_callback_t *update; + Maat_finish_callback_t *finish; + void* u_para; +}; +struct plugin_table_ex_data_desc +{ + Maat_plugin_EX_new_func_t* new_func; + Maat_plugin_EX_free_func_t* free_func; + Maat_plugin_EX_dup_func_t* dup_func; + Maat_plugin_EX_key2index_func_t* key2index_func; + long argl; + void *argp; +}; +struct plugin_table_desc +{ + int key_column; + int valid_flag_column; + int rule_tag_column; + int n_foreign; + int foreign_columns[MAX_FOREIGN_CLMN_NUM]; + int cb_plug_cnt; + int have_exdata; + long long estimate_size; + struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; + struct plugin_table_ex_data_desc ex_desc; +}; +struct expr_table_desc +{ + enum MAAT_CHARSET src_charset; + enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; + int src_charset_in_dst; + int do_charset_merge; + int cross_cache_size; + int quick_expr_switch; + long long iconv_err_cnt; +}; +struct ip_table_desc +{ + int ipv4_rule_cnt; + int ipv6_rule_cnt; +}; + +struct Maat_table_desc +{ + int table_id; + int conj_cnt; + int updating_name; + char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN]; + enum MAAT_TABLE_TYPE table_type; + union + { + struct compile_table_desc compile; + struct expr_table_desc expr; + struct ip_table_desc ip; + struct plugin_table_desc plugin; + void* others;//group, interval and digest don't have sperate description info. + }; +//for stat>>>>>>>> + unsigned long long udpate_err_cnt; + unsigned long long unmatch_tag_cnt; + int stat_line_id; +}; + diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index 23e22ec..e8d7114 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -9,9 +9,20 @@ #define ALLOC(type, number) ((type *)calloc(sizeof(type), number)) #define FREE(p) {free(*p);*p=NULL;} -#define ATOMIC_INC(x) __atomic_fetch_add(x,1,__ATOMIC_RELAXED) -#define ATOMIC_READ(x) __atomic_fetch_add(x,0,__ATOMIC_RELAXED) - +#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411) +#define atomic_inc(x) __sync_add_and_fetch((x),1) +#define atomic_dec(x) __sync_sub_and_fetch((x),1) +#define atomic_add(x,y) __sync_add_and_fetch((x),(y)) +#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y)) + typedef int atomic_t; +#define ATOMIC_INIT(i) { (i) } +#define atomic_read(x) __sync_add_and_fetch((x),0) +#define atomic_set(x,y) __sync_lock_test_and_set((x),y) +#else +#include +#endif +#define TRUE 1 +#define FALSE 0 #ifndef MAX #define MAX(a, b) (((a) > (b)) ? (a) : (b)) @@ -21,6 +32,16 @@ #define MIN(a, b) (((a) < (b)) ? (a) : (b)) #endif +#ifndef offsetof +#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) +#endif + +#ifndef container_of +#define container_of(ptr, type, member) ({ \ + const typeof( ((type *)0)->member ) *__mptr = (ptr); \ + (type *)( (char *)__mptr - offsetof(type,member) );}) +#endif + #define UNUSED __attribute__((unused)) const char* module_name_str(const char*name); #define maat_module (module_name_str("MAAT_Frame")) diff --git a/test/table_info.conf b/test/table_info.conf index 4b03043..55b0825 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -28,5 +28,6 @@ 9 SIM_URL similar -- 10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff 11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} -- -12 TEST_FOREIGN_KEY plugin {"key":2,"valid":4,"foreign":[6,8],"tag":3, "estimate_size":4096} -- +12 TEST_FOREIGN_KEY plugin {"valid":4,"foreign":[6,8],"tag":3} -- 13 COMPILE_ALIAS compile escape -- +14 TEST_PLUGIN_EXDATA_TABLE plugin {"key":2,"valid":4,"tag":5,"estimate_size":1024} -- diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 41b6b5e..4a5c084 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -640,7 +640,7 @@ struct rule_ex_param pthread_mutex_t lock; }; -void ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, +void compile_ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp) { int *counter=(int*)argp; @@ -657,7 +657,7 @@ void ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_l *ad=param; return; } -void ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp) +void compile_ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp) { if(*ad==NULL) { @@ -674,7 +674,7 @@ void ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_ free(param); return; } -void ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp) +void compile_ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp) { struct rule_ex_param* from_param=*((struct rule_ex_param**)from); pthread_mutex_lock(&(from_param->lock)); @@ -686,9 +686,9 @@ void ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long -TEST(EX_DATA_INDEX, all) +TEST(EX_DATA_INDEX, MaatRuleEXData) { -#define EX_data_index +#define rule_EX_data_index int ret=0; int table_id=0, ex_data_counter=0; @@ -701,7 +701,7 @@ TEST(EX_DATA_INDEX, all) ASSERT_GT(table_id, 0); int ex_param_idx=Maat_rule_get_ex_new_index(g_feather, "COMPILE_ALIAS", - ex_param_new, ex_param_free, ex_param_dup, + compile_ex_param_new, compile_ex_param_free, compile_ex_param_dup, 0, &ex_data_counter); ASSERT_TRUE(ex_param_idx>=0); EXPECT_EQ(ex_data_counter, 1); @@ -716,7 +716,7 @@ TEST(EX_DATA_INDEX, all) struct rule_ex_param* param=(struct rule_ex_param*)ex_data; EXPECT_EQ(param->id, 7799); EXPECT_EQ(strcmp(param->name, expect_name),0); - ex_param_free(0, NULL, NULL, &ex_data, 0, NULL); + compile_ex_param_free(0, NULL, NULL, &ex_data, 0, NULL); Maat_clean_status(&mid); return; } @@ -1418,6 +1418,98 @@ TEST_F(MaatCmdTest, SetFile) return; } +struct user_info +{ + char name[256]; + char ip_addr[32]; + int id; + int ref_cnt; +}; +void plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) +{ + int *counter=(int *)argp; + struct user_info* u=(struct user_info*)calloc(sizeof(struct user_info), 1); + int valid=0, tag=0, ret; + ret=sscanf(table_line, "%d\t%s\t%s%d\t%d", &(u->id), u->ip_addr, u->name, &valid, &tag); + EXPECT_EQ(ret, 5); + u->ref_cnt=1; + *ad=u; + (*counter)++; + return; +} +void plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) +{ + struct user_info* u=(struct user_info*)(*ad); + u->ref_cnt--; + if(u->ref_cnt>0) return; + free(u); + *ad=NULL; +} +void plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) +{ + struct user_info* u=(struct user_info*)(*from); + u->ref_cnt++; + *to=u; +} + +TEST_F(MaatCmdTest, PluginEXData) +{ +#define plugin_EX_data_index + Maat_feather_t feather=MaatCmdTest::_shared_feather; + + int ret=0, i=0; + int table_id=0, ex_data_counter=0; + const char* table_name="TEST_PLUGIN_EXDATA_TABLE"; + + + const int TEST_CMD_LINE_NUM=4; + const struct Maat_line_t *p_line[TEST_CMD_LINE_NUM]; + struct Maat_line_t line_rule[TEST_CMD_LINE_NUM]; + const char* table_line[TEST_CMD_LINE_NUM]={"1\t192.168.0.1\tmahuateng\t1\t0", + "2\t192.168.0.2\tliuqiangdong\t1\t0", + "3\t192.168.0.3\tmayun\t1\t0", + "4\t192.168.0.4\tliyanhong\t1\t0"}; + table_id=Maat_table_register(feather, table_name); + ASSERT_GT(table_id, 0); + memset(&line_rule,0,sizeof(line_rule)); + for(i=0;i=0); + EXPECT_EQ(ex_data_counter, TEST_CMD_LINE_NUM); + + struct user_info* uinfo=NULL; + uinfo=(struct user_info*)Maat_plugin_get_EX_data(feather, table_id, "192.168.0.2"); + ASSERT_TRUE(uinfo!=NULL); + EXPECT_EQ(0, strcmp(uinfo->name, "liuqiangdong")); + EXPECT_EQ(uinfo->id, 2); + plugin_EX_free_cb(table_id, (void**)&uinfo, 0, NULL); + + ret=Maat_cmd_set_lines(feather, p_line+1,1, MAAT_OP_DEL); + EXPECT_GT(ret, 0); + + usleep(WAIT_FOR_EFFECTIVE_US); + uinfo=(struct user_info*)Maat_plugin_get_EX_data(feather, table_id, "192.168.0.2"); + ASSERT_TRUE(uinfo==NULL); + + return; +} int main(int argc, char ** argv) {