From d67aa1b8b2229cf261470ce20ec08a6dc7bc20f2 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Sat, 13 Jun 2020 21:05:42 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E9=80=82=E9=85=8DMaat=20comm?= =?UTF-8?q?and=E3=80=81maat=20json=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/Maat_command.h | 86 +-- inc/Maat_rule.h | 22 +- src/CMakeLists.txt | 2 +- src/entry/Maat_api.cpp | 386 +++---------- src/entry/Maat_command.cpp | 579 ++----------------- src/entry/Maat_garbage_collection.cpp | 16 +- src/entry/Maat_hierarchy.cpp | 620 ++++++++++++--------- src/entry/Maat_rule.cpp | 76 +-- src/entry/Maat_stat.cpp | 2 +- src/entry/Maat_table.cpp | 18 +- src/entry/json2iris.cpp | 180 ++++-- src/inc_internal/Maat_garbage_collection.h | 4 +- src/inc_internal/Maat_hierarchy.h | 46 ++ src/inc_internal/Maat_rule_internal.h | 95 +--- src/inc_internal/Maat_table.h | 4 +- src/inc_internal/json2iris.h | 2 +- src/version.map | 2 +- test/CMakeLists.txt | 8 +- test/maat_demo.cpp | 8 +- test/perf_test_maatframe.cpp | 8 +- test/test_maatframe.cpp | 282 +++++----- tools/maat_redis_tool.cpp | 2 +- 22 files changed, 935 insertions(+), 1513 deletions(-) diff --git a/inc/Maat_command.h b/inc/Maat_command.h index 5e0c6e1..a0031a6 100644 --- a/inc/Maat_command.h +++ b/inc/Maat_command.h @@ -1,7 +1,7 @@ #ifndef H_MAAT_COMMAND_H_INCLUDE #define H_MAAT_COMMAND_H_INCLUDE -#ifndef __cplusplus -#error("This file should be compiled with C++ compiler") +#ifdef __cplusplus +extern "C"{ #endif #include "Maat_rule.h" enum MAAT_OPERATION @@ -106,17 +106,22 @@ struct Maat_region_t struct Maat_rgn_sim_t similarity_rule; }; }; -struct Maat_group_t +struct Maat_cmd_region { const char* table_name; - const char* virtual_table_name; - int group_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number. - int parent_id; - int not_flag; - enum MAAT_GROUP_RELATION parent_type; - int region_num; - struct Maat_region_t *regions; + int region_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must appoint a unique number. + enum MAAT_REGION_TYPE region_type; + union + { + struct Maat_rgn_str_t expr_rule; + struct Maat_rgn_addr_t ip_rule; + struct Maat_rgn_intv_t interval_rule; + struct Maat_rgn_digest_t digest_rule; + struct Maat_rgn_sim_t similarity_rule; + }; }; + + struct Maat_cmd_t { //This Struct MUST alloced by Maat_create_cmd(), then released by Maat_free_cmd(). @@ -125,8 +130,8 @@ struct Maat_cmd_t int expire_after; //expired after $expire_after$ seconds, set to 0 for never timeout. int label_id; //>0, to be indexed and quried by Maat_cmd_select; =0 not index struct Maat_group_t* groups;// Add regions with Maat_add_region2cmd -}; -struct Maat_line_t +}; +struct Maat_cmd_line { const char* table_name; const char* table_line; @@ -134,37 +139,15 @@ struct Maat_line_t int label_id; int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout. }; -struct Maat_cmd_t* Maat_create_cmd(const struct Maat_rule_t* rule, int group_num); -int Maat_cmd_set_opt(struct Maat_cmd_t* cmd, enum MAAT_RULE_OPT type, const char* val, int size); -//input: which_group 0~group_num -//input: region can be freed after added. -void Maat_add_region2cmd(struct Maat_cmd_t* cmd,int which_group,const struct Maat_region_t* region); -void Maat_free_cmd(struct Maat_cmd_t* cmd); -int Maat_format_cmd(struct Maat_cmd_t* cmd, char* buffer, int size); //Input string of REGION_EXPR and REGION_SIMILARITY need to be escapeed. char* Maat_str_escape(char* dst,int size,const char*src); -//Deletion failed due to not complete synchronize with Redis. -//To make sure the delete command is excecuted, user should try again after MAAT_OPT_SCANDIR_INTERVAL_MS ms. -//Returns number of successfully updated rule. -//The following functions are NOT thread safe. -int Maat_cmd(Maat_feather_t feather,struct Maat_cmd_t* cmd,enum MAAT_OPERATION op); - -//pipeline model -int Maat_cmd_append(Maat_feather_t feather,struct Maat_cmd_t* cmd,enum MAAT_OPERATION op); - -//Return number of successfully updated rule. -//Return -1 for failed. -int Maat_cmd_commit(Maat_feather_t feather); - - -int Maat_cmd_set_group(Maat_feather_t feather, int group_id, const struct Maat_region_t* region, enum MAAT_OPERATION op); //Returns number of successfully updated rule. //Return -1 for failed. -int Maat_cmd_set_line(Maat_feather_t feather,const struct Maat_line_t* line_rule, enum MAAT_OPERATION op); -int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_rule, int line_num ,enum MAAT_OPERATION op); +int Maat_cmd_set_line(Maat_feather_t feather,const struct Maat_cmd_line* line_rule, enum MAAT_OPERATION op); +int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_rule, int line_num ,enum MAAT_OPERATION op); int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value, size_t size, enum MAAT_OPERATION op); //Return the value of key after the increment. @@ -180,13 +163,38 @@ int Maat_cmd_key_select(Maat_feather_t feather, int label_id, struct Maat_cmd_ke int Maat_cmd_select(Maat_feather_t feather, int label_id, int * output_ids, unsigned int size); int Maat_cmd_flushDB(Maat_feather_t feather); -int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num); -int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_region_t* region, int group_id); -int Maat_command_raw_set_group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_group_t* group); +struct Maat_cmd_group2group +{ + const char* table_name; + int group_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number. + int superior_group_id; +}; +struct Maat_cmd_group2compile +{ + const char* table_name; + const char* virtual_table_name; + int group_id; + int compile_id; + int Nth_clause; + int not_flag; +}; + +int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_region* region, int group_id); +int Maat_command_raw_set_group2group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g); +int Maat_command_raw_set_group2compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c); + +//@param expire_after: expired after $expire_after$ seconds, set to 0 for never timeout. +//@param label_id: bigger than 0 means this compile rule is to be indexed and quried by Maat_cmd_select; =0 not index +int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int clause_num, int label_id, int expire_after); + int Maat_cmd_get_new_group_id(Maat_feather_t feather); int Maat_cmd_get_new_region_id(Maat_feather_t feather); +#ifdef __cplusplus +} //end extern"C" +#endif + #endif diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index cea3938..f825658 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -5,16 +5,14 @@ * Her feather was the measure that determined whether the souls (considered * to reside in the heart) of the departed would reach the paradise of afterlife * successfully. -* Author: zhengchao@iie.ac.cn, MESA -* Version 2018-12-07 Plugin Extra Data. -* NOTE: MUST compile with G++ -* All right reserved by Institute of Infomation Engineering,Chinese Academic of Science 2014~2018 +* Author: zhengchao, MESA +* Version 2020-06-13 version 3.0.0 ********************************************************* */ #ifndef H_MAAT_RULE_H_INCLUDE #define H_MAAT_RULE_H_INCLUDE -#ifndef __cplusplus -#error("This file should be compiled with C++ compiler") +#ifdef __cplusplus +extern "C"{ #endif #include enum MAAT_CHARSET @@ -56,10 +54,10 @@ struct Maat_rule_t { int config_id; int service_id; - char do_log; - char do_blacklist; - char action; - char reserved; + unsigned char do_log; + unsigned char do_blacklist; + unsigned char action; + unsigned char reserved; int serv_def_len; char service_defined[MAX_SERVICE_DEFINE_LEN]; }; @@ -328,6 +326,10 @@ enum MAAT_RULE_OPT }; int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum MAAT_RULE_OPT type, void* value, int size); +#ifdef __cplusplus +}//end extern "C" +#endif + #endif // H_MAAT_RULE_H_INCLUDE diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4f240fb..10dcab8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) -set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp) +set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) include_directories(/opt/MESA/include/MESA/) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 92f5e54..9cdcb69 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -9,6 +9,8 @@ #include "Maat_rule.h" #include "Maat_rule_internal.h" #include "Maat_utils.h" +#include "Maat_garbage_collection.h" +#include "Maat_hierarchy.h" #include "dynamic_array.h" #include "alignment_int64.h" #include "config_monitor.h" @@ -33,8 +35,8 @@ inline int scan_status_should_compile_NOT(struct _OUTER_scan_status_t* _mid) { if( _mid && _mid->is_last_region==1 && - _mid->inner && - _mid->inner->not_grp_compile_hitted_flag) + _mid->compile_mid && + Maat_hierarchy_compile_mid_has_NOT_clause(_mid->compile_mid)) { return 1; } @@ -43,24 +45,7 @@ inline int scan_status_should_compile_NOT(struct _OUTER_scan_status_t* _mid) return 0; } } -int exprid2region_id(struct Maat_group_inner* group_rule, int expr_id, int* district_id, struct Maat_scanner* scanner) -{ - int region_id=-1; - struct Maat_region_inner* region_rule=NULL; - assert(group_rule->group_id>=0); - int array_idx=(int)(long)HASH_fetch_by_id(scanner->exprid_hash, expr_id); - pthread_mutex_lock(&(group_rule->mutex)); - assert(array_idxregion_boundary); - region_rule=(struct Maat_region_inner*)dynamic_array_read(group_rule->regions, array_idx); - if(region_rule) - { - assert(expr_id>=region_rule->expr_id_lb&&expr_id<=region_rule->expr_id_ub); - region_id=region_rule->region_id; - *district_id=region_rule->district_id; - } - pthread_mutex_unlock(&(group_rule->mutex)); - return region_id; -} + //return 1 if insert a unique id //return 0 if id is duplicated @@ -166,17 +151,16 @@ size_t Maat_rule_sort_by_evaluation_order(Maat_feather_t feather, struct Maat_ru memcpy(copy_rule_array, rule_array, sizeof(struct Maat_rule_t)*n_rule); - struct Maat_compile_inner *p=NULL; + struct Maat_compile_rule *p=NULL; size_t i=0, j=0; for(i=0; iscanner->compile_hash, rule_array[i].config_id); - if(p && 0==pthread_rwlock_tryrdlock(&(p->rwlock)))//rule maybe already deleted. + p=(struct Maat_compile_rule*)Maat_hierarchy_compile_read_user_data(_feather->scanner->hier, rule_array[i].config_id); + if(p) { - compile_sort_para_set(sort_para+i, p, copy_rule_array+i); + compile_sort_para_set(sort_para+j, p, copy_rule_array+j); j++; - pthread_rwlock_unlock(&(p->rwlock)); } } qsort(sort_para, j, sizeof(struct compile_sort_para), @@ -195,174 +179,12 @@ struct scan_region_hit_wraper int* virtual_table_ids; size_t elem_size; size_t n_elem; - size_t group_offset; + size_t user_data_offset; size_t expr_id_offset; int virtual_table_id; int is_last_region; }; -struct scan_hit_status* scan_hit_status_new(void) -{ - struct scan_hit_status* inner_mid=NULL; - inner_mid=ALLOC(struct scan_hit_status, 1); - TAILQ_INIT(&inner_mid->hit_path_qhead); - return inner_mid; -} -void scan_hit_status_free(struct scan_hit_status* p) -{ - struct Maat_hit_path_inner * tmp = TAILQ_FIRST(&p->hit_path_qhead); - while(tmp != NULL) - { - TAILQ_REMOVE(&p->hit_path_qhead, tmp, entries); - free(tmp); - p->hit_path_cnt--; - tmp = TAILQ_FIRST(&p->hit_path_qhead); - } - assert(p->hit_path_cnt==0); - free(p->all_hit_group_array); - p->all_hit_group_array=NULL; - free(p); -} -void hit_path_init(struct Maat_hit_path_t* hit_path) -{ - hit_path->Nth_scan=-1; - hit_path->region_id=-1; - hit_path->sub_group_id=-1; - hit_path->top_group_id=-1; - hit_path->vt_id=-1; - hit_path->compile_id=-1; -} -size_t scan_hit_status_select_hit_path_inner(const struct hit_path_q *hit_path_qhead, struct Maat_hit_path_t* condition, - struct Maat_hit_path_inner* hit_paths, size_t n_path) -{ - struct Maat_hit_path_inner* p=NULL; - size_t i=0; - TAILQ_FOREACH(p, hit_path_qhead, entries) - { - if((condition->compile_id==p->path.compile_id||condition->compile_id<0) - && (condition->Nth_scan==p->path.Nth_scan||condition->Nth_scan<0) - && (condition->region_id=p->path.region_id||condition->region_id<0) - && (condition->sub_group_id=p->path.sub_group_id||condition->sub_group_id<0) - && (condition->top_group_id=p->path.top_group_id||condition->top_group_id<0)) - { - if(itop_group_cnt==0) - { - hit_path=ALLOC(struct Maat_hit_path_inner, 1); - hit_path_init(&(hit_path->path)); - hit_path->Nth_hit_region=Nth_region_result; - hit_path->path.Nth_scan=Nth_scan; - hit_path->path.region_id=region_id; - hit_path->path.sub_group_id=group_rule->group_id; - hit_path->path.vt_id=virtual_table_id; - TAILQ_INSERT_TAIL(&hit_status->hit_path_qhead, hit_path, entries); - hit_status->hit_path_cnt++; - } - else - { - for(i=0; itop_group_cnt; i++) - { - hit_path=ALLOC(struct Maat_hit_path_inner, 1); - hit_path_init(&(hit_path->path)); - hit_path->Nth_hit_region=Nth_region_result; - hit_path->path.Nth_scan=Nth_scan; - hit_path->path.region_id=region_id; - hit_path->path.sub_group_id=group_rule->group_id; - hit_path->path.top_group_id=group_rule->top_groups[i]; - hit_path->path.vt_id=virtual_table_id; - TAILQ_INSERT_TAIL(&hit_status->hit_path_qhead, hit_path, entries); - hit_status->hit_path_cnt++; - ret=insert_set_id(&(hit_status->all_hit_group_array), - &(hit_status->all_hit_group_array_sz), - hit_status->all_hit_group_cnt, - TO_RELATION_ID(virtual_table_id, group_rule->top_groups[i])); - hit_status->all_hit_group_cnt+=ret; - } - } - return; -} -size_t scan_hit_status_update_by_compile(struct scan_hit_status* hit_status, struct Maat_compile_inner* compile_rule, int Nth_scan) -{ - size_t i=0, j=0; - struct Maat_hit_path_inner* p=NULL, *q=NULL; - struct bool_expr a_set; - unsigned char has_not=0; - - struct Maat_hit_path_t condition; - size_t n_exsited_path=0; - struct hit_path_q shared_path_qhead; - TAILQ_INIT(&shared_path_qhead); - make_group_set(compile_rule, &a_set, &has_not); - for(i=0; ihit_path_qhead, entries) - { - n_exsited_path=0; - if(TO_RELATION_ID(p->path.vt_id, p->path.top_group_id)==a_set.items[i].item_id - && p->path.compile_id!=compile_rule->compile_id) - { - if(p->path.compile_id<0) - { - p->path.compile_id=compile_rule->compile_id; - } - else //current path already have a compile as endpoint, so we duplicate a new path. - { - condition=p->path; - condition.compile_id=compile_rule->compile_id; - n_exsited_path=scan_hit_status_select_hit_path_inner(&shared_path_qhead, &condition, NULL, 0); - if(n_exsited_path==0) - { - q=ALLOC(struct Maat_hit_path_inner, 1); - *q=*p; - q->path.compile_id=compile_rule->compile_id; - TAILQ_INSERT_TAIL(&shared_path_qhead, q, entries); - hit_status->hit_path_cnt++; - } - } - if(p->path.Nth_scan==Nth_scan && n_exsited_path==0)//Compile was satisfied by new region hits. - { - j++; - } - } - } - } - p = TAILQ_FIRST(&shared_path_qhead); - while(p != NULL) - { - TAILQ_REMOVE(&shared_path_qhead, p, entries); - TAILQ_INSERT_TAIL(&hit_status->hit_path_qhead, p, entries); - p = TAILQ_FIRST(&shared_path_qhead); - } - return j; -} - -size_t scan_hit_status_read_group_ids(const struct scan_hit_status* hit_status, const unsigned long long** const grp_ids) -{ - *grp_ids=hit_status->all_hit_group_array; - return hit_status->all_hit_group_cnt; -} - void scan_region_hit_wraper_build_with_rulescan(struct scan_region_hit_wraper* region_hit, scan_result_t* rulescan_rslt, size_t n_rslt, int is_last_region, int virtual_table_id, int Nth_scan) { @@ -371,7 +193,7 @@ void scan_region_hit_wraper_build_with_rulescan(struct scan_region_hit_wraper* r region_hit->n_elem=n_rslt; region_hit->elem_size=sizeof(scan_result_t); region_hit->expr_id_offset=offsetof(scan_result_t, expr_id); - region_hit->group_offset=offsetof(scan_result_t, tag); + region_hit->user_data_offset=offsetof(scan_result_t, tag); region_hit->is_last_region=is_last_region; region_hit->virtual_table_id=virtual_table_id; region_hit->Nth_scan=Nth_scan; @@ -385,7 +207,7 @@ void scan_region_hit_wraper_build_with_GIE(struct scan_region_hit_wraper* region region_hit->n_elem=n_rslt; region_hit->elem_size=sizeof(GIE_result_t); region_hit->expr_id_offset=offsetof(GIE_result_t, id); - region_hit->group_offset=offsetof(GIE_result_t, tag); + region_hit->user_data_offset=offsetof(GIE_result_t, tag); region_hit->is_last_region=is_last_region; region_hit->virtual_table_id=virtual_table_id; region_hit->Nth_scan=Nth_scan; @@ -393,35 +215,26 @@ void scan_region_hit_wraper_build_with_GIE(struct scan_region_hit_wraper* region return; } -int region_compile(_Maat_feather_t*feather, struct scan_hit_status *_mid, struct Maat_hierarchy_compile_mid* compile_mid, const struct scan_region_hit_wraper* region_hit_wraper, struct Maat_rule_t* result, int size,int thread_num) +int region_compile(_Maat_feather_t*feather, struct Maat_hierarchy_compile_mid* compile_mid, const struct scan_region_hit_wraper* region_hit_wraper, struct Maat_rule_t* result, int size,int thread_num) { int is_last_region=region_hit_wraper->is_last_region; void* region_hit=region_hit_wraper->elem_array; size_t region_type_size=region_hit_wraper->elem_size; - size_t expr_id_offset=region_hit_wraper->expr_id_offset; - size_t group_offset=region_hit_wraper->group_offset; + size_t user_data_offset=region_hit_wraper->user_data_offset; size_t region_hit_num=region_hit_wraper->n_elem; - unsigned int expr_id=0; - int region_id=0; - int scan_ret=0, result_cnt=0; - int tmp=0; - size_t i=0; - size_t r_in_c_cnt=0; + int scan_ret=0; + int i=0; struct Maat_compile_rule* compile_rule_array[MAX_SCANNER_HIT_NUM]; struct Maat_compile_rule* compile_rule=NULL; int virtual_table_id=0; - const unsigned long long* hit_group_ids=NULL; - size_t hit_group_id_cnt=0; struct Maat_region_inner* region=NULL; - int region_ids[MAX_SCANNER_HIT_NUM]; - for(i=0;iregion_id; + region=*(struct Maat_region_inner**)((char*)region_hit+region_type_size*i+user_data_offset); if(region_hit_wraper->virtual_table_ids) { virtual_table_id=region_hit_wraper->virtual_table_ids[i]; @@ -430,64 +243,48 @@ int region_compile(_Maat_feather_t*feather, struct scan_hit_status *_mid, struct { virtual_table_id=region_hit_wraper->virtual_table_id; } - Maat_hierarchy_compile_mid_udpate(compile_mid, region_id, virtual_table_id, region_hit_wraper->Nth_scan, i); + Maat_hierarchy_compile_mid_udpate(compile_mid, region->region_id, virtual_table_id, region_hit_wraper->Nth_scan, i); } - scan_ret=Maat_hierarchy_region_compile(compile_mid, int * region_ids, int * virtual_tables, size_t n_region, int Nth_scan, compile_rule_array, MAX_SCANNER_HIT_NUM); + scan_ret=Maat_hierarchy_region_compile(compile_mid, is_last_region, (void**)compile_rule_array, MAX_SCANNER_HIT_NUM); if(scan_ret>1) { qsort(compile_rule_array, scan_ret, sizeof(struct Maat_compile_rule*), compare_compile_rule); } - for(i=0; i<(unsigned int)scan_ret&&result_cntmagic_num==COMPILE_RULE_MAGIC); - - if(compile_rule->not_group_cnt>0 && !is_last_region) - { - _mid->not_grp_compile_hitted_flag=1; - } - else - { - r_in_c_cnt=scan_hit_status_update_by_compile(_mid, compile_rule, region_hit_wraper->Nth_scan); - if(r_in_c_cnt>0 || //compile hitted becasue of new reigon - region_hit_num==0) //or hit a compile that refer a NOT-logic group in previous scan. - { - fill_maat_rule(&(result[result_cnt]), &(compile_rule->compile->head), - compile_rule->compile->service_defined, compile_rule->compile->head.serv_def_len); - result_cnt++; - } - } - + + fill_maat_rule(&(result[i]), &(compile_rule->head), + compile_rule->service_defined, compile_rule->head.serv_def_len); } - if(result_cnt>0) + if(scan_ret>0) { alignment_int64_array_add(feather->hit_cnt, thread_num, 1); } - if(region_hit_num==0&&result_cnt>0) + if(region_hit_num==0&&scan_ret>0) { alignment_int64_array_add(feather->not_grp_hit_cnt, thread_num, 1); } - return result_cnt; + return scan_ret; } -int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num, Maat_scanner* scanner) +int match_district(struct _OUTER_scan_status_t *_mid, scan_result_t *region_hit, int region_hit_num) { - struct Maat_group_inner* group_rule=NULL; + struct Maat_region_inner* region=NULL; int i=0; - int district_id=-1,region_id=-1; int ret_region_num=region_hit_num; while(i0&&district_id!=_mid->district_id) + region=(struct Maat_region_inner*)(region_hit[i].tag); + if(region->district_id!=_mid->district_id) { ret_region_num--; //todo: replace memmove with high performance solution. @@ -568,7 +365,7 @@ int hit_pos_RS2Maat(struct sub_item_pos_t* maat_sub_item,int size,rule_result_t* } return k; } -int fill_region_hit_detail(const char* scan_buff, const scan_hit_status* _mid, +int fill_region_hit_detail(const char* scan_buff, const Maat_hierarchy_compile_mid* mid, scan_result_t *region_hit, int region_cnt, struct Maat_rule_t* compile, int compile_cnt, int Nth_scan, @@ -578,13 +375,12 @@ int fill_region_hit_detail(const char* scan_buff, const scan_hit_status* _mid, int i=0; size_t j=0; char r_in_c_flag[region_cnt]; - int district_id=-1; //Indifference + memset(r_in_c_flag, 0, sizeof(r_in_c_flag)); - struct Maat_group_inner* group_rule=NULL; + struct Maat_region_inner* region=NULL; - - struct Maat_hit_path_inner hit_paths[MAX_SCANNER_HIT_NUM]; + struct Maat_hierarchy_hit_path* hit_paths[MAX_SCANNER_HIT_NUM]; struct Maat_hit_path_t condition; size_t hit_path_cnt=0; int Nth_hit_region=0; @@ -593,14 +389,14 @@ int fill_region_hit_detail(const char* scan_buff, const scan_hit_status* _mid, for(i=0; ihit_path_qhead, &condition, hit_paths, MAX_SCANNER_HIT_NUM); + hit_path_cnt=Maat_hierarchy_hit_path_select0(mid, &condition, hit_paths, MAX_SCANNER_HIT_NUM); for(j=0; jNth_hit_region; + hit_detail[i].region_pos[j].region_id=hit_paths[j]->path.region_id; hit_detail[i].region_pos[j].sub_item_num=region_hit[Nth_hit_region].rnum; hit_pos_RS2Maat(hit_detail[i].region_pos[j].sub_item_pos, MAAT_MAX_EXPR_ITEM_NUM, region_hit[Nth_hit_region].result, region_hit[Nth_hit_region].rnum, scan_buff); @@ -613,10 +409,10 @@ int fill_region_hit_detail(const char* scan_buff, const scan_hit_status* _mid, { if(r_in_c_flag[j]==0) { - group_rule=(struct Maat_group_inner*)(region_hit[j].tag); + region=(struct Maat_region_inner*)(region_hit[j].tag); hit_detail[i].config_id=-2; hit_detail[i].hit_region_cnt=1; - hit_detail[i].region_pos[0].region_id=exprid2region_id(group_rule, region_hit[j].expr_id, &district_id, scanner); + hit_detail[i].region_pos[0].region_id=region->region_id; hit_detail[i].region_pos[0].sub_item_num=region_hit[j].rnum; hit_pos_RS2Maat(hit_detail[i].region_pos[0].sub_item_pos,MAAT_MAX_EXPR_ITEM_NUM, region_hit[j].result, region_hit[j].rnum, scan_buff); @@ -647,11 +443,10 @@ struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* fe if(is_hit_region==1) { - if(_mid->inner==NULL) + if(_mid->compile_mid==NULL) { _mid->compile_mid=Maat_hierarchy_compile_mid_new(feather->scanner->hier, thread_num); - _mid->inner=scan_hit_status_new(); - alignment_int64_array_add(feather->inner_mid_cnt,thread_num,1); + alignment_int64_array_add(feather->compile_mid_cnt, thread_num, 1); } } return _mid; @@ -704,7 +499,7 @@ int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char } ret=json2iris(json_buff, maat_json_fn, - feather->compile_tn, feather->group_tn, + feather->compile_tn, feather->group2compile_tn, feather->group2group_tn, NULL, feather->json_ctx.iris_file, sizeof(feather->json_ctx.iris_file), @@ -748,7 +543,8 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* goto failed; } Maat_table_get_compile_table_name(feather->table_mgr, feather->compile_tn, sizeof(feather->compile_tn)); - Maat_table_get_group_table_name(feather->table_mgr, feather->group_tn, sizeof(feather->group_tn)); + Maat_table_get_group2compile_table_name(feather->table_mgr, feather->group2compile_tn, sizeof(feather->group2compile_tn)); + Maat_table_get_group2group_table_name(feather->table_mgr, feather->group2group_tn, sizeof(feather->group2group_tn)); feather->logger=logger; feather->scan_thread_num=max_thread_num; @@ -759,7 +555,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* feather->rule_scan_type=2; feather->thread_call_cnt=alignment_int64_array_alloc(max_thread_num); feather->outer_mid_cnt=alignment_int64_array_alloc(max_thread_num); - feather->inner_mid_cnt=alignment_int64_array_alloc(max_thread_num); + feather-> compile_mid_cnt=alignment_int64_array_alloc(max_thread_num); feather->hit_cnt=alignment_int64_array_alloc(max_thread_num); feather->not_grp_hit_cnt=alignment_int64_array_alloc(max_thread_num); feather->maat_version=0; @@ -1223,24 +1019,20 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, return 1; } -void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user) +void rule_ex_data_new_cb(void * user_data, void * param) { - struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user; - struct Maat_compile_inner *relation=(struct Maat_compile_inner *)data; - struct Maat_compile_rule* compile_rule=relation->compile; + struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)param; + struct Maat_compile_rule* compile=(struct Maat_compile_rule*)user_data; MAAT_RULE_EX_DATA ad=NULL; - if(compile_rule==NULL|| compile_rule->ref_table->table_id!=ex_desc->table_id) + if(compile->ref_table->table_id!=ex_desc->table_id) { return; } - - pthread_rwlock_wrlock(&(relation->rwlock)); - ad=rule_ex_data_new(&(relation->compile->head), - relation->compile->service_defined, + ad=rule_ex_data_new(&(compile->head), + compile->service_defined, ex_desc); - compile_rule->ads[ex_desc->idx]=ad; - pthread_rwlock_unlock(&(relation->rwlock)); + compile->ads[ex_desc->idx]=ad; return; } int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name, @@ -1272,7 +1064,7 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table if(_feather->scanner!=NULL) { - MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_ex_desc); + Maat_hierarchy_compile_user_data_iterate(_feather->scanner->hier, rule_ex_data_new_cb, compile_ex_desc); } pthread_mutex_unlock(&(_feather->background_update_mutex)); @@ -1282,24 +1074,19 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - struct Maat_compile_inner *relation=NULL; struct Maat_compile_rule* compile=NULL; const struct compile_table_schema* compile_desc=NULL; const struct compile_ex_data_idx* ex_desc=NULL; MAAT_RULE_EX_DATA ad=NULL; - - relation=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); - if(relation==NULL||relation->compile==NULL) + compile=(struct Maat_compile_rule*)Maat_hierarchy_compile_read_user_data(_feather->scanner->hier, rule->config_id); + if(compile==NULL) { return NULL; } - pthread_rwlock_rdlock(&(relation->rwlock)); - compile=relation->compile; compile_desc=&(compile->ref_table->compile); assert(idxex_data_num); ex_desc=compile_desc->ex_desc+idx; ex_desc->dup_func(ex_desc->idx, &ad, compile->ads+idx, ex_desc->argl,ex_desc->argp); - pthread_rwlock_unlock(&(relation->rwlock)); return ad; } @@ -1498,7 +1285,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS) { - hit_region_cnt=match_district(_mid,region_result,hit_region_cnt, my_scanner); + hit_region_cnt=match_district(_mid, region_result, hit_region_cnt); } if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid)) { @@ -1510,11 +1297,10 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id struct scan_region_hit_wraper region_hit_wraper; scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, virtual_table_id, _mid->scan_cnt); - compile_ret=region_compile(_feather,_mid->inner, + compile_ret=region_compile(_feather, _mid->compile_mid, ®ion_hit_wraper, result, rule_num, thread_num); - Maat_hierarchy_region_compile(struct Maat_hierarchy_compile_mid * mid, int * region_ids, int * virtual_tables, size_t n_region, int Nth_scan, void * * user_data_array, size_t ud_array_sz) assert(_mid->is_last_region<2); if(_mid->is_last_region==1) { @@ -1522,7 +1308,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } if(hit_region_cnt>0&&hit_detail!=NULL&&_feather->rule_scan_type!=0) { - *detail_ret=fill_region_hit_detail(data, _mid->inner, + *detail_ret=fill_region_hit_detail(data, _mid->compile_mid, region_result, hit_region_cnt, result, compile_ret, _mid->scan_cnt, @@ -1621,7 +1407,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id struct scan_region_hit_wraper region_hit_wraper; scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_ret, _mid->is_last_region, virtual_table_id, _mid->scan_cnt); - compile_ret=region_compile(_feather,_mid->inner, + compile_ret=region_compile(_feather,_mid->compile_mid, ®ion_hit_wraper, result, rule_num, thread_num); @@ -1706,7 +1492,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id struct scan_region_hit_wraper region_hit_wraper; scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, virtual_table_id, _mid->scan_cnt); - compile_ret=region_compile(_feather,_mid->inner, + compile_ret=region_compile(_feather,_mid->compile_mid, ®ion_hit_wraper, result, rule_num, thread_num); @@ -1940,7 +1726,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id { region_hit_wraper.virtual_table_ids=region_rslt_virtual_table_id; } - compile_ret=region_compile(_feather,_mid->inner, + compile_ret=region_compile(_feather,_mid->compile_mid, ®ion_hit_wraper, result, rule_num, thread_num); @@ -2150,7 +1936,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para } if(hit_region_cnt>0&&sp->p_real_table->table_type==TABLE_TYPE_EXPR_PLUS) { - hit_region_cnt=match_district(_mid, region_result, hit_region_cnt, scanner); + hit_region_cnt=match_district(_mid, region_result, hit_region_cnt); } if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid)) { @@ -2163,7 +1949,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, sp->virtual_table_id, _mid->scan_cnt); - compile_ret=region_compile(sp->feather,_mid->inner, + compile_ret=region_compile(sp->feather, _mid->compile_mid, ®ion_hit_wraper, result, rule_num, sp->thread_num); @@ -2176,7 +1962,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para { if(sp->scan_buff!=NULL) { - *detail_ret=fill_region_hit_detail(sp->scan_buff, _mid->inner, + *detail_ret=fill_region_hit_detail(sp->scan_buff, _mid->compile_mid, region_result, hit_region_cnt, result, compile_ret, _mid->scan_cnt, @@ -2184,7 +1970,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para } else { - *detail_ret=fill_region_hit_detail(data, _mid->inner, + *detail_ret=fill_region_hit_detail(data, _mid->compile_mid, region_result, hit_region_cnt, result, compile_ret, _mid->scan_cnt, @@ -2408,7 +2194,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int struct scan_region_hit_wraper region_hit_wraper; scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, sp->virtual_table_id, _mid->scan_cnt); - compile_ret=region_compile(sp->feather,_mid->inner, + compile_ret=region_compile(sp->feather, _mid->compile_mid, ®ion_hit_wraper, result, rule_num, sp->thread_num); @@ -2460,21 +2246,19 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum { int ret=0; struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - struct Maat_compile_inner *compile_inner=NULL; + struct Maat_compile_rule* compile=NULL; switch(type) { case MAAT_RULE_SERV_DEFINE: - compile_inner=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); - if(compile_inner==NULL) + compile=(struct Maat_compile_rule*)Maat_hierarchy_compile_read_user_data(_feather->scanner->hier, rule->config_id); + if(compile) { - ret=0; + ret=MIN(size, compile->head.serv_def_len); + memcpy(value, compile->service_defined, ret); } else { - pthread_rwlock_rdlock(&(compile_inner->rwlock)); - ret=MIN(size,compile_inner->compile->head.serv_def_len); - memcpy(value,compile_inner->compile->service_defined,ret); - pthread_rwlock_unlock(&(compile_inner->rwlock)); + ret=0; } break; default: @@ -2491,7 +2275,7 @@ int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCA { return 0; } - _mid=grab_mid(mid,_feather, 0, 0); + _mid=grab_mid(mid, _feather, 0, 0); if(_mid==NULL) { _mid=_make_outer_status(_feather,0); @@ -2530,12 +2314,12 @@ int Maat_get_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_S struct _OUTER_scan_status_t* _mid=NULL; struct Maat_hit_path_t* paths; struct Maat_hit_path_t condition; - struct Maat_hit_path_inner *path_inner; + struct Maat_hierarchy_hit_path **path_inner=NULL; size_t n_read=0, i=0; int ret=0; _mid=grab_mid(mid, _feather, 0, 0); - if(_mid->inner==NULL) + if(_mid->compile_mid==NULL) { return 0; } @@ -2548,12 +2332,12 @@ int Maat_get_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_S } paths=(struct Maat_hit_path_t*)value; n_read=size/sizeof(struct Maat_hit_path_t); - path_inner=ALLOC(struct Maat_hit_path_inner, n_read); - hit_path_init(&condition); - n_read=scan_hit_status_select_hit_path_inner(&_mid->inner->hit_path_qhead, &condition, path_inner, n_read); + path_inner=ALLOC(struct Maat_hierarchy_hit_path*, n_read); + Maat_hit_path_init(&condition); + n_read=Maat_hierarchy_hit_path_select0(_mid->compile_mid, &condition, path_inner, n_read); for(i=0; ipath; } free(path_inner); ret=n_read; @@ -2618,11 +2402,11 @@ void Maat_clean_status(scan_status_t* mid) } _mid=(struct _OUTER_scan_status_t*)(*mid); alignment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1); - if(_mid->inner!=NULL) + if(_mid->compile_mid!=NULL) { - scan_hit_status_free(_mid->inner); - _mid->inner=NULL; - alignment_int64_array_add(_mid->feather->inner_mid_cnt, _mid->thread_num, -1); + Maat_hierarchy_compile_mid_free(_mid->compile_mid); + _mid->compile_mid=NULL; + alignment_int64_array_add(_mid->feather->compile_mid_cnt, _mid->thread_num, -1); } _mid->feather=NULL; free(_mid); diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 270cf8f..3ecb07b 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -244,20 +244,22 @@ int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) line[i]='0'; return 0; } -void serialize_group(const struct Maat_group_t* p_group, enum MAAT_OPERATION op, char* buff, size_t sz) + +void serialize_group2group(enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g, char* buff, size_t sz) { - if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD; - const char* vt_name="null"; - if(p_group->virtual_table_name!=NULL) - { - vt_name=p_group->virtual_table_name; - } - snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d\t%s", p_group->group_id, - p_group->parent_id, - op, - p_group->not_flag, - p_group->parent_type, - vt_name); + snprintf(buff, sz, "%d\t%d\t%d", g2g->group_id, + g2g->superior_group_id, + op); + return; +} +void serialize_group2compile(enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c, char* buff, size_t sz) +{ + snprintf(buff, sz, "%d\t%d\t%d\t%d\t%s\t%d", g2c->group_id, + g2c->compile_id, + op, + g2c->not_flag, + g2c->virtual_table_name?g2c->virtual_table_name:"null", + g2c->Nth_clause==0?1:g2c->Nth_clause); return; } void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int group_num, enum MAAT_OPERATION op, char* buff, size_t sz) @@ -276,7 +278,7 @@ void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_serv group_num); return; } -void serialize_region(const struct Maat_region_t* p, int group_id, char* buff, size_t sz) +void serialize_region(const struct Maat_cmd_region* p, int group_id, char* buff, size_t sz) { UNUSED size_t ret=0; switch(p->region_type) @@ -842,174 +844,7 @@ int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rul } return 0; } -int calculate_serial_rule_num(struct _Maat_cmd_inner_t* _cmd,int * new_region_cnt, int* new_group_cnt) -{ - int serial_num=0; - int i=0; - struct Maat_cmd_t* cmd=&(_cmd->cmd); - serial_num++;//compile rule - for(i=0;igroup_num;i++) - { - serial_num++; - if(cmd->groups[i].regions==NULL) - { - continue; - } - if(_cmd->op==MAAT_OP_ADD) - { - *new_region_cnt+=cmd->groups[i].region_num; - (*new_group_cnt)++; - } - //for MAAT_OP_DEL, if the group's refcnt>1, group->region_num=0. - //so it's OK to add. - serial_num+=cmd->groups[i].region_num; - } - return serial_num; -} -int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _cmd) -{ - int i=0,j=0,grp_idx=0; - struct Maat_cmd_t* cmd=&(_cmd->cmd); - struct Maat_group_t* group_cmd=NULL; - struct Maat_region_t* region_cmd=NULL; - struct Maat_compile_inner *relation=NULL; - struct Maat_group_inner* group_inner=NULL; - struct Maat_region_inner* region_inner=NULL; - void* logger=feather->logger; - - int config_id=cmd->compile.config_id; - if(feather->scanner==NULL) - { - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_command - ,"MAAT not ready."); - return -1; - } - relation=(struct Maat_compile_inner *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id); - //Operation on relation is thread safe, no immediate memory free when delete a compile rule or a scanner. - //In another words, if the relation is accessable from compile means, its was valid in at least 10 seconds (garbage bury). - if(relation==NULL) - { - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_command - ,"config %d not exist." - ,config_id); - return -1; - } - pthread_rwlock_rdlock(&(relation->rwlock)); - cmd->group_num=relation->group_cnt; - assert(cmd->groups==NULL); - cmd->groups=ALLOC(struct Maat_group_t, cmd->group_num); - for(i=0;igroup_boundary;i++) - { - group_inner=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i); - if(group_inner==NULL) - { - continue; - } - group_cmd=&(cmd->groups[grp_idx]); - group_cmd->group_id=group_inner->group_id; - - if(group_inner->ref_by_parent_cnt>1) - { - continue; - } - group_cmd->region_num=group_inner->region_cnt; - group_cmd->regions=ALLOC(struct Maat_region_t, group_cmd->region_num); - for(j=0;jregion_boundary;j++) - { - region_inner=(struct Maat_region_inner*)dynamic_array_read(group_inner->regions,j); - if(region_inner==NULL) - { - continue; - } - region_cmd=&(group_cmd->regions[j]); - region_cmd->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, region_inner->table_id)); - region_cmd->region_id=region_inner->region_id; - //NOTICE: region_type only avilable when OP_ADD, - region_cmd->region_type=REGION_EXPR; - } - grp_idx++; - } - pthread_rwlock_unlock(&(relation->rwlock)); - return 0; -} - -int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd, struct serial_rule_t* list, int size) -{ - struct Maat_group_t* p_group=NULL; - struct Maat_region_t* p_region=NULL; - struct Maat_rule_t* p_m_rule=NULL; - struct Maat_cmd_t* cmd=&(_cmd->cmd); - enum MAAT_OPERATION op=_cmd->op; - - int rule_num=0,i=0,j=0; - p_m_rule=&(cmd->compile); - char line[MAX_TABLE_LINE_SIZE]; - time_t timeout=0; - if(_cmd->cmd.expire_after>0) - { - timeout=feather->server_time+_cmd->cmd.expire_after; - } - if(op==MAAT_OP_ADD) - { - serialize_compile(p_m_rule, _cmd->huge_service_defined, cmd->group_num, MAAT_OP_ADD, line, sizeof(line)); - set_serial_rule(list+rule_num, MAAT_OP_ADD, cmd->compile.config_id, cmd->label_id, feather->compile_tn, line, timeout); - } - else - { - set_serial_rule(list+rule_num, op, cmd->compile.config_id, cmd->label_id, feather->compile_tn, NULL, timeout); - } - rule_num++; - for(i=0;igroup_num;i++) - { - p_group=&(cmd->groups[i]); - if(op==MAAT_OP_ADD) - { - if(feather->AUTO_NUMBERING_ON==1) - { - p_group->group_id=feather->base_grp_seq; - feather->base_grp_seq++; - } - p_group->parent_id=p_m_rule->config_id; - p_group->parent_type=PARENT_TYPE_COMPILE; - serialize_group(p_group, MAAT_OP_ADD, line, sizeof(line)); - set_serial_rule(list+rule_num, MAAT_OP_ADD, p_group->group_id, 0, feather->group_tn, line, timeout); - } - else - { - set_serial_rule(list+rule_num,op,p_group->group_id,0,feather->group_tn,NULL,timeout); - } - rule_num++; - if(p_group->regions==NULL)//group reuse. - { - continue; - } - for(j=0;jregion_num;j++) - { - p_region=&(p_group->regions[j]); - if(op==MAAT_OP_ADD) - { - if(feather->AUTO_NUMBERING_ON==1) - { - p_region->region_id=feather->base_rgn_seq; - feather->base_rgn_seq++; - } - serialize_region(p_region, p_group->group_id, line, sizeof(line)); - set_serial_rule(list+rule_num,MAAT_OP_ADD - ,p_region->region_id,0,p_region->table_name,line,timeout); - } - else - { - set_serial_rule(list+rule_num,op - ,p_region->region_id,0,p_region->table_name,NULL,timeout); - - } - rule_num++; - } - } - assert(rule_num<=size); - return rule_num; -} int mr_transaction_success(redisReply* data_reply) { if(data_reply->type==REDIS_REPLY_NIL) @@ -1380,58 +1215,7 @@ error_out: return success_cnt; } -int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) -{ - int i=0, j=0; - const char *table_name=NULL; - int table_id=0; - struct Maat_group_t* p_group=NULL; - struct Maat_region_t* p_region=NULL; - enum MAAT_TABLE_TYPE table_type; - struct Maat_compile_inner *compile_rule=NULL; - if(feather->scanner!=NULL) - { - compile_rule=(struct Maat_compile_inner*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id); - if(compile_rule!=NULL) - { - MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module - ,"Maat rule %d already exisits.",cmd->compile.config_id); - return -1; - } - } - for(i=0;igroup_num;i++) - { - p_group=&(cmd->groups[i]); - for(j=0;jregion_num;j++) - { - p_region=&(p_group->regions[j]); - table_name=p_region->table_name; - table_id=Maat_table_get_id_by_name(feather->table_mgr, table_name); - if(table_id<0) - { - MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module - ,"Unknown table %s of Maat_cmd_t[%d]->group[%d]->region[%d]." - ,table_name,cmd->compile.config_id,i,j); - - return -1; - } - table_type=type_region2table(p_region); - if(table_type!=Maat_table_get_type_by_id(feather->table_mgr, table_id)) - { - MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module - ,"Table %s not support region type %d of Maat_cmd_t[%d]->group[%d]->region[%d]." - ,table_name - ,p_region->region_type - ,cmd->compile.config_id,i,j); - return -1; - } - free((char*)p_region->table_name); - p_region->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, table_id)); - } - } - return 0; -} void check_maat_expiration(redisContext *ctx, void *logger) { unsigned int i=0,s_rule_num=0; @@ -2023,70 +1807,8 @@ void _maat_empty_region(struct Maat_region_t* p) return; } -struct Maat_cmd_t* Maat_create_cmd(const struct Maat_rule_t* rule, int group_num) -{ - int i=0; - struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t*)calloc(sizeof(struct _Maat_cmd_inner_t),1); - memcpy(&(_cmd->cmd.compile),rule,sizeof(_cmd->cmd.compile)); - _cmd->ref_cnt=1; - _cmd->cmd.group_num=group_num; - if(group_num>0) - { - _cmd->cmd.groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),group_num); - } - else - { - _cmd->cmd.groups=NULL; - } - for(i=0;icmd.groups[i].regions=(struct Maat_region_t*)calloc(sizeof(struct Maat_region_t),1); - _cmd->region_size[i]=1; - } - return (struct Maat_cmd_t*)_cmd; -} -int Maat_cmd_set_opt(struct Maat_cmd_t* cmd, enum MAAT_RULE_OPT type, const char* val, int size) -{ - struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t* )cmd; - int ret=-1; - switch(type) - { - case MAAT_RULE_SERV_DEFINE: - if(size>MAX_HUGE_SERVICE_DEFINE_LEN) - { - ret=-1; - } - else - { - if(_cmd->huge_service_defined!=NULL) - { - free(_cmd->huge_service_defined); - } - _cmd->huge_service_defined=(char*)calloc(sizeof(char),size+1); - memcpy(_cmd->huge_service_defined,val, size); - ret=0; - } - break; - default: - break; - } - return ret; -} -int Maat_cmd_set_group(Maat_feather_t feather,int group_id, const struct Maat_region_t* region, enum MAAT_OPERATION op) -{ - _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - if(_feather->AUTO_NUMBERING_ON==1) - { - return -1; - } - //struct Maat_group_inner* group_inner=NULL; - //group_inner=(struct Maat_group_inner*)HASH_fetch_by_id(_feather->scanner->group_hash, group_id); - //NOT implemented yet. - assert(0); - return 0; -} -int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_rule, int line_num ,enum MAAT_OPERATION op) +int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_rule, int line_num ,enum MAAT_OPERATION op) { int i=0, j=0; _Maat_feather_t* _feather=(_Maat_feather_t*)feather; @@ -2200,7 +1922,7 @@ error_out: return ret; } -int Maat_cmd_set_line(Maat_feather_t feather,const struct Maat_line_t* line_rule, enum MAAT_OPERATION op) +int Maat_cmd_set_line(Maat_feather_t feather,const struct Maat_cmd_line* line_rule, enum MAAT_OPERATION op) { int ret=0; ret=Maat_cmd_set_lines(feather,&line_rule, 1, op); @@ -2254,203 +1976,7 @@ int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value, freeReplyObject(reply); return 1; } -void Maat_add_region2cmd(struct Maat_cmd_t* cmd,int which_group,const struct Maat_region_t* region) -{ - struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t*)cmd; - struct Maat_region_t* dst=NULL; - struct Maat_group_t* p_group=NULL; - p_group=&(cmd->groups[which_group]); - assert(which_groupgroup_num); - assert(region->table_name!=NULL); - if(p_group->region_num==_cmd->region_size[which_group]) - { - _cmd->region_size[which_group]*=2; - p_group->regions=(struct Maat_region_t*)realloc(p_group->regions,sizeof(struct Maat_region_t)*_cmd->region_size[which_group]); - } - dst=&(p_group->regions[p_group->region_num]); - p_group->region_num++; - _maat_copy_region(dst, region); - return; -} -void Maat_free_cmd(struct Maat_cmd_t* cmd) -{ - struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t*)cmd; - int i=0,j=0; - _cmd->ref_cnt--; - if(_cmd->ref_cnt>0) - { - return; - } - for(i=0;igroup_num;i++) - { - for(j=0;jgroups[i].region_num;j++) - { - _maat_empty_region(&(cmd->groups[i].regions[j])); - } - free(cmd->groups[i].regions); - cmd->groups[i].regions=NULL; - } - free(cmd->groups); - cmd->groups=NULL; - if(_cmd->huge_service_defined!=NULL) - { - free(_cmd->huge_service_defined); - _cmd->huge_service_defined=NULL; - } - - _cmd->next=NULL; - free(_cmd); - return; -} -int Maat_format_cmd(struct Maat_cmd_t* rule, char* buffer, int size) -{ - //TODO - return 0; -} -int Maat_cmd(Maat_feather_t feather,struct Maat_cmd_t* raw_rule,enum MAAT_OPERATION op) -{ - int ret=0; - ret=Maat_cmd_append(feather,raw_rule,op); - if(ret<0) - { - return ret; - } - ret=Maat_cmd_commit(feather); - return ret; -} - -int Maat_cmd_append(Maat_feather_t feather,struct Maat_cmd_t* cmd,enum MAAT_OPERATION op) -{ - _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t*)cmd; - int ret=0; - _cmd->op=op; - assert(op==MAAT_OP_DEL||op==MAAT_OP_ADD||op==MAAT_OP_RENEW_TIMEOUT); - assert(_cmd->next==NULL); - if(op==MAAT_OP_RENEW_TIMEOUT) - { - assert(cmd->expire_after>0); - } - switch(op) - { - case MAAT_OP_DEL: - case MAAT_OP_RENEW_TIMEOUT: - ret=reconstruct_cmd(_feather, _cmd); - break; - case MAAT_OP_ADD: - ret=fix_table_name(_feather, cmd); - break; - } - if(ret<0) - { - return -1; - } - _cmd->ref_cnt++; - if(_feather->cmd_q_cnt==0) - { - assert(_feather->cmd_qtail==NULL); - assert(_feather->cmd_qtail==_feather->cmd_qhead); - _feather->cmd_qhead=_feather->cmd_qtail=_cmd; - } - else - { - _feather->cmd_qtail->next=_cmd; - _feather->cmd_qtail=_cmd; - } - _feather->cmd_q_cnt++; - return 0; -} - -int Maat_cmd_commit(Maat_feather_t feather) -{ - _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - - int ret=-1, i=0; - int new_region_num=0,new_group_num=0; - int serial_rule_num=0,serial_rule_idx=0; - UNUSED int transaction_success=1; - struct _Maat_cmd_inner_t* p=NULL,*n=NULL; - - redisContext* write_ctx=NULL; - redisReply* data_reply=NULL; - - struct serial_rule_t* s_rule=NULL; - if(_feather->input_mode!=SOURCE_REDIS) - { - return ret; - } - if(_feather->cmd_q_cnt==0) - { - return 0; - } - write_ctx=get_redis_ctx_for_write(_feather); - if(write_ctx==NULL) - { - goto error_out; - - } - for(i=0,p=_feather->cmd_qhead;i<_feather->cmd_q_cnt;i++) - { - serial_rule_num+=calculate_serial_rule_num(p, &new_region_num, &new_group_num); - p=p->next; - } - _feather->server_time=redis_server_time(write_ctx); - if(!_feather->server_time) - { - goto error_out; - } - if(_feather->AUTO_NUMBERING_ON==1) - { - data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_region_id_var, new_region_num); - if(data_reply->type!=REDIS_REPLY_INTEGER) - { - freeReplyObject(data_reply); - goto error_out; - } - _feather->base_rgn_seq=data_reply->integer-new_region_num; - freeReplyObject(data_reply); - - data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_group_id_var, new_group_num); - if(data_reply->type!=REDIS_REPLY_INTEGER) - { - freeReplyObject(data_reply); - goto error_out; - } - _feather->base_grp_seq=data_reply->integer-new_group_num; - freeReplyObject(data_reply); - } - s_rule=(struct serial_rule_t*)calloc(sizeof(struct serial_rule_t),serial_rule_num); - - for(i=0,p=_feather->cmd_qhead;i<_feather->cmd_q_cnt;i++) - { - serial_rule_idx+=build_serial_rule(_feather,p,s_rule+serial_rule_idx, serial_rule_num-serial_rule_idx); - p=p->next; - } - assert(serial_rule_idx==serial_rule_num); - transaction_success=0; - transaction_success=exec_serial_rule(write_ctx, s_rule,serial_rule_num,_feather->server_time,_feather->logger); - assert(transaction_success==serial_rule_num); - ret=_feather->cmd_q_cnt; - _feather->cmd_acc_num+=_feather->cmd_q_cnt; -error_out: - p=_feather->cmd_qhead; - for(i=0;i<_feather->cmd_q_cnt;i++) - { - n=p->next; - Maat_free_cmd((struct Maat_cmd_t* )p); - p=n; - } - _feather->cmd_qhead=_feather->cmd_qtail=NULL; - _feather->cmd_q_cnt=0; - - for(i=0;iserver_time=redis_server_time(write_ctx); - if(!_feather->server_time) - { - return -1; - } - struct serial_rule_t s_rule; - set_serial_rule(&s_rule, op, id, 0, table_name, line, 0); - int transaction_success=0; - transaction_success=exec_serial_rule(write_ctx, &s_rule, 1, _feather->server_time, _feather->logger); - empty_serial_rules(&s_rule); - return transaction_success; -} -int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num) -{ - struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather; + struct Maat_cmd_line line_cmd; char line[MAX_TABLE_LINE_SIZE]; - serialize_compile(compile, huge_service_defined, group_num, op, line, sizeof(line)); - int ret=_Maat_command_set_one_line(_feather, op, compile->config_id, table_name, line); + serialize_compile(compile, huge_service_defined, clause_num, op, line, sizeof(line)); + memset(&line_cmd, 0, sizeof(line_cmd)); + line_cmd.table_name=table_name; + line_cmd.rule_id=compile->config_id; + line_cmd.table_line=line; + line_cmd.label_id=label_id; + line_cmd.expire_after=expire_after; + int ret=Maat_cmd_set_line(feather, &line_cmd, op); return ret; } -int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_region_t* region, int group_id) +int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_region* region, int group_id) { - struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather; + struct Maat_cmd_line line_cmd; char line[MAX_TABLE_LINE_SIZE]; + serialize_region(region, group_id, line, sizeof(line)); - int ret=_Maat_command_set_one_line(_feather, op, region->region_id, region->table_name, line); + memset(&line_cmd, 0, sizeof(line_cmd)); + line_cmd.table_name=region->table_name; + line_cmd.rule_id=region->region_id; + line_cmd.table_line=line; + int ret=Maat_cmd_set_line(feather, &line_cmd, op); return ret; } -int Maat_command_raw_set_group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_group_t* group) +int Maat_command_raw_set_group2compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c) { - struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather; + struct Maat_cmd_line line_cmd; char line[MAX_TABLE_LINE_SIZE]; - serialize_group(group, op, line, sizeof(line)); - int ret=_Maat_command_set_one_line(_feather, op, group->group_id*1024*1204+group->parent_id, group->table_name, line); + + serialize_group2compile(op, g2c, line, sizeof(line)); + assert(g2c->group_id<1024*1024); + memset(&line_cmd, 0, sizeof(line_cmd)); + line_cmd.table_name=g2c->table_name; + line_cmd.rule_id=g2c->compile_id*1024*1024+g2c->group_id; + line_cmd.table_line=line; + int ret=Maat_cmd_set_line(feather, &line_cmd, op); return ret; } +int Maat_command_raw_set_group2group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g) +{ + struct Maat_cmd_line line_cmd; + char line[MAX_TABLE_LINE_SIZE]; + + assert(g2g->group_id<1024*1024); + serialize_group2group(op, g2g, line, sizeof(line)); + memset(&line_cmd, 0, sizeof(line_cmd)); + line_cmd.table_name=g2g->table_name; + line_cmd.rule_id=g2g->superior_group_id*1024*1024+g2g->group_id; + line_cmd.table_line=line; + int ret=Maat_cmd_set_line(feather, &line_cmd, op); + return ret; +} + int Maat_cmd_flushDB(Maat_feather_t feather) { _Maat_feather_t* _feather=(_Maat_feather_t*)feather; diff --git a/src/entry/Maat_garbage_collection.cpp b/src/entry/Maat_garbage_collection.cpp index c821ec3..eb98253 100644 --- a/src/entry/Maat_garbage_collection.cpp +++ b/src/entry/Maat_garbage_collection.cpp @@ -1,4 +1,10 @@ + +#include "Maat_utils.h" + +#include +#include #include +#include struct Maat_garbage_bag { @@ -9,7 +15,7 @@ struct Maat_garbage_bag void (* garbage_free)(void *garbage); TAILQ_ENTRY(Maat_garbage_bag) entries; }; -TAILQ_HEAD(Maat_garbage_bag, Maat_garbage_q); +TAILQ_HEAD(Maat_garbage_q, Maat_garbage_bag); struct Maat_garbage_bin { @@ -20,7 +26,7 @@ struct Maat_garbage_bin struct Maat_garbage_bin* Maat_garbage_bin_new(int default_timeout) { struct Maat_garbage_bin* bin=ALLOC(struct Maat_garbage_bin, 1); - TAILQ_INIT(bin->garbage_q); + TAILQ_INIT(&bin->garbage_q); bin->timeout_seconds=default_timeout; return bin; } @@ -37,7 +43,7 @@ void Maat_garbage_bin_free(struct Maat_garbage_bin* bin) free(bin); return; } -void Maat_garbage_bag(struct Maat_garbage_bin* bin, void* garbage, void (* func)(void *)) +void Maat_garbage_bagging(struct Maat_garbage_bin* bin, void* garbage, void (* func)(void *)) { struct Maat_garbage_bag* bag=ALLOC( struct Maat_garbage_bag, 1); bag->create_time=time(NULL); @@ -47,10 +53,10 @@ void Maat_garbage_bag(struct Maat_garbage_bin* bin, void* garbage, void (* func) TAILQ_INSERT_TAIL(&bin->garbage_q, bag, entries); bin->bag_cnt++; } -void Maat_garbage_collect(struct Maat_garbage_bin* bin) +void Maat_garbage_collect_routine(struct Maat_garbage_bin* bin) { struct Maat_garbage_bag* p=NULL, *tmp=NULL; - int n_clollected=0, n_bag=0; + size_t n_clollected=0, n_bag=0; time_t now=time(NULL); for(p=TAILQ_FIRST(&bin->garbage_q); p!=NULL; p=tmp) diff --git a/src/entry/Maat_hierarchy.cpp b/src/entry/Maat_hierarchy.cpp index d7acecb..a521593 100644 --- a/src/entry/Maat_hierarchy.cpp +++ b/src/entry/Maat_hierarchy.cpp @@ -1,13 +1,24 @@ -#include "uthash.h" +#include "Maat_hierarchy.h" +#include "Maat_utils.h" + +#include "uthash/uthash.h" +#include "uthash/utarray.h" +#include "igraph/igraph.h" +#include "bool_matcher.h" + +#include + +#include +#include + #define module_maat_hierarchy "MAAT_HIERARCHY" -const char* sql_create_hier_table[]={"create table MAAT_HIERARCHY_REGION(region_id interger PRIMARY KEY, group_id interger, district_id interger, table_id interger, expr_id_lb interger, expr_id_ub interger)", - "create table MAAT_HIERARCHY_GROUP(group_id interger PRIMARY KEY, vertex_id interger)", - "create table MAAT_HIERARCHY_COMPILE(compile_id interger PRIMARY KEY, declared_clause_num interger, user_data_pointer interger)", - "create table MAAT_HIERARCHY_CLAUSE(compile_id interger, nth_clause interger, not_flag interger, UNIQUE(compile_id, nth_clause))", - "create table MAAT_HIERARCHY_LITERAL(group_id interger, virtual_table_id interger, clause_id interger)", - NULL}; + +#define TO_CLAUSE_ID(Nth_clause, compile_id) ((long long)Nth_clause<<32|compile_id) +#define TO_CLAUSE_ID_COMPATBILE(vid, gid) ((unsigned long long)vid<<32|gid) +#define TO_LITERAL_ID(vt_id, group_id) ((long long)vt_id<<32|group_id) + struct Maat_hierarchy_group @@ -38,7 +49,7 @@ struct Maat_hierarchy_region struct Maat_hierarchy_literal_id { int group_id; - int vt_id; + int virtual_table_id; }; struct Maat_hierarchy_clause_id { @@ -80,6 +91,11 @@ struct Maat_hierarchy_compile UT_array *literal_ids; }; +static void _group_vertex_free(struct Maat_hierarchy_group* group) +{ + free(group->top_group_ids); + free(group); +} @@ -114,6 +130,115 @@ struct Maat_hierarchy void* logger; }; + +static struct Maat_hierarchy_literal* Maat_hierarchy_literal_new(struct Maat_hierarchy* hier, int group_id, int vt_id) +{ + struct Maat_hierarchy_literal* literal=ALLOC(struct Maat_hierarchy_literal, 1); + literal->literal_id.group_id=group_id; + literal->literal_id.virtual_table_id=vt_id; + literal->hash_clause_by_id=NULL; + + HASH_ADD(hh, hier->hash_literal_by_id, literal_id, sizeof(literal->literal_id), literal); + return literal; +} +static void Maat_hierarchy_literal_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_literal* literal) +{ + struct Maat_hierarchy_clause* clause=NULL, *tmp=NULL; + HASH_ITER(hh, literal->hash_clause_by_id, clause, tmp) + { + HASH_DELETE(hh, literal->hash_clause_by_id, clause); + free(clause); + } + HASH_DELETE(hh, hier->hash_literal_by_id, literal); + free(literal); +} +static int Maat_hierarchy_literal_join_clause(struct Maat_hierarchy_literal* literal, int not_flag, int Nth_clause, int compile_id) +{ + struct Maat_hierarchy_clause* clause=NULL; + struct Maat_hierarchy_clause_id clause_id={Nth_clause, compile_id}; + HASH_FIND(hh, literal->hash_clause_by_id, &clause_id, sizeof(clause_id), clause); + if(clause) + { + return -1; + } + clause=ALLOC(struct Maat_hierarchy_clause, 1); + clause->clause_id=clause_id; + clause->not_flag=not_flag; + + HASH_ADD(hh, literal->hash_clause_by_id, clause_id, sizeof(clause->clause_id), clause); + return 0; +} +static int Maat_hierarchy_literal_leave_clause(struct Maat_hierarchy_literal* literal, int Nth_clause, int compile_id) +{ + struct Maat_hierarchy_clause* clause=NULL; + struct Maat_hierarchy_clause_id clause_id={Nth_clause, compile_id}; + HASH_FIND(hh, literal->hash_clause_by_id, &clause_id, sizeof(clause_id), clause); + if(!clause) + { + return -1; + } + HASH_DELETE(hh, literal->hash_clause_by_id, clause); + free(clause); + return 0; +} + + +int compare_literal_id(const void *pa, const void *pb) +{ + struct Maat_hierarchy_literal_id *la=(struct Maat_hierarchy_literal_id *)pa; + struct Maat_hierarchy_literal_id *lb=(struct Maat_hierarchy_literal_id *)pb; + return TO_LITERAL_ID(la->virtual_table_id, la->group_id)-TO_LITERAL_ID(lb->virtual_table_id, lb->group_id); + +} + +static struct Maat_hierarchy_compile* Maat_hierarchy_compile_new(struct Maat_hierarchy* hier, int compile_id) +{ + struct Maat_hierarchy_compile* compile=NULL; + compile=ALLOC(struct Maat_hierarchy_compile, 1); + compile->compile_id=compile_id; + HASH_ADD_INT(hier->hash_compile_by_id, compile_id, compile); + utarray_new(compile->literal_ids, &ut_literal_id_icd); + return compile; +} +static void Maat_hierarchy_compile_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_compile* compile) +{ + HASH_DEL(hier->hash_compile_by_id, compile); + if(hier->compile_user_data_free) + { + hier->compile_user_data_free(compile->user_data); + } + utarray_free(compile->literal_ids); + free(compile); +} + +static struct Maat_hierarchy_region* Maat_hierarchy_region_new(struct Maat_hierarchy* hier, int region_id, int group_id, int table_id, struct Maat_hierarchy_group* parent_group, void* user_data) +{ + struct Maat_hierarchy_region* region=NULL; + region=ALLOC(struct Maat_hierarchy_region, 1); + region->group_id=group_id; + region->region_id=region_id; + region->table_id=table_id; + region->ref_parent_group=parent_group; + region->user_data=user_data; + HASH_ADD_INT(hier->hash_region_by_id, region_id, region); + parent_group->ref_by_region_cnt++; + return region; +} +static void Maat_hierarchy_region_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_region* region) +{ + HASH_DELETE(hh, hier->hash_region_by_id, region); + region->ref_parent_group->ref_by_region_cnt--; + if(region->user_data) + { + hier->region_user_data_free(region->user_data); + region->user_data=NULL; + } + free(region); + return; +} + + + struct Maat_hierarchy* Maat_hierarchy_new(int thread_num, void* mesa_handle_logger) { struct Maat_hierarchy* hier=ALLOC(struct Maat_hierarchy, 1); @@ -134,11 +259,10 @@ struct Maat_hierarchy* Maat_hierarchy_new(int thread_num, void* mesa_handle_logg } void Maat_hierarchy_free(struct Maat_hierarchy* hier) { - int ret=0; - struct Maat_hierarchy_compile* compile=NULL, tmp_compile=NULL; - struct Maat_hierarchy_group* group=NULL, tmp_group=NULL; - struct Maat_hierarchy_literal* literal=NULL, tmp_literal=NULL; - struct Maat_hierarchy_region* region=NULL, tmp_region=NULL; + struct Maat_hierarchy_compile* compile=NULL, *tmp_compile=NULL; + struct Maat_hierarchy_group* group=NULL, *tmp_group=NULL; + struct Maat_hierarchy_literal* literal=NULL, *tmp_literal=NULL; + struct Maat_hierarchy_region* region=NULL, *tmp_region=NULL; pthread_rwlock_wrlock(&hier->rwlock); //Reference: https://troydhanson.github.io/uthash/userguide.html#_what_can_it_do @@ -190,31 +314,6 @@ void Maat_hierarchy_set_region_user_data_free_func(struct Maat_hierarchy* hier, } -#define TO_CLAUSE_ID(Nth_clause, compile_id) ((long long)Nth_clause<<32|compile_id) -#define TO_CLAUSE_ID_COMPATBILE(vid, gid) ((unsigned long long)vid<<32|gid) -#define TO_LITERAL_ID(vt_id, group_id) ((long long)vt_id<<32|group_id) - - -static struct Maat_hierarchy_compile* Maat_hierarchy_compile_new(struct Maat_hierarchy* hier, int policy_id) -{ - struct Maat_hierarchy_compile* compile=NULL; - compile=ALLOC(struct Maat_hierarchy_compile, 1); - compile->compile_id=policy_id; - HASH_ADD_INT(hier->hash_compile_by_id, policy_id, compile); - utarray_new(compile->literal_ids, &ut_literal_id_icd); - return compile; -} -static void Maat_hierarchy_compile_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_compile* compile) -{ - HASH_DEL(hier->hash_compile_by_id, compile); - if(hier->compile_user_data_free) - { - hier->compile_user_data_free(compile->user_data); - } - utarray_free(compile->literal_ids); - free(compile); -} - int Maat_hierarchy_compile_add(struct Maat_hierarchy* hier, int compile_id, int declared_clause_num, void* user_data) { int ret=0; @@ -231,7 +330,7 @@ int Maat_hierarchy_compile_add(struct Maat_hierarchy* hier, int compile_id, int } else { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, "Add compile %d failed, compile is already exisited.", compile_id); ret=-1; @@ -254,7 +353,7 @@ int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id) } else { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, "Remove compile %d failed, compile is not exisited.", compile_id); ret=-1; @@ -263,8 +362,7 @@ int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id) pthread_rwlock_unlock(&hier->rwlock); return ret; } - -void* Maat_hierarchy_compile_dettach_user_data(struct Maat_hierarchy* hier, int compile_id) +static void* Maat_hier_get_user_data(struct Maat_hierarchy* hier, int compile_id, int is_dettach) { struct Maat_hierarchy_compile* compile=NULL; void* ret=NULL; @@ -274,12 +372,39 @@ void* Maat_hierarchy_compile_dettach_user_data(struct Maat_hierarchy* hier, int if(compile) { ret=compile->user_data; - compile->user_data=NULL; + if(is_dettach) + { + compile->user_data=NULL; + } } pthread_rwlock_unlock(&hier->rwlock); return ret; -} +} +void* Maat_hierarchy_compile_dettach_user_data(struct Maat_hierarchy* hier, int compile_id) +{ + void* user_data=NULL; + user_data=Maat_hier_get_user_data(hier, compile_id, 1); + return user_data; +} +void* Maat_hierarchy_compile_read_user_data(struct Maat_hierarchy* hier, int compile_id) +{ + void* user_data=NULL; + user_data=Maat_hier_get_user_data(hier, compile_id, 0); + return user_data; + +} +void Maat_hierarchy_compile_user_data_iterate(struct Maat_hierarchy* hier, void (*callback)(void *user_data, void* apram), void* param) +{ + struct Maat_hierarchy_compile* compile=NULL, *tmp_compile=NULL; + pthread_rwlock_rdlock(&hier->rwlock); + HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) + { + callback(compile->user_data, param); + } + pthread_rwlock_unlock(&hier->rwlock); + return; +} struct Maat_hierarchy_group* Maat_hierarchy_group_new(struct Maat_hierarchy* hier, int group_id) { struct Maat_hierarchy_group* group=NULL; @@ -315,10 +440,17 @@ static size_t effective_vertices_count(igraph_vector_t *vids) } return i; } +size_t print_igraph_vector(igraph_vector_t *v, char* buff, size_t sz) { + long int i; + int printed=0; + for (i=0; iref_by_compile_cnt==0&&group->ref_by_group_cnt==0); @@ -327,7 +459,7 @@ static void Maat_hierarchy_group_free(struct Maat_hierarchy* hier, struct Maat_h if(igraph_vector_size(&v)>0) { print_igraph_vector(&v, buff, sizeof(buff)); - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, "Del group %d exception, still reached by %s.", group->vertex_id, buff); assert(0); @@ -340,62 +472,11 @@ static void Maat_hierarchy_group_free(struct Maat_hierarchy* hier, struct Maat_h HASH_DELETE(hh_group_id, hier->hash_group_by_id, group); HASH_DELETE(hh_vertex_id, hier->hash_group_by_vertex, group); - free(group->top_group_ids); - free(group); + _group_vertex_free(group); return; } -static struct Maat_hierarchy_literal* Maat_hierarchy_literal_new(struct Maat_hierarchy* hier, int group_id, int vt_id) -{ - struct Maat_hierarchy_literal* literal=ALLOC(struct Maat_hierarchy_literal, 1); - literal->literal_id.group_id=group_id; - literal->literal_id.vt_id=vt_id; - literal->hash_clause_by_id=NULL; - - HASH_ADD(hh, hier->hash_literal_by_id, literal_id, sizeof(literal->literal_id), literal); - return literal; -} -static void Maat_hierarchy_literal_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_literal* literal) -{ - struct Maat_hierarchy_clause* clause=NULL, *tmp=NULL; - HASH_ITER(hh, literal->hash_clause_by_id, clause, tmp) - { - HASH_DELETE(hh, literal->hash_clause_by_id, clause); - free(clause); - } - HASH_DELETE(hh, hier->hash_literal_by_id, literal); - free(literal); -} -static int Maat_hierarchy_literal_join_clause(struct Maat_hierarchy_literal* literal, int not_flag, int Nth_clause, int compile_id) -{ - struct Maat_hierarchy_clause* clause=NULL; - struct Maat_hierarchy_clause_id clause_id={Nth_clause, compile_id}; - HASH_FIND(hh, literal->hash_clause_by_id, &clause_id, sizeof(clause_id), clause); - if(clause) - { - return -1; - } - clause=ALLOC(struct Maat_hierarchy_clause, 1); - clause->clause_id=clause_id; - clause->not_flag=not_flag; - - HASH_ADD(hh, literal->hash_clause_by_id, clause_id, sizeof(clause->clause_id), clause); - return 0; -} -static int Maat_hierarchy_literal_leave_clause(struct Maat_hierarchy_literal* literal, int Nth_clause, int compile_id) -{ - struct Maat_hierarchy_clause* clause=NULL; - struct Maat_hierarchy_clause_id clause_id={Nth_clause, compile_id}; - HASH_FIND(hh, literal->hash_clause_by_id, &clause_id, sizeof(clause_id), clause); - if(!clause) - { - return -1; - } - HASH_DELETE(hh, literal->hash_clause_by_id, clause); - free(clause); - return 0; -} int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int Nth_clause, int compile_id) { int ret=0; @@ -404,7 +485,7 @@ int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_i struct Maat_hierarchy_literal_id literal_id={group_id, vt_id}; pthread_rwlock_wrlock(&hier->rwlock); - HASH_FIND(hh_group_id, hier->hash_group_by_id, group_id, sizeof(group_id), group); + HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); if(!group) { group=Maat_hierarchy_group_new(hier, group_id); @@ -414,11 +495,11 @@ int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_i { literal=Maat_hierarchy_literal_new(hier, group_id, vt_id); } - ret=Maat_hierarchy_literal_join_clause(literal, Nth_clause, compile_id); + ret=Maat_hierarchy_literal_join_clause(literal, not_flag, Nth_clause, compile_id); pthread_rwlock_unlock(&hier->rwlock); if(ret<0) { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, "Add group %d vt_id %d to clause %d of compile %d failed, group is already exisited.", group_id, vt_id, Nth_clause, compile_id); ret=-1; @@ -433,16 +514,16 @@ int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_i } int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int Nth_clause, int compile_id) { - struct Maat_hierarchy_compile* compile=NULL; struct Maat_hierarchy_group* group=NULL; struct Maat_hierarchy_literal* literal=NULL; struct Maat_hierarchy_literal_id literal_id={group_id, vt_id}; + int ret=0; pthread_rwlock_wrlock(&hier->rwlock); - HASH_FIND(hh_group_id, hier->hash_group_by_id, group_id, sizeof(group_id), group); + HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); if(!group) { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, "Remove group %d from compile %d failed, group is not exisited.", group_id, compile_id); goto error_out; @@ -450,7 +531,7 @@ int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int gr HASH_FIND(hh, hier->hash_literal_by_id, &literal_id, sizeof(literal_id), literal); if(!literal) { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, "Remove group %d from compile %d failed, literal is not exisited.", group_id, compile_id); goto error_out; @@ -458,7 +539,7 @@ int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int gr ret=Maat_hierarchy_literal_leave_clause(literal, Nth_clause, compile_id); if(ret<0) { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, "Remove group %d vt_id %d from clause %d of compile %d failed, clause is not exisited.", group_id, vt_id, Nth_clause, compile_id); goto error_out; @@ -468,7 +549,7 @@ int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int gr error_out: pthread_rwlock_unlock(&hier->rwlock); - return NULL; + return -1; } @@ -476,7 +557,7 @@ int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id, { int ret=0; igraph_integer_t edge_id; - struct Maat_hierarchy_group* group=NULL, superior_group=NULL; + struct Maat_hierarchy_group* group=NULL, *superior_group=NULL; pthread_rwlock_wrlock(&hier->rwlock); HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); @@ -493,7 +574,7 @@ int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id, ret=igraph_get_eid(&hier->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); if(edge_id>0)//No duplicated edges between two groups. { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, "Add group %d to group %d failed, relation already exisited.", group->group_id, superior_group->group_id); ret=-1; @@ -511,15 +592,14 @@ int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id, int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id) { int ret=0; - igraph_integer_t edge_id; - struct Maat_hierarchy_group* group=NULL, superior_group=NULL; + struct Maat_hierarchy_group* group=NULL, *superior_group=NULL; //No hash write operation, LOCK protection is unnecessary. HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); if(group==NULL) { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, "Del group %d from group %d failed, group %d not exisited.", group_id, superior_group_id, group_id); return -1; @@ -527,7 +607,7 @@ int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int grou HASH_FIND(hh_group_id, hier->hash_group_by_id, &superior_group_id, sizeof(superior_group_id), superior_group); if(superior_group==NULL) { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, "Del group %d from group %d failed, superior group %d not exisited.", group_id, superior_group_id, superior_group_id); return -1; @@ -552,41 +632,16 @@ int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int grou if(ret!=IGRAPH_SUCCESS||edge_num_before-edge_num_after!=1) { - assert(0) + assert(0); return -1; } - else - { - group->ref_by_group_cnt--; - return 0; - } -} -static struct Maat_hierarchy_region* Maat_hierarchy_region_new(struct Maat_hierarchy* hier, int region_id, int group_id, int table_id, struct Maat_hierarchy_group* parent_group, void* user_data) -{ - struct Maat_hierarchy_region* region=NULL; - region=ALLOC(struct Maat_hierarchy_region, 1); - region->group_id=group_id - region->region_id=region_id; - region->table_id=table_id; - region->ref_parent_group=parent_group; - region->user_data=user_data; - HASH_ADD_INT(hier->hash_region_by_id, region_id, region); - parent_group->ref_by_region_cnt++; - return region; -} -static void Maat_hierarchy_region_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_region* region) -{ - HASH_DELETE(hh, hier->hash_region_by_id, region); - region->ref_parent_group->ref_by_region_cnt--; - if(region->user_data) - { - hier->region_user_data_free(region->user_data); - region->user_data=NULL; - } - free(region); - return; + + group->ref_by_group_cnt--; + return 0; + } + int Maat_hierarchy_add_region_to_group(struct Maat_hierarchy* hier, int group_id, int region_id, int table_id, void* user_data) { //A region rule belongs to ONE group only. @@ -627,7 +682,7 @@ void* Maat_hierarchy_region_dettach_user_data(struct Maat_hierarchy* hier, int r HASH_FIND(hh, hier->hash_region_by_id, ®ion_id, sizeof(region_id), region); if(region) { - *ret=region->user_data; + ret=region->user_data; region->user_data=NULL; } pthread_rwlock_unlock(&hier->rwlock); @@ -637,7 +692,6 @@ int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int gro { struct Maat_hierarchy_group* group=NULL; struct Maat_hierarchy_region* region=NULL; - int ret=0; pthread_rwlock_wrlock(&hier->rwlock); HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); if(!group) @@ -646,7 +700,7 @@ int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int gro "Remove region %d from group %d failed, group is not existed.", region_id, group_id); - goto error_out + goto error_out; } HASH_FIND(hh, hier->hash_region_by_id, ®ion_id, sizeof(region_id), region); if(!region) @@ -659,7 +713,7 @@ int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int gro } assert(region->group_id==group->group_id); - Maat_hierarchy_region_free(hier, region) + Maat_hierarchy_region_free(hier, region); pthread_rwlock_unlock(&hier->rwlock); return 0; @@ -681,7 +735,8 @@ static struct bool_matcher* Maat_hierarchy_build_bool_matcher(struct Maat_hierar //STEP 1, for building a new bool matcher, we need to reset previous clause states of each compile. HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) { - memset(compile->clause_states); + memset(compile->clause_states, 0, sizeof(compile->clause_states)); + utarray_clear(compile->literal_ids); } @@ -696,14 +751,14 @@ static struct bool_matcher* Maat_hierarchy_build_bool_matcher(struct Maat_hierar { if(compile->clause_states[clause->clause_id.Nth_clause].in_use==1) { - assert(compile->clause_states[clause->clause_id.Nth_clause].not_flag==clause.not_flag); + assert(compile->clause_states[clause->clause_id.Nth_clause].not_flag==clause->not_flag); } else { compile->clause_states[clause->clause_id.Nth_clause].in_use=1; - compile->clause_states[clause->clause_id.Nth_clause].not_flag=clause.not_flag; + compile->clause_states[clause->clause_id.Nth_clause].not_flag=clause->not_flag; } - utarray_sort + utarray_push_back(compile->literal_ids, &literal->literal_id); } } } @@ -732,7 +787,7 @@ static struct bool_matcher* Maat_hierarchy_build_bool_matcher(struct Maat_hierar bool_expr_array[expr_cnt].item_num=j; expr_cnt++; } - + utarray_sort(compile->literal_ids, compare_literal_id); } //Final STEP, build the bool matcher. @@ -754,7 +809,7 @@ static struct bool_matcher* Maat_hierarchy_build_bool_matcher(struct Maat_hierar } static int Maat_hierarchy_build_top_groups(struct Maat_hierarchy* hier) { - struct Maat_hierarchy_group* group=NULL, tmp=NULL; + struct Maat_hierarchy_group* group=NULL, *tmp=NULL; struct Maat_hierarchy_group* superior_group=NULL; int tmp_vid=0; size_t i=0, top_group_cnt=0; @@ -780,7 +835,7 @@ static int Maat_hierarchy_build_top_groups(struct Maat_hierarchy* hier) pthread_rwlock_wrlock(&hier->rwlock); free(group->top_group_ids); - Maat_hierarchy_group_free(hier, group->group_id); + Maat_hierarchy_group_free(hier, group); pthread_rwlock_unlock(&hier->rwlock); continue; } @@ -792,13 +847,13 @@ static int Maat_hierarchy_build_top_groups(struct Maat_hierarchy* hier) { //fast path, group is only referenced by compile rules. top_group_cnt=1; - temp_group_ids=ALLOC(long long, top_group_cnt); + temp_group_ids=ALLOC(int, top_group_cnt); temp_group_ids[0]=group->group_id; } else { igraph_vector_t *vids=&(hier->dfs_vids); - igraph_dfs(&hier->group_graph), group->vertex_id, IGRAPH_OUT, + igraph_dfs(&hier->group_graph, group->vertex_id, IGRAPH_OUT, 0, vids, NULL, NULL, NULL, NULL, NULL, NULL); temp_group_ids=ALLOC(int, effective_vertices_count(vids)); @@ -810,7 +865,7 @@ static int Maat_hierarchy_build_top_groups(struct Maat_hierarchy* hier) { break; } - HASH_FIND(hh_vertex_id, hier->hash_group_by_vertex, tmp_vid, sizeof(tmp_vid), superior_group); + HASH_FIND(hh_vertex_id, hier->hash_group_by_vertex, &tmp_vid, sizeof(tmp_vid), superior_group); if(superior_group->ref_by_compile_cnt>0)//including itself { temp_group_ids[top_group_cnt]=superior_group->group_id; @@ -837,7 +892,7 @@ static int Maat_hierarchy_build_top_groups(struct Maat_hierarchy* hier) int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier) { int ret=0; - struct bool_matcher* new_bm=NULL, old_bm=NULL; + struct bool_matcher* new_bm=NULL, *old_bm=NULL; new_bm=Maat_hierarchy_build_bool_matcher(hier); old_bm=hier->bm; @@ -849,13 +904,8 @@ int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier) ret=Maat_hierarchy_build_top_groups(hier); return ret; } -struct Maat_hierarchy_hit_path_inner -{ - int Nth_hit_region; - struct Maat_hit_path_t path; - TAILQ_ENTRY(Maat_hierarchy_hit_path_inner) entries; -}; -TAILQ_HEAD(hit_path_q, Maat_hierarchy_hit_path_inner); + +TAILQ_HEAD(hit_path_q, Maat_hierarchy_hit_path); struct Maat_hierarchy_compile_mid { @@ -863,7 +913,7 @@ struct Maat_hierarchy_compile_mid int thread_num; int Nth_scan; size_t this_scan_region_hits; - int not_grp_compile_hitted_flag; + int not_clause_hitted_flag; size_t hit_path_cnt; struct hit_path_q hit_path_qhead; @@ -878,11 +928,11 @@ struct Maat_hierarchy_compile_mid* Maat_hierarchy_compile_mid_new(struct Maat_hi TAILQ_INIT(&mid->hit_path_qhead); mid->thread_num=thread_num; mid->ref_hier=hier; - return; + return mid; } void Maat_hierarchy_compile_mid_free(struct Maat_hierarchy_compile_mid* mid) { - struct Maat_hierarchy_hit_path_inner * tmp = TAILQ_FIRST(&mid->hit_path_qhead); + struct Maat_hierarchy_hit_path * tmp = TAILQ_FIRST(&mid->hit_path_qhead); while(tmp != NULL) { TAILQ_REMOVE(&mid->hit_path_qhead, tmp, entries); @@ -896,6 +946,11 @@ void Maat_hierarchy_compile_mid_free(struct Maat_hierarchy_compile_mid* mid) mid->ref_hier=NULL; free(mid); } +int Maat_hierarchy_compile_mid_has_NOT_clause(struct Maat_hierarchy_compile_mid* mid) +{ + return mid->not_clause_hitted_flag; +} + //return 1 if insert a unique id //return 0 if id is duplicated //return -1 if set is full @@ -924,12 +979,58 @@ int insert_clause_id(unsigned long long **set, size_t* size, size_t cnt, unsigne return 0; } } +void Maat_hit_path_init(struct Maat_hit_path_t* hit_path) +{ + hit_path->Nth_scan=-1; + hit_path->region_id=-1; + hit_path->sub_group_id=-1; + hit_path->top_group_id=-1; + hit_path->virtual_table_id=-1; + hit_path->compile_id=-1; +} +static size_t hit_path_select(const struct hit_path_q *hit_path_qhead, struct Maat_hit_path_t* condition, + struct Maat_hierarchy_hit_path** hit_paths, size_t n_path) +{ + + struct Maat_hierarchy_hit_path* p=NULL; + size_t i=0; + TAILQ_FOREACH(p, hit_path_qhead, entries) + { + if((condition->compile_id==p->path.compile_id||condition->compile_id<0) + && (condition->Nth_scan==p->path.Nth_scan||condition->Nth_scan<0) + && (condition->region_id=p->path.region_id||condition->region_id<0) + && (condition->sub_group_id=p->path.sub_group_id||condition->sub_group_id<0) + && (condition->top_group_id=p->path.top_group_id||condition->top_group_id<0)) + { + if(ihit_path_qhead, condition, hit_paths, n_path); + return ret; +} void Maat_hierarchy_compile_mid_udpate(struct Maat_hierarchy_compile_mid* mid, int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result) { - size_t i=0; - int ret=0; - struct Maat_hit_path_inner* hit_path=NULL; + int ret=0, i=0; + struct Maat_hierarchy_hit_path* hit_path=NULL; struct Maat_hierarchy_region* region=NULL; struct Maat_hierarchy_group* group=NULL; @@ -945,18 +1046,18 @@ void Maat_hierarchy_compile_mid_udpate(struct Maat_hierarchy_compile_mid* mid, i mid->Nth_scan=Nth_scan; } pthread_rwlock_rdlock(&hier->rwlock); - HASH_FIND_INT(hier->hash_region_by_id, region_id, region); + HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region); group=region->ref_parent_group; if(group->top_group_cnt==0) { - hit_path=ALLOC(struct Maat_hit_path_inner, 1); - hit_path_init(&(hit_path->path)); + hit_path=ALLOC(struct Maat_hierarchy_hit_path, 1); + Maat_hit_path_init(&(hit_path->path)); hit_path->Nth_hit_region=Nth_region_result; hit_path->path.Nth_scan=Nth_scan; hit_path->path.region_id=region_id; hit_path->path.sub_group_id=group->group_id; - hit_path->path.vt_id=virtual_table_id; + hit_path->path.virtual_table_id=virtual_table_id; TAILQ_INSERT_TAIL(&mid->hit_path_qhead, hit_path, entries); mid->hit_path_cnt++; } @@ -964,27 +1065,27 @@ void Maat_hierarchy_compile_mid_udpate(struct Maat_hierarchy_compile_mid* mid, i { for(i=0; itop_group_cnt; i++) { - hit_path=ALLOC(struct Maat_hit_path_inner, 1); - hit_path_init(&(hit_path->path)); + hit_path=ALLOC(struct Maat_hierarchy_hit_path, 1); + Maat_hit_path_init(&(hit_path->path)); hit_path->Nth_hit_region=Nth_region_result; hit_path->path.Nth_scan=Nth_scan; hit_path->path.region_id=region_id; hit_path->path.sub_group_id=group->group_id; hit_path->path.top_group_id=group->top_group_ids[i]; - hit_path->path.vt_id=virtual_table_id; + hit_path->path.virtual_table_id=virtual_table_id; TAILQ_INSERT_TAIL(&mid->hit_path_qhead, hit_path, entries); mid->hit_path_cnt++; - literal_id.vt_id=virtual_table_id; + literal_id.virtual_table_id=virtual_table_id; literal_id.group_id=group->top_group_ids[i]; - HASH_FIND(hh, hier->hash_literal_by_id, &literal_id, literal); + HASH_FIND(hh, hier->hash_literal_by_id, &literal_id, sizeof(literal_id), literal); if(!literal) { continue; } HASH_ITER(hh, literal->hash_clause_by_id, clause, tmp_clause) { - ret=insert_clause_id(mid->all_hit_clause_array, mid->all_hit_clause_array_sz, mid->all_hit_clause_cnt, TO_CLAUSE_ID(clause->clause_id.Nth_clause, clause->clause_id.compile_id)); + ret=insert_clause_id(&mid->all_hit_clause_array, &mid->all_hit_clause_array_sz, mid->all_hit_clause_cnt, TO_CLAUSE_ID(clause->clause_id.Nth_clause, clause->clause_id.compile_id)); mid->all_hit_clause_cnt+=ret; } } @@ -992,113 +1093,86 @@ void Maat_hierarchy_compile_mid_udpate(struct Maat_hierarchy_compile_mid* mid, i pthread_rwlock_unlock(&hier->rwlock); return; } -int compare_literal_id(const void *pa, const void *pb) -{ - struct Maat_hierarchy_literal_id *la=(struct Maat_hierarchy_literal_id *)pa; - struct Maat_hierarchy_literal_id *lb=(struct Maat_hierarchy_literal_id *)pb; - return TO_LITERAL_ID(la->vt_id, la->group_id)-TO_LITERAL_ID(lb->vt_id, lb->group_id); -} static size_t Maat_hierarchy_compile_mid_update_by_compile(struct Maat_hierarchy_compile_mid* mid, struct Maat_hierarchy_compile* compile) { - int i=0; - size_t r_in_c_cnt=0; - struct Maat_hierarchy_hit_path_inner* p; - struct Maat_hierarchy_literal_id literal_id={0,0}, *l; - for(i=0; ithis_scan_region_hits; i++) - { - - } + size_t r_in_c_cnt=0, this_scan_hit_region_cnt=0; + struct Maat_hierarchy_hit_path* p=NULL, *q=NULL; + struct Maat_hierarchy_literal_id literal_id={0,0}, *l=NULL; + struct Maat_hit_path_t condition; + + size_t n_exsited_path=0; + struct hit_path_q new_path_qhead; + TAILQ_INIT(&new_path_qhead); + + TAILQ_FOREACH(p, &mid->hit_path_qhead, entries) - { - literal_id={p->path.top_group_id, p->path.virtual_table_id}; + { + n_exsited_path=0; + if(p->path.Nth_scan==mid->Nth_scan) + { + this_scan_hit_region_cnt++; + } + if(p->path.compile_id==compile->compile_id) + { + continue; + } + literal_id.group_id=p->path.top_group_id; + literal_id.virtual_table_id=p->path.virtual_table_id; l=(struct Maat_hierarchy_literal_id*)utarray_find(compile->literal_ids, &literal_id, compare_literal_id); if(!l) { continue; } - assert(*l==literal_id); - - - } - mid->this_scan_region_hits=0; - - - - - - - size_t i=0, j=0; - struct Maat_hit_path_inner* p=NULL, *q=NULL; - struct bool_expr a_set; - unsigned char has_not=0; - - struct Maat_hit_path_t condition; - size_t n_exsited_path=0; - struct hit_path_q shared_path_qhead; - TAILQ_INIT(&shared_path_qhead); - struct Maat_hierarchy* hier=mid->ref_hier; - struct Maat_hierarchy_compile* compile=NULL; - make_group_set(compile_rule, &a_set, &has_not); - for(i=0; ihit_path_qhead, entries) + assert(l->group_id==literal_id.group_id && l->virtual_table_id==literal_id.virtual_table_id); + if(p->path.compile_id<0) { - n_exsited_path=0; - if(TO_RELATION_ID(p->path.vt_id, p->path.top_group_id)==a_set.items[i].item_id - && p->path.compile_id!=compile_rule->compile_id) - { - if(p->path.compile_id<0) - { - p->path.compile_id=compile_rule->compile_id; - } - else //current path already have a compile as endpoint, so we duplicate a new path. - { - condition=p->path; - condition.compile_id=compile_rule->compile_id; - n_exsited_path=scan_hit_status_select_hit_path_inner(&shared_path_qhead, &condition, NULL, 0); - if(n_exsited_path==0) - { - q=ALLOC(struct Maat_hit_path_inner, 1); - *q=*p; - q->path.compile_id=compile_rule->compile_id; - TAILQ_INSERT_TAIL(&shared_path_qhead, q, entries); - hit_status->hit_path_cnt++; - } - } - if(p->path.Nth_scan==mid->Nth_scan && n_exsited_path==0)//Compile was satisfied by new region hits. - { - j++; - } - } + p->path.compile_id=compile->compile_id; } + else //p->path.compile_id!=compile->compile_id, current literal already in a path that has a compile, may be a new path. + { + condition=p->path; + condition.compile_id=compile->compile_id; + n_exsited_path=hit_path_select(&new_path_qhead, &condition, NULL, 0); + if(n_exsited_path==0) + { + q=ALLOC(struct Maat_hierarchy_hit_path, 1); + *q=*p; + q->path.compile_id=compile->compile_id; + TAILQ_INSERT_TAIL(&new_path_qhead, q, entries); + mid->hit_path_cnt++; + } + + } + + if(p->path.Nth_scan==mid->Nth_scan && n_exsited_path==0)//Compile was satisfied by new region hits. + { + r_in_c_cnt++; + } + } - p = TAILQ_FIRST(&shared_path_qhead); + assert(this_scan_hit_region_cnt==mid->this_scan_region_hits); + mid->this_scan_region_hits=0; + + p = TAILQ_FIRST(&new_path_qhead); while(p != NULL) { - TAILQ_REMOVE(&shared_path_qhead, p, entries); - TAILQ_INSERT_TAIL(&hit_status->hit_path_qhead, p, entries); - p = TAILQ_FIRST(&shared_path_qhead); - } - return j; + TAILQ_REMOVE(&new_path_qhead, p, entries); + TAILQ_INSERT_TAIL(&mid->hit_path_qhead, p, entries); + p = TAILQ_FIRST(&new_path_qhead); + } + return r_in_c_cnt; } int Maat_hierarchy_region_compile(struct Maat_hierarchy_compile_mid* mid, int is_last_compile, void** user_data_array, size_t ud_array_sz) { - int bool_match_ret=0, ret=0; - size_t n_clauses=0; + int bool_match_ret=0, i=0; struct Maat_hierarchy* hier=mid->ref_hier; - struct Maat_hierarchy_literal_id literal_id={0,0}; - unsigned long long * clause_ids=NULL; - const struct Maat_hierarchy_region* region=NULL; - struct Maat_hierarchy_group* group=NULL; - struct Maat_hierarchy_literal* literal=NULL; - struct Maat_hierarchy_clause* clause=NULL, *tmp_clause=NULL; - struct Maat_hierarchy_compile* compile[ud_array_sz]; + struct Maat_hierarchy_compile* compile_array[ud_array_sz]; - size_t i=0, r_in_c_cnt=0, this_scan_region_hits=mid->this_scan_region_hits; + size_t r_in_c_cnt=0, this_scan_region_hits=mid->this_scan_region_hits; size_t ud_result_cnt=0; if(!hier->bm) { @@ -1108,21 +1182,21 @@ int Maat_hierarchy_region_compile(struct Maat_hierarchy_compile_mid* mid, int is pthread_rwlock_rdlock(&hier->rwlock); bool_match_ret=bool_matcher_match(hier->bm, mid->thread_num, mid->all_hit_clause_array, mid->all_hit_clause_cnt, - compile, ud_array_sz); + (void**)compile_array, ud_array_sz); for(i=0; i0 && !is_last_compile) + r_in_c_cnt=Maat_hierarchy_compile_mid_update_by_compile(mid, compile_array[i]); + if(compile_array[i]->not_clause_cnt>0 && !is_last_compile) { - mid->not_grp_compile_hitted_flag=1; + mid->not_clause_hitted_flag=1; } else { if(r_in_c_cnt>0 || //compile hitted becasue of new reigon this_scan_region_hits==0) //or hit a compile that refer a NOT-logic group in previous scan. { - user_data_array[ud_result_cnt]=compile[i]->user_data; - ud_result_cnt++ + user_data_array[ud_result_cnt]=compile_array[i]->user_data; + ud_result_cnt++; } } } diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index b5fef23..399cc4b 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -21,6 +21,9 @@ #include "Maat_rule.h" #include "Maat_rule_internal.h" #include "Maat_utils.h" +#include "Maat_hierarchy.h" +#include "Maat_garbage_collection.h" + #include "json2iris.h" #include "cJSON.h" #include "dynamic_array.h" @@ -28,8 +31,6 @@ #include "config_monitor.h" #include "map_str2int.h" -#include "rulescan.h" -#include "bool_matcher.h" #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" @@ -712,6 +713,18 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule) op_expr->rule_type=p_rule->rule_type; return; } +void Maat_region_inner_free(struct Maat_region_inner* region) +{ + free(region); +} + +void Maat_region_inner_cancel_last_expr_id(struct Maat_region_inner* region) +{ + assert(region->expr_id_cnt==region->expr_id_ub-region->expr_id_lb); + region->expr_id_ub--; + region->expr_id_cnt--; + return; +} struct Maat_scanner* create_maat_scanner(unsigned int version, _Maat_feather_t *feather) { @@ -721,8 +734,8 @@ struct Maat_scanner* create_maat_scanner(unsigned int version, _Maat_feather_t * struct Maat_scanner* scanner=NULL; scanner=ALLOC(struct Maat_scanner, 1); scanner->hier=Maat_hierarchy_new(scan_thread_num, feather->logger); - Maat_hierarchy_set_compile_user_data_free_func(scanner->hier, destroy_compile_rule); - Maat_hierarchy_set_region_user_data_free_func(scanner->hier, Maat_region_inner_free); + Maat_hierarchy_set_compile_user_data_free_func(scanner->hier, (void (*)(void*))destroy_compile_rule); + Maat_hierarchy_set_region_user_data_free_func(scanner->hier, (void (*)(void*))Maat_region_inner_free); scanner->district_map=map_create(); @@ -1038,18 +1051,7 @@ void Maat_region_inner_add_expr_id(struct Maat_region_inner* region, int expr_id region->expr_id_cnt++; return; } -void Maat_region_inner_free(struct Maat_region_inner* region) -{ - free(region); -} -void Maat_region_inner_cancel_last_expr_id(struct Maat_region_inner* region) -{ - assert(region->expr_id_cnt==region->expr_id_ub-region->expr_id_lb); - region->expr_id_ub--; - region->expr_id_cnt--; - return; -} int add_expr_rule(struct Maat_table_schema* table,struct db_str_rule_t* db_rule,struct Maat_scanner *scanner,void* logger) { @@ -1307,7 +1309,7 @@ int add_ip_rule(struct Maat_table_schema* table, struct db_ip_rule_t* db_ip_rule scan_rule_t* p_rule=NULL; struct op_expr_t* op_expr=NULL; struct Maat_region_inner* u_para=NULL; - int expr_id=0, district_id=-1, ret=0; + int expr_id=0, ret=0; u_para=Maat_region_inner_new(db_ip_rule->group_id, db_ip_rule->region_id, table->table_id, -1); ret=Maat_hierarchy_add_region_to_group(scanner->hier, db_ip_rule->group_id, db_ip_rule->region_id, table->table_id, u_para); @@ -1386,7 +1388,6 @@ int add_digest_rule(struct Maat_table_schema* table, struct db_digest_rule* db_r int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id, int rule_type, struct Maat_scanner *maat_scanner, void* logger) { int i=0; - int expr_num=0; struct Maat_table_runtime* table_rt=NULL; struct op_expr_t* op_expr=NULL; @@ -1426,7 +1427,7 @@ int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id assert(0); break; } - Maat_garbage_bag(maat_scanner->ref_garbage_bin, region, Maat_region_inner_free); + Maat_garbage_bagging(maat_scanner->ref_garbage_bin, region, (void (*)(void*))Maat_region_inner_free); return 0; } @@ -1437,7 +1438,7 @@ void update_group2compile_rule(struct Maat_table_schema* table, const char* tabl int ret=0; char virtual_table_name[MAX_TABLE_NAME_LEN]={0}; memset(&db_g2c_rule, 0, sizeof(db_g2c_rule)); - ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%d\t%s", &(db_g2c_rule.group_id), + ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%s\t%d", &(db_g2c_rule.group_id), &(db_g2c_rule.compile_id), &(db_g2c_rule.is_valid), &(db_g2c_rule.not_flag), @@ -1486,7 +1487,7 @@ void update_group2compile_rule(struct Maat_table_schema* table, const char* tabl if(ret==0) { table_rt->origin_rule_num++; - if(db_group2compile_rule.not_flag) + if(db_g2c_rule.not_flag) { table_rt->group2compile.not_flag_group++; } @@ -1528,10 +1529,6 @@ void update_group2group_rule(struct Maat_table_schema* table, const char* table_ if(ret==0) { table_rt->origin_rule_num++; - if(db_group2compile_rule.not_flag) - { - table_rt->group2compile.not_flag_group++; - } scanner->to_update_group_cnt++; } } @@ -1627,16 +1624,6 @@ void update_expr_rule(struct Maat_table_schema* table,const char* table_line,str table->udpate_err_cnt++; goto error_out; } - ret=sync_region(scanner->region_hash, - maat_str_rule->region_id, - maat_str_rule->group_id, - table->table_name[table->updating_name], - maat_str_rule->is_valid,logger); - if(ret<0) - { - table->udpate_err_cnt++; - goto error_out; - } if(maat_str_rule->is_valid==FALSE) { @@ -1876,16 +1863,6 @@ void update_ip_rule(struct Maat_table_schema* table, const char* table_line, str goto error_out; } } - ret=sync_region(scanner->region_hash, - ip_rule->region_id, - ip_rule->group_id, - table->table_name[table->updating_name], - ip_rule->is_valid,logger); - if(ret<0) - { - table->udpate_err_cnt++; - goto error_out; - } if(ip_rule->is_valid==FALSE) { ret=del_region_rule(table, @@ -2045,7 +2022,7 @@ void update_compile_rule(struct Maat_table_schema* table,const char* table_line ret=Maat_hierarchy_compile_remove(scanner->hier, m_rule_tmp.config_id); assert(ret==0); table_rt->origin_rule_num--; - Maat_garbage_bag(scanner->ref_garbage_bin, p_compile, destroy_compile_rule); + Maat_garbage_bagging(scanner->ref_garbage_bin, p_compile, (void (*)(void*))destroy_compile_rule); } else { @@ -2246,7 +2223,6 @@ void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char* void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* logger) { - struct bool_matcher *tmp1=NULL,*tmp2=NULL; MESA_htable_handle tmp_map=NULL; struct Maat_table_runtime* table_rt=NULL; struct ip_matcher* old_ip_matcher=NULL; @@ -2295,7 +2271,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* case TABLE_TYPE_IP_PLUGIN: Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt); old_ip_matcher=Maat_table_runtime_dettach_old_ip_matcher(table_rt); - Maat_garbage_bag(scanner->ref_garbage_bin, old_ip_matcher, ip_matcher_free); + Maat_garbage_bagging(scanner->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); break; default: break; @@ -2307,7 +2283,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* tmp_map=scanner->district_map; scanner->district_map=scanner->tmp_district_map; scanner->tmp_district_map=NULL; - Maat_garbage_bag(scanner->ref_garbage_bin, tmp_map, map_destroy) + Maat_garbage_bagging(scanner->ref_garbage_bin, tmp_map, (void (*)(void*))map_destroy); } scanner->last_update_time=time(NULL); scanner->gie_update_q_size=0; @@ -2592,7 +2568,7 @@ void *thread_rule_monitor(void *arg) old_scanner->version, feather->scanner->version); } feather->zombie_rs_stream+=alignment_int64_array_sum(old_scanner->ref_cnt,old_scanner->max_thread_num); - Maat_garbage_bag(feather->garbage_bin, old_scanner, destroy_maat_scanner); + Maat_garbage_bagging(feather->garbage_bin, old_scanner, (void (*)(void*))destroy_maat_scanner); } feather->update_tmp_scanner=NULL; feather->maat_version=feather->scanner->version; @@ -2628,7 +2604,7 @@ void *thread_rule_monitor(void *arg) Maat_garbage_bin_free(feather->garbage_bin); alignment_int64_array_free(feather->thread_call_cnt); - alignment_int64_array_free(feather->inner_mid_cnt); + alignment_int64_array_free(feather-> compile_mid_cnt); alignment_int64_array_free(feather->outer_mid_cnt); alignment_int64_array_free(feather->hit_cnt); alignment_int64_array_free(feather->not_grp_hit_cnt); diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index 9816bfb..edafff7 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -182,7 +182,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) } active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num); - inner_mid_cnt=alignment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num); + inner_mid_cnt=alignment_int64_array_sum(feather->compile_mid_cnt,feather->scan_thread_num); not_grp_hit_cnt=alignment_int64_array_sum(feather->not_grp_hit_cnt,feather->scan_thread_num); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version); diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table.cpp index 07dfda3..03601d4 100644 --- a/src/entry/Maat_table.cpp +++ b/src/entry/Maat_table.cpp @@ -682,12 +682,12 @@ int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char } return 0; } -int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) +int Maat_table_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) { int i=0; for(i=0; i< MAX_TABLE_NUM; i++) { - if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP) + if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP2COMPILE) { strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz); return 1; @@ -695,6 +695,20 @@ int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* } return 0; } +int Maat_table_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) +{ + int i=0; + for(i=0; i< MAX_TABLE_NUM; i++) + { + if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP2GROUP) + { + strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz); + return 1; + } + } + return 0; +} + const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id) { if(table_id>MAX_TABLE_NUM) diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 56d7a01..acae5b6 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -47,6 +47,8 @@ struct iris_description_t char index_path[MAX_PATH_LINE]; struct iris_table_t* group_table; + struct iris_table_t* group2group_table; + struct iris_table_t* group2compile_table; struct iris_table_t* compile_table; MESA_htable_handle group_name_map; MESA_htable_handle iris_table_map; @@ -123,7 +125,7 @@ static int get_region_seq(struct iris_description_t* iris_cfg) return sequence; } -int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_key, const char* encrypt_algo, const char*compile_tn,const char* group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger) +int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_key, const char* encrypt_algo, const char*compile_tn, const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger) { memset(iris_cfg,0,sizeof(struct iris_description_t)); snprintf(iris_cfg->tmp_iris_dir,sizeof(iris_cfg->tmp_iris_dir),"%s_iris_tmp",json_file); @@ -190,7 +192,8 @@ int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_k map_register(iris_cfg->str2int_map, "case plain",2); iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn, TABLE_TYPE_COMPILE); - iris_cfg->group_table=query_table_info(iris_cfg, group_tn, TABLE_TYPE_GROUP); + iris_cfg->group2compile_table=query_table_info(iris_cfg, group2compile_tn, TABLE_TYPE_GROUP2COMPILE); + iris_cfg->group2group_table=query_table_info(iris_cfg, group2compile_tn, TABLE_TYPE_GROUP2GROUP); if(encrypt_key && encrypt_algo) { @@ -795,12 +798,42 @@ int write_region_rule(cJSON* region_json, int compile_id, int group_id, iris_des int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void * logger) { - int compile_id=-1,cmd_cnt=0,ret=-1; + int compile_id=-1,cmd_cnt=0,ret=-1, clause_num=0, group_num=0, i=0, nth_clause=0; cJSON* item=NULL; struct iris_table_t* table_info=NULL; - cJSON* g_rules=cJSON_GetObjectItem(compile, "groups"); - int group_cnt=cJSON_GetArraySize(g_rules); - cJSON_AddNumberToObject(compile, "group_num", group_cnt); + cJSON* group_array=NULL, *group_obj=NULL; + int* clause_ids=NULL; + group_array=cJSON_GetObjectItem(compile, "groups"); + group_num=cJSON_GetArraySize(group_array); + clause_ids=ALLOC(int, group_num); + cJSON_ArrayForEach(group_obj, group_array); + { + item=cJSON_GetObjectItem(compile,"nth_clause"); + if(item) + { + nth_clause=item->valueint; + for(i=0; ivalueint; return compile_id; } -int write_group_line(int group_id, int parent_id, int group_not_flag, int parent_type, const char* virtual_table, struct iris_description_t *p_iris, void * logger) +int write_group2compile_line(int group_id, int compile_id, int group_not_flag, int Nth_clause, const char* virtual_table, struct iris_description_t *p_iris, void * logger) { char buff[1024*4]; - struct iris_table_t* table=p_iris->group_table; - snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%d\t%s\n", group_id, parent_id, group_not_flag, parent_type, virtual_table); + struct iris_table_t* table=p_iris->group2compile_table; + snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_id, compile_id, group_not_flag, virtual_table, Nth_clause); table->write_pos+=memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); table->line_count++; return 0; } +int write_group2group_line(int group_id, int superior_gorup_id, struct iris_description_t *p_iris, void * logger) +{ + char buff[1024*4]; + struct iris_table_t* table=p_iris->group2group_table; + snprintf(buff, sizeof(buff), "%d\t%d\t1\n", group_id, superior_gorup_id); + table->write_pos+=memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); + table->line_count++; + return 0; +} + void table_idx_write_cb(const uchar * key, uint size, void * data, void * user) { struct iris_description_t *p_iris=(struct iris_description_t *)user; @@ -957,11 +1000,12 @@ static struct group_info_t* group_info_add_unsafe(struct iris_description_t* p_i } return group_info; } -int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger) + +int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, int Nth_group, struct iris_description_t *p_iris, void* logger) { const char* _str_parent_type[2]={"compile", "group"}; - int ret=0; - int group_not_flag=0; + int ret=0, i=0; + int group_not_flag=0, Nth_clause=0; cJSON *region_json=NULL, *item=NULL; cJSON *sub_groups=NULL, *region_rule=NULL; const char* group_name=NULL, *virtual_table=NULL; @@ -976,26 +1020,37 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac { group_name=item->valuestring; } - - item=cJSON_GetObjectItem(group_json, "virtual_table"); - if(item==NULL||item->type!=cJSON_String) + if(parent_type==PARENT_TYPE_COMPILE) { - virtual_table="null"; + item=cJSON_GetObjectItem(group_json, "virtual_table"); + if(item==NULL||item->type!=cJSON_String) + { + virtual_table="null"; + } + else + { + virtual_table=item->valuestring; + } + item=cJSON_GetObjectItem(group_json,"not_flag"); + if(item==NULL||item->type!=cJSON_Number) + { + group_not_flag=0; + } + else + { + group_not_flag=item->valueint; + } + item=cJSON_GetObjectItem(group_json,"Nth_clause"); + if(item==NULL||item->type!=cJSON_Number) + { + Nth_clause=Nth_group; + } + else + { + Nth_clause=item->valueint; + } } else - { - virtual_table=item->valuestring; - } - item=cJSON_GetObjectItem(group_json,"not_flag"); - if(item==NULL||item->type!=cJSON_Number) - { - group_not_flag=0; - } - else - { - group_not_flag=item->valueint; - } - if(parent_type==PARENT_TYPE_GROUP) { group_not_flag=0; } @@ -1021,14 +1076,16 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac if(sub_groups!=NULL) { //recursively - + i=0; cJSON_ArrayForEach(item, sub_groups) { - ret=write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, p_iris, logger); + i++; + ret=write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, i, p_iris, logger); if(ret<0) { return -1; } + } } if(region_json==NULL && sub_groups==NULL) @@ -1037,7 +1094,14 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac "A group of compile rule %d has neither regions, sub groups, nor refered another exisited group.", tracking_compile_id); } } - ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, virtual_table, p_iris, logger); + if(parent_type==PARENT_TYPE_COMPILE) + { + ret=write_group2compile_line(group_info->group_id, parent_id, group_not_flag, Nth_clause, virtual_table, p_iris, logger); + } + else + { + ret=write_group2group_line(group_info->group_id, parent_id, p_iris, logger); + } if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, @@ -1052,25 +1116,27 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) int i=0; int compile_id=-1, compile_cnt=0, group_cnt=0; int ret=0; - cJSON *c_rules=NULL, *g_rules=NULL, *plug_tables=NULL; - cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL, *item=NULL; + cJSON *compile_array=NULL, *group_array=NULL, *plug_tables=NULL; + cJSON *compile_obj=NULL, *group_obj=NULL, *each_plug_table=NULL, *item=NULL; static struct group_info_t* parent_group=NULL; const char* parent_group_name=NULL; plug_tables=cJSON_GetObjectItem(json,"plugin_table"); if(NULL!=plug_tables) { + i=0; cJSON_ArrayForEach(each_plug_table, plug_tables) { write_plugin_line(each_plug_table, i, p_iris, logger); + i++; } } - g_rules=cJSON_GetObjectItem(json, "groups");//sub-group to group - if(g_rules!=NULL) + group_array=cJSON_GetObjectItem(json, "groups");//sub-group to group + if(group_array!=NULL) { - cJSON_ArrayForEach(group_rule, g_rules) + cJSON_ArrayForEach(group_obj, group_array) { - item=cJSON_GetObjectItem(group_rule, "parent_group"); + item=cJSON_GetObjectItem(group_obj, "parent_group"); if(item==NULL || item->type!=cJSON_String) { parent_group_name=untitled_group_name; @@ -1085,7 +1151,7 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) { parent_group=group_info_add_unsafe(p_iris, p_iris->group_name_map, item->string); } - ret=write_group_rule(group_rule, parent_group->group_id, PARENT_TYPE_GROUP, 0, p_iris, logger); + ret=write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, 0, 0, p_iris, logger); if(ret<0) { return -1; @@ -1093,47 +1159,48 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) } } - c_rules=cJSON_GetObjectItem(json,"rules"); - if(c_rules==NULL) + compile_array=cJSON_GetObjectItem(json,"rules"); + if(compile_array==NULL) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, "have no rules."); return -1; } - compile_cnt=cJSON_GetArraySize(c_rules); + compile_cnt=cJSON_GetArraySize(compile_array); if(compile_cnt<=0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, "have no rules."); return -1; } - cJSON_ArrayForEach(compile_rule, c_rules) + cJSON_ArrayForEach(compile_obj, compile_array) { - compile_id=write_compile_line(compile_rule,p_iris, logger); + compile_id=write_compile_line(compile_obj,p_iris, logger); if(compile_id<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, "In %d compile rule.",i); return -1; } - g_rules=cJSON_GetObjectItem(compile_rule,"groups"); - if(g_rules==NULL) + group_array=cJSON_GetObjectItem(compile_obj, "groups"); + if(group_array==NULL) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, "compile rule %d have no group.",compile_id); return -1; } - group_cnt=cJSON_GetArraySize(g_rules); + group_cnt=cJSON_GetArraySize(group_array); if(group_cnt<=0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, "compile rule %d have no groups.",compile_id); return -1; } - cJSON_ArrayForEach(group_rule, g_rules) + i=1; + cJSON_ArrayForEach(group_obj, group_array) { - ret=write_group_rule(group_rule, compile_id, PARENT_TYPE_COMPILE, compile_id, p_iris, logger); + ret=write_group_rule(group_obj, PARENT_TYPE_COMPILE, compile_id, compile_id, i, p_iris, logger); if(ret<0) { return -1; @@ -1148,7 +1215,7 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) return 0; } // redis_write_ctx is used by maat_redis_tool to write json to redis. -int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger) +int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger) { cJSON *json=NULL, *tmp_obj=NULL; int ret=-1; @@ -1166,12 +1233,17 @@ int json2iris(const char* json_buff, const char* json_filename, const char*compi compile_tn=tmp_obj->valuestring; } - tmp_obj=cJSON_GetObjectItem(json, "group_table"); + tmp_obj=cJSON_GetObjectItem(json, "group2compile_table"); if(tmp_obj) { - group_tn=tmp_obj->valuestring; + group2compile_tn=tmp_obj->valuestring; } - ret=set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, compile_tn, group_tn, redis_write_ctx, &iris_cfg, logger); + tmp_obj=cJSON_GetObjectItem(json, "group2group_table"); + if(tmp_obj) + { + group2group_tn=tmp_obj->valuestring; + } + ret=set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, compile_tn, group2compile_tn, group2group_tn, redis_write_ctx, &iris_cfg, logger); if(ret<0) { goto error_out; diff --git a/src/inc_internal/Maat_garbage_collection.h b/src/inc_internal/Maat_garbage_collection.h index f9538f5..e1d9681 100644 --- a/src/inc_internal/Maat_garbage_collection.h +++ b/src/inc_internal/Maat_garbage_collection.h @@ -1,8 +1,8 @@ #pragma once struct Maat_garbage_bin{}; -struct Maat_garbage_bin* Maat_garbage_bin_new(void); +struct Maat_garbage_bin* Maat_garbage_bin_new(int default_timeout); void Maat_garbage_bin_free(struct Maat_garbage_bin* bin); -void Maat_garbage_bag(struct Maat_garbage_bin* bin, void* garbage, int timeout, void (* func)(void *)); +void Maat_garbage_bagging(struct Maat_garbage_bin* bin, void* garbage, void (* func)(void *)); void Maat_garbage_collect_routine(struct Maat_garbage_bin* bin); diff --git a/src/inc_internal/Maat_hierarchy.h b/src/inc_internal/Maat_hierarchy.h index e7dc26a..b3ee5f3 100644 --- a/src/inc_internal/Maat_hierarchy.h +++ b/src/inc_internal/Maat_hierarchy.h @@ -1,6 +1,52 @@ +#pragma once + +#include "Maat_rule.h" +#include struct Maat_hierarchy; +struct Maat_hierarchy* Maat_hierarchy_new(int thread_num, void* mesa_handle_logger); +void Maat_hierarchy_free(struct Maat_hierarchy* hier); +void Maat_hierarchy_set_compile_user_data_free_func(struct Maat_hierarchy* hier, void (* func)(void *)); +void Maat_hierarchy_set_region_user_data_free_func(struct Maat_hierarchy* hier, void (* func)(void *)); + + + struct Maat_hierarchy_compile_mid; +struct Maat_hierarchy_compile_mid* Maat_hierarchy_compile_mid_new(struct Maat_hierarchy* hier, int thread_num); +void Maat_hierarchy_compile_mid_free(struct Maat_hierarchy_compile_mid* mid); +void Maat_hierarchy_compile_mid_udpate(struct Maat_hierarchy_compile_mid* mid, int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result); +void Maat_hierarchy_compile_user_data_iterate(struct Maat_hierarchy* hier, void (*callback)(void *user_data, void* apram), void* param); + +int Maat_hierarchy_compile_mid_has_NOT_clause(struct Maat_hierarchy_compile_mid* mid); + +struct Maat_hierarchy_hit_path +{ + int Nth_hit_region; + struct Maat_hit_path_t path; + TAILQ_ENTRY(Maat_hierarchy_hit_path) entries; +}; +void Maat_hit_path_init(struct Maat_hit_path_t* hit_path); +size_t Maat_hierarchy_hit_path_select0(const struct Maat_hierarchy_compile_mid* mid, struct Maat_hit_path_t* condition, + struct Maat_hierarchy_hit_path** hit_paths, size_t n_path); + +int Maat_hierarchy_compile_add(struct Maat_hierarchy* hier, int compile_id, int declared_clause_num, void* user_data); +int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id); + +void* Maat_hierarchy_compile_dettach_user_data(struct Maat_hierarchy* hier, int compile_id); +void* Maat_hierarchy_compile_read_user_data(struct Maat_hierarchy* hier, int compile_id); + +void* Maat_hierarchy_region_dettach_user_data(struct Maat_hierarchy* hier, int region_id); +int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int Nth_clause, int compile_id); +int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int Nth_clause, int compile_id); +int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id); +int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id); + +int Maat_hierarchy_add_region_to_group(struct Maat_hierarchy* hier, int group_id, int region_id, int table_id, void* user_data); +int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int group_id, int region_id); + +int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier); +int Maat_hierarchy_region_compile(struct Maat_hierarchy_compile_mid* mid, int is_last_compile, void** user_data_array, size_t ud_array_sz); + diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 4a3b0c7..1b1dd57 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -11,10 +11,8 @@ #include #include #include "dynamic_array.h" -#include "bool_matcher.h" #include "hiredis.h" #include "IPMatcher.h" -#include "igraph/igraph.h" #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" #include "alignment_int64.h" @@ -139,13 +137,6 @@ struct op_expr_t int rule_type; }; -struct region_group_relation -{ - int region_id; - int group_id; - int array_idx; -}; - struct Maat_region_inner { int region_id; @@ -159,51 +150,6 @@ struct Maat_region_inner }; -struct Maat_group_inner -{ - int group_id; - int table_id; - int region_boundary; - int region_cnt; - int ref_by_parent_cnt; - int ref_by_children_cnt; - char* group_name; - int ref_by_compile_cnt; - int vertex_id; - size_t top_group_cnt; - long long* top_groups; - dynamic_array_t *regions; - pthread_mutex_t mutex; -}; - - - - -struct _compile_result_t -{ - int compile_id; - struct bool_expr group_set; -}; -struct Maat_hit_path_inner -{ - int Nth_hit_region; - struct Maat_hit_path_t path; - TAILQ_ENTRY(Maat_hit_path_inner) entries; -}; - -TAILQ_HEAD(hit_path_q, Maat_hit_path_inner); - -struct scan_hit_status -{ - size_t hit_path_cnt; - struct hit_path_q hit_path_qhead; - - size_t all_hit_group_cnt; - size_t all_hit_group_array_sz; - unsigned long long *all_hit_group_array; - - char not_grp_compile_hitted_flag; -}; struct _OUTER_scan_status_t { struct _Maat_feather_t* feather; @@ -213,18 +159,8 @@ struct _OUTER_scan_status_t int district_id; int scan_cnt; struct Maat_hierarchy_compile_mid* compile_mid; - struct scan_hit_status* inner; -}; -enum maat_garbage_type -{ - GARBAGE_SCANNER=0, - GARBAGE_GROUP_RULE, - GARBAGE_COMPILE_RULE, - GARBAGE_COMPILE_INNER, - GARBAGE_BOOL_MATCHER, - GARBAGE_MAP_STR2INT, - GARBAGE_IP_MATCHER }; + struct iconv_handle_t { int is_initialized; @@ -356,6 +292,8 @@ struct _Maat_feather_t char instance_name[MAX_TABLE_NAME_LEN]; char table_info_fn[MAX_TABLE_NAME_LEN]; char compile_tn[MAX_TABLE_NAME_LEN]; + char group2compile_tn[MAX_TABLE_NAME_LEN]; + char group2group_tn[MAX_TABLE_NAME_LEN]; char group_tn[MAX_TABLE_NAME_LEN]; pthread_mutex_t background_update_mutex; char decrypt_key[MAX_TABLE_NAME_LEN]; @@ -386,7 +324,7 @@ struct _Maat_feather_t int fs_status_id[MAX_MAAT_STAT_NUM]; int fs_column_id[MAX_MAAT_STAT_NUM]; mcore_long_t outer_mid_cnt; - mcore_long_t inner_mid_cnt; + mcore_long_t compile_mid_cnt; mcore_long_t hit_cnt; mcore_long_t thread_call_cnt;//size indicate by scan_thread_num, mcore_long_t not_grp_hit_cnt; @@ -401,25 +339,7 @@ struct _Maat_feather_t long long cmd_acc_num; long long line_cmd_acc_num; }; -struct _maat_garbage_t -{ - enum maat_garbage_type type; - time_t create_time; - int expire_after; - int ok_times; - union - { - struct Maat_scanner* scanner; - struct Maat_group_inner* group_rule; - struct Maat_compile_rule* compile_rule; - struct Maat_compile_inner * compile_group_relation; - struct bool_matcher* bool_matcher; - struct ip_matcher* a_ip_matcher; - void * raw; - MESA_htable_handle str2int_map; - char* filename; - }; -}; + struct foreign_key { int column; @@ -439,10 +359,7 @@ struct serial_rule_t //rm= Redis Maat struct foreign_key* f_keys; }; int parse_accept_tag(const char* value, struct rule_tag** result, void* logger); -void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); -void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); -void garbage_bury(MESA_lqueue_head garbage_q,void *logger); -void make_group_set(struct Maat_compile_inner* compile_rule, struct bool_expr* a_set, unsigned char *has_not); + void maat_start_cb(long long new_version,int update_type,void*u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para); void maat_finish_cb(void* u_para); diff --git a/src/inc_internal/Maat_table.h b/src/inc_internal/Maat_table.h index 2daaedf..d794f52 100644 --- a/src/inc_internal/Maat_table.h +++ b/src/inc_internal/Maat_table.h @@ -154,7 +154,9 @@ int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, Maat_finish_callback_t *finish,//u_para void* u_para); int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); -int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); +int Maat_table_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); +int Maat_table_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); + const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id); enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id); diff --git a/src/inc_internal/json2iris.h b/src/inc_internal/json2iris.h index 247a191..4c00193 100644 --- a/src/inc_internal/json2iris.h +++ b/src/inc_internal/json2iris.h @@ -1,6 +1,6 @@ #ifndef H_MAAT_JSON2IRIS_H_INCLUDE #define H_MAAT_JSON2IRIS_H_INCLUDE -int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger); +int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger); int set_file_rulenum(const char* path, int rulenum, void* logger); #endif diff --git a/src/version.map b/src/version.map index a17b821..4f4c55d 100644 --- a/src/version.map +++ b/src/version.map @@ -1,4 +1,4 @@ -VERS_2.4{ +VERS_3.0{ global: extern "C++" { *MAAT_FRAME_VERSION_*; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index db47175..a9c9813 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,5 +1,5 @@ -add_executable(maat_demo maat_demo.cpp) -target_link_libraries(maat_demo maat_frame_shared) +#add_executable(maat_demo maat_demo.cpp) +#target_link_libraries(maat_demo maat_frame_shared) add_executable(test_igraph test_igraph.cpp) target_link_libraries(test_igraph igraph-static) @@ -7,8 +7,8 @@ target_link_libraries(test_igraph igraph-static) add_executable(test_maatframe test_maatframe.cpp) target_link_libraries(test_maatframe maat_frame_shared gtest) -add_executable(perf_test_maatframe perf_test_maatframe.cpp) -target_link_libraries(perf_test_maatframe maat_frame_shared gtest) +#add_executable(perf_test_maatframe perf_test_maatframe.cpp) +#target_link_libraries(perf_test_maatframe maat_frame_shared gtest) configure_file(table_info.conf table_info.conf COPYONLY) configure_file(t2_tableinfo.conf t2_tableinfo.conf COPYONLY) diff --git a/test/maat_demo.cpp b/test/maat_demo.cpp index c4ffd4f..40aed8c 100644 --- a/test/maat_demo.cpp +++ b/test/maat_demo.cpp @@ -678,8 +678,8 @@ int test_table_conjunction(Maat_feather_t feather,const char* table_name,const c #define TEST_CMD_LINE_NUM 200*1000 void test_set_cmd_line(Maat_feather_t feather) { - struct Maat_line_t **p_line=(struct Maat_line_t **)calloc(sizeof(struct Maat_line_t *), TEST_CMD_LINE_NUM); - struct Maat_line_t *line_rule=(struct Maat_line_t *)calloc(sizeof(struct Maat_line_t), TEST_CMD_LINE_NUM); + struct Maat_cmd_line **p_line=(struct Maat_cmd_line **)calloc(sizeof(struct Maat_cmd_line *), TEST_CMD_LINE_NUM); + struct Maat_cmd_line *line_rule=(struct Maat_cmd_line *)calloc(sizeof(struct Maat_cmd_line), TEST_CMD_LINE_NUM); int i=0; const char* line="1\t192.168.0.1\t4444444444\t1"; memset(&line_rule,0,sizeof(line_rule)); @@ -695,14 +695,14 @@ void test_set_cmd_line(Maat_feather_t feather) } - Maat_cmd_set_lines(feather,(const struct Maat_line_t **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD); + Maat_cmd_set_lines(feather,(const struct Maat_cmd_line **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD); usleep(WAIT_FOR_EFFECTIVE_US); for(i=0;i