From 0cf4b2bb2429a0737a25a0ad5ee97c7f30b9ed80 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Fri, 5 Jun 2020 10:21:22 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8A=BD=E8=B1=A1=E5=9E=83=E5=9C=BE=E5=9B=9E?= =?UTF-8?q?=E6=94=B6=E6=A8=A1=E5=9D=97=EF=BC=8C=E6=96=B0=E5=A2=9EMaat=5Fga?= =?UTF-8?q?rbage=5Fcollection.cpp/h?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 1 + src/entry/Maat_hierarchy.cpp | 662 +++++++++++-------- src/inc_internal/Maat_garbage_collection.cpp | 73 ++ src/inc_internal/Maat_garbage_collection.h | 8 + 4 files changed, 486 insertions(+), 258 deletions(-) create mode 100644 src/inc_internal/Maat_garbage_collection.cpp create mode 100644 src/inc_internal/Maat_garbage_collection.h diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index dbe1916..32c2f95 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -513,6 +513,7 @@ int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,i if(region_id>0&&district_id!=_mid->district_id) { ret_region_num--; + //todo: replace memmove with high performance solution. memmove(&(region_hit[i]),&(region_hit[i+1]),sizeof(scan_result_t)*(ret_region_num-i)); } else diff --git a/src/entry/Maat_hierarchy.cpp b/src/entry/Maat_hierarchy.cpp index 597a618..b58d1f6 100644 --- a/src/entry/Maat_hierarchy.cpp +++ b/src/entry/Maat_hierarchy.cpp @@ -1,12 +1,6 @@ - -struct Maat_group_internal -{ - int group_id; - - int ref_by_compile_cnt; -}; +#define module_maat_hierarchy "MAAT_HIERARCHY" struct Maat_CNF_literal { @@ -17,15 +11,16 @@ struct Maat_CNF_clause { int compile_id; int Nth_clause; + long long clause_id; char not_flag; TAILQ_ENTRY(Maat_CNF_clause) entries; }; -TAILQ_HEAD(Maat_clause_id_q, Maat_CNF_clause); +TAILQ_HEAD(Maat_clause_q, Maat_CNF_clause); struct Maat_clause_list { size_t clause_id_count; - Maat_clause_id_q clause_id_q; + Maat_clause_q clause_q; }; @@ -33,6 +28,7 @@ struct _CNF_clause { char not_flag; char in_use; + long long clause_id; //Following varibles are used when no Nth clause is given, another word, this is forward compatible. int vt_id; int group_id; @@ -40,53 +36,38 @@ struct _CNF_clause struct Maat_CNF { int compile_id; - int clause_cnt; + int declared_clause_number; struct _CNF_clause clauses[MAX_ITEMS_PER_BOOL_EXPR]; + void* user_tag; }; -enum MAAT_HIERARCHY_LEVEL -{ - HIERARCHY_LVL_GROUP, - HIERARCHY_LVL_LITERAL, - HIERARCHY_LVL_CLAUSE -}; -struct Maat_hierarchy_element -{ - int vertex_id; - enum MAAT_HIERARCHY_LEVEL level; - union - { - struct Maat_group_internal group; - struct Maat_CNF_literal literal; - struct Maat_CNF_clause clause; - }; - int ref_by_superior_cnt; - int ref_by_subordinate_cnt; -}; + struct Maat_hierarchy { - igraph_t hierarchy_graph; - igraph_t cnf_graph;//? - igraph_integer_t vcount; + struct bool_matcher* bm; + MESA_htable_handle literal2clause_list_hash; //key: virtual_table<<32|group_id, aka literal_id, value: struct Maat_clause_list* + MESA_htable_handle cnf_hash; //key: compile_id, value: struct Maat_CNF * + MESA_htable_handle group_hash; //key: group_id, value: struct Maat_group_vertex* + MESA_htable_handle vertex_id2group_hash; //key:vetex_id, value: struct Maat_group_vertex* (reference of group hash, do NOT free) + + igraph_t group_graph; + igraph_integer_t group_graph_vcount; igraph_vector_t dfs_vids; - int vertex_id_generator; - int global_clause_id_generator; - MESA_htable_handle vertex_id2ele;//key:vertex_id, value: struct Maat_hierarchy_element* - - MESA_htable_handle grp_and_vt2clause_id_hash; //key: virtual_table<<32|group_id, value: struct Maat_clause_list* - struct bool_matcher * bool_matcher_expr_compiler; - - MESA_htable_handle cnf_hash; //key: compile_id, value: struct Maat_CNF * - MESA_htable_handle clause_hash; //key: compile_id+Nth_clause, value: global_clause_id - MESA_htable_handle literal_hash; - MESA_htable_handle group_hash; // + igraph_integer_t grp_vertex_id_generator; + int thread_num; + struct Maat_garbage_bin* garbage_bin; + void* logger; }; -struct Maat_hierarchy* Maat_hierarchy_new() +struct Maat_hierarchy* Maat_hierarchy_new(int thread_num, struct Maat_garbage_bin* garbage_bin, void* mesa_handle_logger) { struct Maat_hierarchy* hier=ALLOC(struct Maat_hierarchy, 1); int ret=0; + hier->logger=mesa_handle_logger; + hier->garbage_bin=garbage_bin; + hier->thread_num=thread_num; + MESA_htable_create_args_t hargs; memset(&hargs,0,sizeof(hargs)); hargs.thread_safe=0; @@ -103,299 +84,464 @@ struct Maat_hierarchy* Maat_hierarchy_new() hargs.data_expire_with_condition = NULL; + + hargs.thread_safe=0; + hargs.data_free = free; + hier->literal2clause_list_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(hier->literal2clause_list_hash, 0); + hargs.thread_safe=0; hargs.data_free = free; hier->cnf_hash=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(hier->cnf_hash, 0); hargs.thread_safe=0; - hargs.data_free = free; - hier->clause_hash=MESA_htable_create(&hargs, sizeof(hargs)); - MESA_htable_print_crtl(hier->clause_hash, 0); - - - hargs.thread_safe=0; - hargs.data_free = free; - hier->literal_hash=MESA_htable_create(&hargs, sizeof(hargs)); - MESA_htable_print_crtl(hier->literal_hash, 0); - - - hargs.thread_safe=0; - hargs.data_free = free; + hargs.data_free = EMPTY_FREE; hier->group_hash=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(hier->group_hash,0); hargs.thread_safe=0; - hargs.data_free = EMPTY_FREE; //data stored in cnf_hash, group_hash, clause_hash - hier->vertex_id2ele=MESA_htable_create(&hargs, sizeof(hargs)); - MESA_htable_print_crtl(hier->vertex_id2ele, 0); + hargs.data_free = EMPTY_FREE; //data stored in group_hash, no need free. + hier->vertex_id2group_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(hier->vertex_id2group_hash, 0); - ret=igraph_empty(&hier->hierarchy_graph, 0, IGRAPH_DIRECTED); + ret=igraph_empty(&hier->group_graph, 0, IGRAPH_DIRECTED); assert(ret==IGRAPH_SUCCESS); return hier; } void Maat_hierarchy_free(struct Maat_hierarchy* hier) { + MESA_htable_destroy(hier->literal2clause_list_hash, NULL); MESA_htable_destroy(hier->cnf_hash, NULL); - MESA_htable_destroy(hier->clause_hash, NULL); - MESA_htable_destroy(hier->literal_hash, NULL); MESA_htable_destroy(hier->group_hash, NULL); + MESA_htable_destroy(hier->vertex_id2group_hash, NULL); - MESA_htable_destroy(hier->vertex_id2ele, NULL); - - igraph_destroy(&hier->hierarchy_graph); + igraph_destroy(&hier->group_graph); free(hier); } -struct Maat_hierarchy_element* Maat_hierarchy_element_new(struct Maat_hierarchy* hier, enum MAAT_HIERARCHY_LEVEL lvl) -{ - int ret=0; - struct Maat_hierarchy_element* ele=NULL; - ele=ALLOC(struct Maat_hierarchy_element, 1); - ele->level=lvl; - ele->vertex_id=hier->vertex_id_generator++; - assert(igraph_vcount(&hier->hierarchy_graph)==ele->vertex_id); - igraph_add_vertices(&hier->hierarchy_graph, 1, NULL); //Add 1 vertice. - ret=HASH_add_by_id(hier->vertex_id2ele, ele->vertex_id, ele); - assert(ret>0); - return ele; -} -void Maat_hierarchy_element_free(struct Maat_hierarchy_element* ele) -{ - -} #define TO_CLAUSE_ID(Nth_clause, compile_id) ((long long)Nth_clause<<32|compile_id) -static long long get_Nth_clause(struct Maat_CNF* cnf, int group_id, int vt_id, int not_flag) -{ - int i=0, Nth_clause=0; - if(Nth_clause==0)//compatible - { - for(i=0; iclauses.group_id==group_id && - cnf->clauses.vt_id==vt_id && - cnf->clauses.not_flag==not_flag) - { - Nth_clause=i+1; - break; - } - } - if(Nth_clause<1) - { - Nth_clause=cnf->clause_cnt; - cnf->clauses[Nth_clause].group_id=group_id; - cnf->clauses[Nth_clause].vt_id=vt_id; - cnf->clause_cnt++; - } - assert(cnf->clause_cntcnf_hash, compile_id); - if(!cnf) - { - cnf=ALLOC(struct Maat_CNF, 1); - cnf->compile_id=compile_id; - } - if(Nth_clause==0)//for compatible - { - Nth_clause=get_Nth_clause(cnf, group_id, vt_id, not_flag); - } - cnf->clauses[Nth_clause-1].in_use=1; - cnf->clauses[Nth_clause-1].not_flag=not_flag; - - ele_group=(struct Maat_hierarchy_element*)HASH_fetch_by_id(hier->group_hash, group_id); - if(!ele_group) - { - ele_group=Maat_hierarchy_element_new(hier, HIERARCHY_LVL_GROUP); - ele_group->group.group_id=group_id; - ret=HASH_add_by_id(hier->group_hash, group_id, ele_group); - assert(ret>0); - } - ele_literal=(struct Maat_hierarchy_element*)MESA_htable_search(hier->literal_hash, TO_LITERAL_ID(vt_id, group_id), sizeof(long long)); - if(!ele_literal) - { - ele_literal=Maat_hierarchy_element_new(hier, HIERARCHY_LVL_LITERAL); - ele_literal->literal.group_id=group_id; - ele_literal->literal.virtual_table_id=vt_id; - MESA_htable_add(hier->literal_hash, TO_LITERAL_ID(vt_id, group_id), sizeof(long long), ele_literal); - ret=igraph_get_eid(&hier->hierarchy_graph, &edge_id, ele_group->vertex_id, ele_literal->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); - assert(edge_id==0);//For new literal, the edge should not exisited. - igraph_add_edge(&hier->hierarchy_graph, ele_group->vertex_id, ele_literal->vertex_id); - ele_group->group.ref_by_compile_cnt++; - ele_group->ref_by_superior_cnt++; - ele_literal->ref_by_subordinate_cnt++; - } - else - { - ret=igraph_get_eid(&hier->hierarchy_graph, &edge_id, ele_group->vertex_id, ele_literal->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); - assert(edge_id>0);//For existed literal, the edge should exisited. - } - ele_clause=(struct Maat_hierarchy_element*)MESA_htable_search(hier->clause_hash, TO_CLAUSE_ID(Nth_clause, compile_id), sizeof(long long)); - if(!ele_clause) - { - ele_clause=Maat_hierarchy_element_new(hier, HIERARCHY_LVL_CLAUSE); - ele_clause->clause.compile_id=compile_id; - ele_clause->clause.Nth_clause=Nth_clause; - ele_clause->clause.not_flag=not_flag; - MESA_htable_add(hier->clause_hash, TO_CLAUSE_ID(Nth_clause, compile_id), sizeof(long long), ele_clause); - } - if(ele_clause->clause.not_flag!=not_flag) - { - return -1; - } - - ret=igraph_get_eid(&hier->hierarchy_graph, &edge_id, ele_literal->vertex_id, ele_clause->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); - if(edge_id>0)//The edge was found. Only one edge between a literal and a clause. - { - return -1; - } - //igraph allow add multiple edges between two vetex, igraph_delete_edges removes one edge per call. - igraph_add_edge(&hier->hierarchy_graph, ele_literal->vertex_id, ele_clause->vertex_id); - ele_literal->ref_by_superior_cnt++; - ele_clause->ref_by_subordinate_cnt++; +int Maat_hierarchy_add_compile(struct Maat_hierarchy* hier, int compile_id, int clause_num, void* user) +{ + struct Maat_CNF* cnf=NULL; + cnf=MESA_htable_search(hier->cnf_hash, &compile_id, sizeof(compile_id)); + if(cnf)//duplicated + { + return -1; + } + cnf=ALLOC(struct Maat_CNF, 1); + cnf->compile_id=compile_id; + cnf->declared_clause_number=clause_num; + cnf->user_tag=user; + MESA_htable_add(hier->cnf_hash, &compile_id, sizeof(compile_id), cnf); return 0; } +int Maat_hierarchy_remove_compile(struct Maat_hierarchy * hier, int compile_id) +{ + struct Maat_CNF* cnf=NULL; + cnf=MESA_htable_search(hier->cnf_hash, &compile_id, sizeof(compile_id)); + if(!cnf) + { + return -1; + } + cnf=MESA_htable_del(hier->cnf_hash, &compile_id, sizeof(compile_id), NULL); + return 0; +} +int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int Nth_clause, int compile_id) +{ + struct Maat_clause_list* clause_list=NULL; + struct Maat_CNF_clause* p=NULL; + struct Maat_group_vertex* group=NULL; + group=(struct Maat_group_vertex*)MESA_htable_search(hier->group_hash, &group_id, sizeof(group_id)); + if(!group) + { + group=Maat_hierarchy_group_vertex_new(hier, group_id); + } + group->ref_by_compile_cnt++; + clause_list=(struct Maat_clause_list*)MESA_htable_search(hier->literal2clause_list_hash, TO_LITERAL_ID(vt_id, group_id), sizeof(long long)); + if(!clause_list) + { + clause_list=ALLOC(struct Maat_clause_list, 1); + TAILQ_INIT(&clause_list->clause_q); + MESA_htable_add(hier->literal2clause_list_hash, TO_LITERAL_ID(vt_id, group_id), sizeof(long long), clause_list); + } + TAILQ_FOREACH(p, &clause_list->clause_q, entries) + { + if(p->compile_id==compile_id && p->not_flag==not_flag && p->Nth_clause==Nth_clause) + { + return -1; //duplicated + } + } + p=ALLOC(struct Maat_CNF_clause, 1); + p->compile_id=compile_id; + p->not_flag=not_flag; + p->Nth_clause=Nth_clause; + p->clause_id=TO_CLAUSE_ID(Nth_clause, compile_id); + + TAILQ_INSERT_TAIL(&clause_list->clause_q, p, entries); + clause_list->clause_id_count++; + return 0; +} +int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int Nth_clause, int compile_id) +{ + struct Maat_clause_list* clause_list=NULL; + struct Maat_CNF_clause* p=NULL, *tmp=NULL; + int found=0; + clause_list=(struct Maat_clause_list*)MESA_htable_search(hier->literal2clause_list_hash, TO_LITERAL_ID(vt_id, group_id), sizeof(long long)); + if(!clause_list) + { + return -1; + } + for(p=TAILQ_FIRST(&clause_list->clause_q); p!=NULL; p=tmp) + { + tmp=TAILQ_NEXT(p, entries); + if(p->compile_id==compile_id && p->not_flag==not_flag && p->Nth_clause==Nth_clause) + { + TAILQ_REMOVE(clause_list->clause_q, p, entries); + clause_list->clause_id_count--; + free(p); + return 0; + } + + } + return -1; +} +struct Maat_group_vertex +{ + igraph_integer_t vertex_id; + int group_id; + int ref_by_compile_cnt; + int ref_by_group_cnt; + + pthread_mutex_t mutex; + int top_group_cnt; + int* top_groups; +}; +struct Maat_group_vertex* Maat_hierarchy_group_vertex_new(struct Maat_hierarchy* hier, int group_id) +{ + int ret=0; + struct Maat_group_vertex* group=NULL; + group=ALLOC(struct Maat_group_vertex, 1); + group->group_id=group_id; + group->vertex_id=hier->grp_vertex_id_generator++; + assert(igraph_vcount(&hier->group_graph)==group->vertex_id); + igraph_add_vertices(&hier->group_graph, 1, NULL); //Add 1 vertice. + pthread_mutex_init(&(group->mutex), NULL); + + ret=MESA_htable_add(hier->vertex_id2group_hash, &group->vertex_id, sizeof(group->vertex_id), group); + assert(ret>0); + ret=MESA_htable_add(hier->group_hash, &group_id, sizeof(group_id), group); + assert(ret>0); + return group; +} +static void group_vertex_free(struct Maat_group_vertex* group) +{ + free(group->top_groups); + free(group); +} +void Maat_hierarchy_group_vertex_free(struct Maat_hierarchy* hier, int group_id) +{ + struct Maat_group_vertex* group=NULL, superior_group=NULL; + group=(struct Maat_group_vertex*)MESA_htable_search(hier->group_graph, &group_id, sizeof(group)); + assert(group!=NULL); + int ret=0; + igraph_vector_t v; + char buff[4096]; + assert(group->ref_by_compile_cnt==0&&group->ref_by_group_cnt==0); + igraph_vector_init(&v, 8); + igraph_neighbors(&hier->group_graph, &v, group->vertex_id, IGRAPH_ALL); + if(igraph_vector_size(&v)>0) + { + print_igraph_vector(&v, buff, sizeof(buff)); + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + "Del group %d exception, still reached by %s.", + group->vertex_id, buff); + assert(0); + } + igraph_vector_destroy(&v); + assert(group->top_groups==NULL); + igraph_delete_vertices(&hier->group_graph, igraph_vss_1(group->vertex_id)); + + ret=MESA_htable_del(hier->grp_vertex_id_generator, group->vertex_id, sizeof(group->vertex_id), NULL); + assert(ret==0); + ret=MESA_htable_del(hier->group_hash, &group_id, sizeof(group_id), NULL); //group is freed by MESA_htable + assert(ret==0); + Maat_garbage_bag(hier->garbage_bin, group, group_vertex_free); + return; +} + int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id) { int ret=0; igraph_integer_t edge_id; - struct Maat_hierarchy_element* ele_group=NULL, *ele_superior_group=NULL; - ele_group=(struct Maat_hierarchy_element*)HASH_fetch_by_id(hier->group_hash, group_id); - if(!ele_group) + struct Maat_group_vertex* group=NULL, superior_group=NULL; + group=(struct Maat_group_vertex*)MESA_htable_search(hier->group_graph, &group_id, sizeof(group_id)); + if(!group) { - ele_group=Maat_hierarchy_element_new(hier, HIERARCHY_LVL_GROUP); - ele_group->group.group_id=group_id; - ret=HASH_add_by_id(hier->group_hash, group_id, ele_group); - assert(ret>0); + group=Maat_hierarchy_group_vertex_new(hier, group_id); } - - ele_superior_group=(struct Maat_hierarchy_element*)HASH_fetch_by_id(hier->group_hash, superior_group_id); - if(!ele_superior_group) + superior_group=(struct Maat_group_vertex*)MESA_htable_search(hier->group_graph, &superior_group_id, sizeof(superior_group_id)); + if(!superior_group) { - ele_superior_group=Maat_hierarchy_element_new(hier, HIERARCHY_LVL_GROUP); - ele_superior_group->group.group_id=superior_group_id; - ret=HASH_add_by_id(hier->group_hash, superior_group_id, ele_superior_group); - assert(ret>0); + superior_group=Maat_hierarchy_group_vertex_new(hier, superior_group_id); } - - ret=igraph_get_eid(&hier->hierarchy_graph, &edge_id, ele_group->vertex_id, ele_superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); + ret=igraph_get_eid(&hier->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); if(edge_id>0)//The edge was found. Only one edge between two groups. { + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + "Add group %d to group %d failed, relation already exisited.", + group->group_id, superior_group->group_id); return -1; } - igraph_add_edge(&hier->hierarchy_graph, ele_group->vertex_id, ele_superior_group->vertex_id); - ele_group->ref_by_superior_cnt++; - ele_superior_group->ref_by_subordinate_cnt++; + igraph_add_edge(&hier->group_graph, group->vertex_id, superior_group->vertex_id); + group->ref_by_group_cnt++; return 0; } -int Maat_hierarchy_add_region(struct Maat_hierarchy* hier, int region_id, int group_id) + +int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id) { -} -static int hierarchy_graph_reomve_edge(igrahp_t* graph, int from_vertex_id, int to_vertex_id) -{ - igraph_es_t es; int ret=0; + igraph_integer_t edge_id; + struct Maat_group_vertex* group=NULL, superior_group=NULL; + group=(struct Maat_group_vertex*)MESA_htable_search(hier->group_graph, &group_id, sizeof(group_id)); + if(group==NULL) + { + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + "Del group %d from group %d failed, group %d not exisited.", + group_id, superior_group_id, group_id); + return -1; + } + superior_group=(struct Maat_group_vertex*)MESA_htable_search(hier->group_graph, &superior_group_id, sizeof(superior_group_id)); + if(superior_group==NULL) + { + MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, + "Del group %d from group %d failed, superior group %d not exisited.", + group_id, superior_group_id, superior_group_id); + return -1; + } + igraph_es_t es; igraph_integer_t edge_num_before=0, edge_num_after=0; - edge_num_before=igraph_ecount(graph); + edge_num_before=igraph_ecount(&hier->group_graph); // The edges between the given pairs of vertices will be included in the edge selection. //The vertex pairs must be given as the arguments of the function call, the third argument //is the first vertex of the first edge, the fourth argument is the second vertex of the //first edge, the fifth is the first vertex of the second edge and so on. The last element //of the argument list must be -1 to denote the end of the argument list. //https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small - ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, from_vertex_id, to_vertex_id, -1); + ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1); assert(ret==IGRAPH_SUCCESS); // ignore no such edge to abort(). igraph_set_error_handler(igraph_error_handler_ignore); - ret=igraph_delete_edges(graph, es); - edge_num_after=igraph_ecount(graph); + ret=igraph_delete_edges(&hier->group_graph, es); + edge_num_after=igraph_ecount(&hier->group_graph); igraph_es_destroy(&es); if(ret!=IGRAPH_SUCCESS||edge_num_before-edge_num_after!=1) { - + assert(0) return -1; } else { + group->ref_by_group_cnt--; return 0; } - } -int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int Nth_clause, int compile_id) + +static void reset_cnf_hash(const uchar * key, uint size, void * data, void * user) { - int ret=0; + struct Maat_CNF* cnf=(struct Maat_CNF*)data; + memset(cnf->clauses, 0, sizeof(cnf->clauses)); + (*(size_t*)user)++; + return; +} +struct clause_list_walker +{ + size_t n_literal; + size_t n_clause; + size_t n_not_flag; + MESA_htable_handle cnf_hash; +}; +static void walk_literal2clause_list_hash(const uchar * key, uint size, void * data, void * user) +{ + struct Maat_clause_list* clause_list=(struct Maat_clause_list*)data; + struct Maat_CNF_clause* p=NULL; + struct clause_list_walker* walker=(struct clause_list_walker*)user; struct Maat_CNF* cnf=NULL; - cnf=(struct Maat_CNF*)HASH_fetch_by_id(hier->cnf_hash, compile_id); - if(!cnf) + walker->n_literal++; + TAILQ_FOREACH(p, &clause_list->clause_q, entries) { - return -2; - } - if(Nth_clause==0) - { - Nth_clause=get_Nth_clause(cnf, group_id, vt_id, not_flag); - cnf->clauses[Nth_clause-1]. - } - struct Maat_hierarchy_element* ele_group=NULL, *ele_literal=NULL, *ele_clause=NULL; - ele_group=(struct Maat_hierarchy_element*)HASH_fetch_by_id(hier->group_hash, group_id); - ele_literal=(struct Maat_hierarchy_element*)MESA_htable_search(hier->literal_hash, TO_LITERAL_ID(vt_id, group_id), sizeof(long long)); - ele_clause=(struct Maat_hierarchy_element*)MESA_htable_search(hier->clause_hash, TO_CLAUSE_ID(Nth_clause, compile_id), sizeof(long long)); - if(!ele_group||!ele_literal||!ele_clause) - { - return -1; - } - ret=hierarchy_graph_reomve_edge(&hier->hierarchy_graph, ele_group->vertex_id, ele_literal->vertex_id); - if(ret<0) - { - return -1; - } - ele_group->group->ref_by_compile_cnt--; - ele_group->ref_by_subordinate_cnt--; - ele_literal->ref_by_superior_cnt--; + cnf=MESA_htable_search(walker->cnf_hash, &p->compile_id, sizeof(p->compile_id)); + if(cnf==NULL)//no compile declared. + { + continue; + } - ret=hierarchy_graph_reomve_edge(&hier->hierarchy_graph, ele_literal->vertex_id, ele_clause->vertex_id); - if(ret<0) - { - return -1; - } - ele_literal->ref_by_superior_cnt--; - ele_clause->ref_by_subordinate_cnt--; - return 0; + if(cnf->clauses[p->Nth_clause].in_use) + { + assert(cnf->clauses[p->Nth_clause].clause_id==p->clause_id); + assert(cnf->clauses[p->Nth_clause].not_flag==p->not_flag); + + } + else + { + walker->n_clause++; + cnf->clauses[p->Nth_clause].in_use=1; + cnf->clauses[p->Nth_clause].clause_id=p->clause_id; + cnf->clauses[p->Nth_clause].not_flag=p->not_flag; + if(cnf->clauses[p->Nth_clause].not_flag) + { + walker->n_not_flag++; + } + } + assert(cnf->compile_id==p->compile_id); + } + return; } -int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id) +struct cnf_walker { - int ret=0; - struct Maat_hierarchy_element* ele_group=NULL, *ele_superior_group=NULL; - ele_group=(struct Maat_hierarchy_element*)HASH_fetch_by_id(hier->group_hash, group_id); - ele_superior_group=(struct Maat_hierarchy_element*)HASH_fetch_by_id(hier->group_hash, superior_group_id); - if(!ele_group||!ele_superior_group) + size_t sz, cnt; + struct bool_expr* bool_expr_array; + +}; +static void walk_cnf_hash(const uchar * key, uint size, void * data, void * user) +{ + struct Maat_CNF* cnf=(struct Maat_CNF*)data; + struct cnf_walker* walker=(struct cnf_walker*)walker; + struct bool_expr* a_expr=walker->bool_expr_array[walker->cnt]; + int i=0, j=0; + for(i=0; iclauses.in_use) + { + a_expr->items[j].item_id=cnf->clauses[i].clause_id; + a_expr->items[j].not_flag=cnf->clauses[i].not_flag; + j++; + } } - ret=hierarchy_graph_reomve_edge(&hier->hierarchy_graph, ele_group->vertex_id, ele_superior_group->vertex_id); - if(ret<0) + if(j==cnf->declared_clause_number) { - return -1; + a_expr->user_tag=cnf->user_tag; + a_expr->item_num=j; + walker->cnt++; } - ele_group->ref_by_superior_cnt--; - ele_superior_group->ref_by_subordinate_cnt--; - return 0; +} + +static struct bool_matcher* Maat_CNF_build_bool_matcher(struct Maat_hierarchy* hier) +{ + struct bool_matcher* bm=NULL; + struct clause_list_walker clause_walker; + memset(&clause_walker, 0, sizeof(clause_walker)); + clause_walker->cnf_hash=hier->cnf_hash; + size_t n_cnf=0; + MESA_htable_iterate(hier->cnf_hash, reset_cnf_hash, &n_cnf); + MESA_htable_iterate(hier->literal2clause_list_hash, walk_literal2clause_list_hash, &clause_walker); + MESA_handle_runtime_log(hier->logger, RLOG_LV_INFO, module_maat_hierarchy, + "Maat CNF number: %zu, clause number: %zu (NOT: %zu), literal number: %zu .", + n_cnf, clause_walker->n_clause, clause_walker->n_not_flag, clause_walker->n_literal); + struct cnf_walker cnf_walker; + cnf_walker.bool_expr_array=ALLOC(struct bool_expr, n_cnf); + cnf_walker.sz=n_cnf; + cnf_walker.cnt=0; + MESA_htable_iterate(hier->cnf_hash, walk_cnf_hash, &cnf_walker); + + MESA_handle_runtime_log(hier->logger, RLOG_LV_INFO, module_maat_hierarchy, + "Valid CNF count: %zu .", cnf_walker.cnt); + + size_t mem_size=0; + bm=bool_matcher_new(cnf_walker.bool_expr_array, cnf_walker.cnt, hier->thread_num, &mem_size); + if(bm!=NULL) + { + MESA_handle_runtime_log(hier->logger, RLOG_LV_INFO, module_maat_hierarchy, + "Build bool matcher use %zu bytes memory", mem_size); + } + else + { + MESA_handle_runtime_log(hier->logger,RLOG_LV_FATAL, module_maat_hierarchy, + "Build bool matcher failed!"); + } + free(cnf_walker.bool_expr_array); + cnf_walker.bool_expr_array=NULL; + return bm; +} + +int Maat_hierarchy_add_region(struct Maat_hierarchy* hier, int region_id, int group_id) +{ +} +static void _walk_group_hash(const uchar * key, uint size, void * data, void * user) +{ + struct Maat_hierarchy* hier=(struct Maat_hierarchy*)user; + struct Maat_group_vertex* group=(struct Maat_group_vertex*)data; + struct Maat_group_vertex* superior_group=NULL; + int tmp_vid=0; + size_t i=0, top_group_cnt=0; + long long* temp_group_ids=NULL; + + if(group->ref_by_compile_cnt==0&&group->ref_by_group_cnt==0) + { + free(group->top_groups); + Maat_hierarchy_group_vertex_free(hier, group->group_id); + } + if(group->ref_by_group_cnt==0) + { + //fast path, group is only referenced by compile rules. + top_group_cnt=1; + temp_group_ids=ALLOC(long long, top_group_cnt); + temp_group_ids[0]=group->group_id; + } + else + { + igraph_vector_t *vids=&(hier->dfs_vids); + igraph_dfs(&hier->group_graph), group->vertex_id, IGRAPH_OUT, + 0, vids, NULL, NULL, NULL, NULL, NULL, NULL); + + temp_group_ids=ALLOC(long long, effective_vertices_count(vids)); + + for(i=0; i<(size_t)igraph_vector_size(vids); i++) + { + tmp_vid=(int) VECTOR(*vids)[i]; + if(tmp_vid<0) + { + break; + } + superior_group=(struct Maat_group_vertex*)MESA_htable_search(hier->vertex_id2group_hash, tmp_vid, sizeof(tmp_vid)); + if(superior_group->ref_by_compile_cnt>0)//including itself + { + temp_group_ids[top_group_cnt]=superior_group->group_id; + top_group_cnt++; + } + } + + } + pthread_mutex_lock(&(group->mutex)); + free(group->top_groups); + group->top_group_cnt=top_group_cnt; + group->top_groups=ALLOC(long long, group->top_group_cnt); + memcpy(group->top_groups, temp_group_ids, sizeof(long long)*group->top_group_cnt); + pthread_mutex_unlock(&(group->mutex)); + free(temp_group_ids); + temp_group_ids=NULL; + return; +} + +static void Maat_hierarchy_build_top_group(struct Maat_hierarchy* hier) +{ + MESA_htable_iterate(hier->group_hash, _walk_group_hash, &n_cnf); + } int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier) { - + struct bool_matcher* bm=NULL; + bm=Maat_CNF_build_bool_matcher(hier); + Maat_garbage_bag(hier->garbage_bin, hier->bm, 10, bool_matcher_free); + hier->bm=bm; + + } diff --git a/src/inc_internal/Maat_garbage_collection.cpp b/src/inc_internal/Maat_garbage_collection.cpp new file mode 100644 index 0000000..c821ec3 --- /dev/null +++ b/src/inc_internal/Maat_garbage_collection.cpp @@ -0,0 +1,73 @@ +#include + +struct Maat_garbage_bag +{ + time_t create_time; + int timeout; + int ok_times; + void *garbage; + void (* garbage_free)(void *garbage); + TAILQ_ENTRY(Maat_garbage_bag) entries; +}; +TAILQ_HEAD(Maat_garbage_bag, Maat_garbage_q); + +struct Maat_garbage_bin +{ + Maat_garbage_q garbage_q; + size_t bag_cnt; + int timeout_seconds; +}; +struct Maat_garbage_bin* Maat_garbage_bin_new(int default_timeout) +{ + struct Maat_garbage_bin* bin=ALLOC(struct Maat_garbage_bin, 1); + TAILQ_INIT(bin->garbage_q); + bin->timeout_seconds=default_timeout; + return bin; +} +void Maat_garbage_bin_free(struct Maat_garbage_bin* bin) +{ + struct Maat_garbage_bag* p=NULL; + while (p=TAILQ_FIRST(&bin->garbage_q)) + { + p->garbage_free(p->garbage); + TAILQ_REMOVE(&bin->garbage_q, p, entries); + free(p); + } + + free(bin); + return; +} +void Maat_garbage_bag(struct Maat_garbage_bin* bin, void* garbage, void (* func)(void *)) +{ + struct Maat_garbage_bag* bag=ALLOC( struct Maat_garbage_bag, 1); + bag->create_time=time(NULL); + bag->timeout=bin->timeout_seconds; + bag->garbage=garbage; + bag->garbage_free=func; + TAILQ_INSERT_TAIL(&bin->garbage_q, bag, entries); + bin->bag_cnt++; +} +void Maat_garbage_collect(struct Maat_garbage_bin* bin) +{ + struct Maat_garbage_bag* p=NULL, *tmp=NULL; + int n_clollected=0, n_bag=0; + time_t now=time(NULL); + + for(p=TAILQ_FIRST(&bin->garbage_q); p!=NULL; p=tmp) + { + tmp=TAILQ_NEXT(p, entries); + if(now-p->create_time>p->timeout) + { + p->garbage_free(p->garbage); + TAILQ_REMOVE(&bin->garbage_q, p, entries); + free(p); + n_clollected++; + } + n_bag++; + } + + assert(bin->bag_cnt==n_bag); + bin->bag_cnt-=n_clollected; + return; +} + diff --git a/src/inc_internal/Maat_garbage_collection.h b/src/inc_internal/Maat_garbage_collection.h new file mode 100644 index 0000000..c70fd9a --- /dev/null +++ b/src/inc_internal/Maat_garbage_collection.h @@ -0,0 +1,8 @@ +#pragma once + +struct Maat_garbage_bin{}; +struct Maat_garbage_bin* Maat_garbage_bin_new(void); +void Maat_garbage_bin_free(struct Maat_garbage_bin* bin); +void Maat_garbage_bag(struct Maat_garbage_bin* bin, void* garbage, int timeout, void (* func)(void *)); +void Maat_garbage_collect(struct Maat_garbage_bin* bin); +