diff --git a/inc/Maat_command.h b/inc/Maat_command.h index f0c8ce9..816dd63 100644 --- a/inc/Maat_command.h +++ b/inc/Maat_command.h @@ -10,7 +10,11 @@ enum MAAT_OPERATION MAAT_OP_ADD, MAAT_OP_RENEW_TIMEOUT //Rule expire time is changed to now+cmd->expire_after }; - +enum MAAT_GROUP_RELATION +{ + PARENT_TYPE_COMPILE=0, + PARENT_TYPE_GROUP +}; enum MAAT_REGION_TYPE { REGION_EXPR, @@ -104,8 +108,12 @@ struct Maat_region_t }; struct Maat_group_t { - int region_num; + const char* table_name; int group_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number. + int parent_id; + int not_flag; + enum MAAT_GROUP_RELATION parent_type; + int region_num; struct Maat_region_t *regions; }; struct Maat_cmd_t @@ -170,5 +178,14 @@ void Maat_cmd_key_free(struct Maat_cmd_key**keys, int number); int Maat_cmd_key_select(Maat_feather_t feather, int label_id, struct Maat_cmd_key** keys); int Maat_cmd_select(Maat_feather_t feather, int label_id, int * output_ids, unsigned int size); int Maat_cmd_flushDB(Maat_feather_t feather); + +int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num); +int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_region_t* region, int group_id); +int Maat_command_raw_set_group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_group_t* group); + +int Maat_cmd_get_new_group_id(Maat_feather_t feather); +int Maat_cmd_get_new_region_id(Maat_feather_t feather); + + #endif diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 22571d6..fdd1955 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -181,7 +181,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int { continue; } - if(group_rule->compile_shortcut!=NULL&&group_rule->parent_ref_cnt==1&&shortcut_avilable_cntcompile_shortcut!=NULL&&group_rule->ref_by_parent_cnt==1&&shortcut_avilable_cntcompile_shortcut); shortcut_avilable_cnt++; @@ -662,12 +662,12 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo break; case MAAT_OPT_JSON_FILE_PATH: assert(_feather->input_mode==SOURCE_NONE); - ret=json2iris((const char*)value - ,_feather->compile_tn,_feather->group_tn - ,NULL - ,_feather->json_ctx.iris_file - ,sizeof(_feather->json_ctx.iris_file) - ,_feather->logger); + ret=json2iris((const char*)value, + _feather->compile_tn,_feather->group_tn, + NULL, + _feather->json_ctx.iris_file, + sizeof(_feather->json_ctx.iris_file), + _feather->logger); if(ret<0) { return -1; diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 7db6e6c..a86d303 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -242,13 +242,39 @@ int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) line[i]='0'; return 0; } -void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int size) +void serialize_group(const struct Maat_group_t* p_group, enum MAAT_OPERATION op, char* buff, size_t sz) { - UNUSED int ret=0; + if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD; + snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d", p_group->group_id, + p_group->parent_id, + op, + p_group->not_flag, + p_group->parent_type); + return; +} +void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int group_num, enum MAAT_OPERATION op, char* buff, size_t sz) +{ + if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD; + const char* service_define=huge_service_defined?huge_service_defined:(strlen(p_m_rule->service_defined)?p_m_rule->service_defined:"null"); + + snprintf(buff, sz,"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t%d\t%d", + p_m_rule->config_id, + p_m_rule->service_id, + p_m_rule->action, + p_m_rule->do_blacklist, + p_m_rule->do_log, + service_define, + op, + group_num); + return; +} +void serialize_region(const struct Maat_region_t* p, int group_id, char* buff, size_t sz) +{ + UNUSED size_t ret=0; switch(p->region_type) { case REGION_IP: - ret=snprintf(buff,size,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1" + ret=snprintf(buff,sz,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1" ,p->region_id ,group_id ,p->ip_rule.addr_type @@ -266,7 +292,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int case REGION_EXPR: if(p->expr_rule.district==NULL) { - ret=snprintf(buff,size,"%d\t%d\t%s\t%d\t%d\t%d\t1" + ret=snprintf(buff,sz,"%d\t%d\t%s\t%d\t%d\t%d\t1" ,p->region_id ,group_id ,p->expr_rule.keywords @@ -276,7 +302,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int } else //expr_plus { - ret=snprintf(buff,size,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1" + ret=snprintf(buff,sz,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1" ,p->region_id ,group_id ,p->expr_rule.keywords @@ -287,14 +313,14 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int } break; case REGION_INTERVAL: - ret=snprintf(buff,size,"%d\t%d\t%u\t%u\t1" + ret=snprintf(buff,sz,"%d\t%d\t%u\t%u\t1" ,p->region_id ,group_id ,p->interval_rule.low_boundary ,p->interval_rule.up_boundary); break; case REGION_DIGEST: - ret=snprintf(buff,size,"%d\t%d\t%llu\t%s\t%hd\t1" + ret=snprintf(buff,sz,"%d\t%d\t%llu\t%s\t%hd\t1" ,p->region_id ,group_id ,p->digest_rule.orgin_len @@ -302,7 +328,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int ,p->digest_rule.confidence_degree); break; case REGION_SIMILARITY: - ret=snprintf(buff,size,"%d\t%d\t%s\t%hd\t1" + ret=snprintf(buff,sz,"%d\t%d\t%s\t%hd\t1" ,p->region_id ,group_id ,p->similarity_rule.target @@ -311,7 +337,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int default: assert(0); } - assert(retop=op; rule->rule_id=rule_id; rule->label_id=label_id; rule->timeout=timeout; assert(strlen(table_name)table_name)); - memset(rule->table_name, 0, sizeof(rule->table_name)); - memcpy(rule->table_name,table_name,strlen(table_name)); + strncpy(rule->table_name, table_name, sizeof(rule->table_name)); if(line!=NULL) { rule->table_line=_maat_strdup(line); @@ -875,7 +901,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _ group_cmd=&(cmd->groups[grp_idx]); group_cmd->group_id=group_inner->group_id; - if(group_inner->parent_ref_cnt>1) + if(group_inner->ref_by_parent_cnt>1) { continue; } @@ -900,7 +926,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _ return 0; } -int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,struct serial_rule_t* list, int size) +int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd, struct serial_rule_t* list, int size) { struct Maat_group_t* p_group=NULL; struct Maat_region_t* p_region=NULL; @@ -918,19 +944,12 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st } if(op==MAAT_OP_ADD) { - snprintf(line,sizeof(line),"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t1\t%d",p_m_rule->config_id - ,p_m_rule->service_id - ,p_m_rule->action - ,p_m_rule->do_blacklist - ,p_m_rule->do_log - ,(_cmd->huge_service_defined!=NULL)?(_cmd->huge_service_defined):(p_m_rule->service_defined) - ,cmd->group_num); - set_serial_rule(list+rule_num,MAAT_OP_ADD,cmd->compile.config_id,cmd->label_id,feather->compile_tn,line,timeout); - + serialize_compile(p_m_rule, _cmd->huge_service_defined, cmd->group_num, MAAT_OP_ADD, line, sizeof(line)); + set_serial_rule(list+rule_num, MAAT_OP_ADD, cmd->compile.config_id, cmd->label_id, feather->compile_tn, line, timeout); } else { - set_serial_rule(list+rule_num,op,cmd->compile.config_id,cmd->label_id,feather->compile_tn,NULL,timeout); + set_serial_rule(list+rule_num, op, cmd->compile.config_id, cmd->label_id, feather->compile_tn, NULL, timeout); } rule_num++; for(i=0;igroup_num;i++) @@ -943,9 +962,10 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st p_group->group_id=feather->base_grp_seq; feather->base_grp_seq++; } - snprintf(line,sizeof(line),"%d\t%d\t1",p_group->group_id - ,p_m_rule->config_id); - set_serial_rule(list+rule_num,MAAT_OP_ADD,p_group->group_id,0,feather->group_tn,line,timeout); + p_group->parent_id=p_m_rule->config_id; + p_group->parent_type=PARENT_TYPE_COMPILE; + serialize_group(p_group, MAAT_OP_ADD, line, sizeof(line)); + set_serial_rule(list+rule_num, MAAT_OP_ADD, p_group->group_id, 0, feather->group_tn, line, timeout); } else { @@ -2374,7 +2394,7 @@ int Maat_cmd_commit(Maat_feather_t feather) if(_feather->AUTO_NUMBERING_ON==1) { - data_reply=_wrap_redisCommand(write_ctx,"INCRBY SEQUENCE_REGION %d",new_region_num); + data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_region_id_var, new_region_num); if(data_reply->type!=REDIS_REPLY_INTEGER) { freeReplyObject(data_reply); @@ -2384,7 +2404,7 @@ int Maat_cmd_commit(Maat_feather_t feather) _feather->base_rgn_seq=data_reply->integer-new_region_num; freeReplyObject(data_reply); - data_reply=_wrap_redisCommand(write_ctx,"INCRBY SEQUENCE_GROUP %d",new_group_num); + data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_group_id_var, new_group_num); if(data_reply->type!=REDIS_REPLY_INTEGER) { freeReplyObject(data_reply); @@ -2448,6 +2468,19 @@ long long Maat_cmd_incrby(Maat_feather_t feather,const char* key, int increment) freeReplyObject(data_reply); return result; } +int Maat_cmd_get_new_group_id(Maat_feather_t feather) +{ + int group_id=0; + group_id=(int) Maat_cmd_incrby(feather, mr_group_id_var, 1); + return group_id; +} +int Maat_cmd_get_new_region_id(Maat_feather_t feather) +{ + int region_id=0; + region_id=(int) Maat_cmd_incrby(feather, mr_region_id_var, 1); + return region_id; +} + void Maat_cmd_key_free(struct Maat_cmd_key**keys, int size) { int i=0; @@ -2550,9 +2583,9 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger) append_cmd_cnt++; redisAppendCommand(ctx,"SET MAAT_PRE_VER %lld",maat_redis_version); append_cmd_cnt++; - redisAppendCommand(ctx,"SET SEQUENCE_REGION 1",maat_redis_version); + redisAppendCommand(ctx,"SET %s 1", mr_region_id_var); append_cmd_cnt++; - redisAppendCommand(ctx,"SET SEQUENCE_GROUP 1",maat_redis_version); + redisAppendCommand(ctx,"SET %s 1", mr_group_id_var); append_cmd_cnt++; redisAppendCommand(ctx,"EXEC"); append_cmd_cnt++; @@ -2574,6 +2607,41 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger) } return redis_transaction_success; } +static int _Maat_command_set_one_line(struct _Maat_feather_t* _feather, enum MAAT_OPERATION op, int id, const char* table_name, const char* line) +{ + redisContext* write_ctx=get_redis_ctx_for_write(_feather); + _feather->server_time=redis_server_time(write_ctx); + struct serial_rule_t s_rule; + set_serial_rule(&s_rule, op, id, 0, table_name, line, 0); + int transaction_success=0; + transaction_success=exec_serial_rule(write_ctx, &s_rule, 1, _feather->server_time, _feather->logger); + empty_serial_rules(&s_rule); + return transaction_success; +} +int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num) +{ + struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather; + char line[MAX_TABLE_LINE_SIZE]; + serialize_compile(compile, huge_service_defined, group_num, op, line, sizeof(line)); + int ret=_Maat_command_set_one_line(_feather, op, compile->config_id, table_name, line); + return ret; +} +int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_region_t* region, int group_id) +{ + struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather; + char line[MAX_TABLE_LINE_SIZE]; + serialize_region(region, group_id, line, sizeof(line)); + int ret=_Maat_command_set_one_line(_feather, op, region->region_id, region->table_name, line); + return ret; +} +int Maat_command_raw_set_group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_group_t* group) +{ + struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather; + char line[MAX_TABLE_LINE_SIZE]; + serialize_group(group, op, line, sizeof(line)); + int ret=_Maat_command_set_one_line(_feather, op, group->group_id*1024*1204+group->parent_id, group->table_name, line); + return ret; +} int Maat_cmd_flushDB(Maat_feather_t feather) { _Maat_feather_t* _feather=(_Maat_feather_t*)feather; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 56e2087..eb78d9d 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -814,14 +814,14 @@ int read_table_description(struct Maat_table_desc** p_table_info,int num,const c conj_table=p_table_info[p->table_id]; if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Maat read table info %s line %d error:reach tableid %d conjunction upper limit." ,table_info_path,i,p->table_id); goto invalid_table; } memcpy(conj_table->table_name[conj_table->conj_cnt],p->table_name[0],MAX_TABLE_NAME_LEN); conj_table->conj_cnt++; - MESA_handle_runtime_log(logger, RLOG_LV_INFO,maat_module, + MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module, "Maat read table info %s:%d:conjunction %s with %s (id=%d,total=%d)." ,table_info_path,i,p->table_name[0] ,conj_table->table_name[0],conj_table->table_id,conj_table->conj_cnt); @@ -847,7 +847,7 @@ struct Maat_group_inner* create_group_rule(int group_id, int table_id, struct Ma group->group_id=group_id; group->region_cnt=0; group->region_boundary=0; - group->parent_ref_cnt=0; + group->ref_by_parent_cnt=0; group->regions=dynamic_array_create(1,8); group->compile_shortcut=NULL; group->table_id=table_id; @@ -868,7 +868,7 @@ void _destroy_group_rule(struct Maat_group_inner* group) group->region_cnt=0; group->region_boundary=0; group->regions=NULL; - group->parent_ref_cnt=0; + group->ref_by_parent_cnt=0; group->group_id=-1; group->table_id=-1; free(group->group_name); @@ -879,8 +879,18 @@ void _destroy_group_rule(struct Maat_group_inner* group) free(group); } +size_t print_igraph_vector(igraph_vector_t *v, char* buff, size_t sz) { + long int i; + int printed=0; + for (i=0; iparent_ref_cnt--; + group_rule->ref_by_parent_cnt--; + break; + case DESTROY_GROUP_BY_CHILD: + group_rule->ref_by_children_cnt--; break; default: assert(0); break; } - if(group_rule->parent_ref_cnt==0&&group_rule->region_cnt==0) + igraph_vector_t v; + char buff[4096]; + if(group_rule->ref_by_parent_cnt==0&&group_rule->ref_by_children_cnt==0&&group_rule->region_cnt==0) { HASH_delete_by_id(scanner->group_hash, group_rule->group_id); - HASH_delete_by_id(scanner->vertex_id2group, group_rule->vertex_id); + HASH_delete_by_id(scanner->vertex_id2group, group_rule->vertex_id); + igraph_vector_init(&v, 8); + igraph_neighbors(&scanner->group_graph, &v, group_rule->vertex_id, IGRAPH_ALL); + if(igraph_vector_size(&v)>0) + { + print_igraph_vector(&v, buff, sizeof(buff)); + MESA_handle_runtime_log(scanner->logger_ref, RLOG_LV_FATAL, maat_module, + "Del group %d exception, still reached by %s.", + group_rule->vertex_id, buff); + assert(0); + } + igraph_vector_destroy(&v); //Calling _destroy_group_rule on garbage collection to free memory. garbage_bagging(GARBAGE_GROUP_RULE, group_rule, scanner->tomb_ref); @@ -1413,6 +1439,7 @@ struct Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t rulescan_set_param(scanner->region,RULESCAN_REGEX_GROUP,NULL,0); } scanner->tomb_ref=feather->garbage_q; + scanner->logger_ref=feather->logger; scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num); for(i=0;igroup_cnt++; //member group->compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish. - if(a_rule_group->parent_ref_cnt==1&&relation->group_cnt==1) + if(a_rule_group->ref_by_parent_cnt==1&&relation->group_cnt==1) { a_rule_group->compile_shortcut=relation; } @@ -1867,7 +1894,7 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa { a_rule_group->compile_shortcut=NULL; } - a_rule_group->parent_ref_cnt++; + a_rule_group->ref_by_parent_cnt++; //update group's shortcut when compile has more than one group. if(relation->group_cnt!=1) { @@ -1900,7 +1927,7 @@ struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relati } if(group_rule->group_id==group_id) { - group_rule->parent_ref_cnt--; + group_rule->ref_by_parent_cnt--; dynamic_array_write(relation->groups,i,NULL); if(relation->not_flag[i]==1) { @@ -2387,6 +2414,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro struct Maat_group_inner* group_rule=NULL, *parent_group=NULL; struct Maat_compile_group_relation*compile_rule=NULL; int ret=0; + igraph_integer_t edge_id; group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); if(group_rule==NULL) { @@ -2400,7 +2428,19 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro { parent_group=create_group_rule(db_group_rule->parent_id, table->table_id, scanner); } - group_rule->parent_ref_cnt++; + group_rule->ref_by_parent_cnt++; + parent_group->ref_by_children_cnt++; + ret=igraph_get_eid(&scanner->group_graph, &edge_id, group_rule->vertex_id, parent_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); + if(edge_id>0)//if the edge was not found and error is false, then -1 will be assigned to eid. + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "update error, add sub group: %s %d to group %d error, sub group exist.", + table->table_name[table->updating_name], + db_group_rule->group_id, + db_group_rule->parent_id); + return -1; + } + //igraph allow add multiple edges between two vetex, igraph_delete_edges removes one edge per call. igraph_add_edge(&scanner->group_graph, group_rule->vertex_id, parent_group->vertex_id); } else @@ -2422,7 +2462,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro return -1; } } - return 0; + return 1; } int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner_t *scanner, void* logger) { @@ -2430,6 +2470,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro struct Maat_group_inner* group_rule=NULL, *parent_group=NULL; igraph_es_t es; int ret=0; + igraph_integer_t edge_num_before=0, edge_num_after=0; if(db_group_rule->parent_type==PARENT_TYPE_GROUP) @@ -2454,15 +2495,21 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro db_group_rule->parent_id); return 0; } + edge_num_before=igraph_ecount(&scanner->group_graph); // The edges between the given pairs of vertices will be included in the edge selection. //The vertex pairs must be given as the arguments of the function call, the third argument //is the first vertex of the first edge, the fourth argument is the second vertex of the //first edge, the fifth is the first vertex of the second edge and so on. The last element //of the argument list must be -1 to denote the end of the argument list. //https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small - igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group_rule->vertex_id, parent_group->vertex_id, -1); + ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group_rule->vertex_id, parent_group->vertex_id, -1); + assert(ret==IGRAPH_SUCCESS); + // ignore no such edge to abort(). + igraph_set_error_handler(igraph_error_handler_ignore); ret=igraph_delete_edges(&scanner->group_graph, es); - if(ret!=IGRAPH_SUCCESS) + edge_num_after=igraph_ecount(&scanner->group_graph); + + if(ret!=IGRAPH_SUCCESS||edge_num_before-edge_num_after!=1) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module , "update error, delete %s group %d from parent group %d error, not such relation before.", @@ -2471,6 +2518,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro db_group_rule->parent_id); } igraph_es_destroy(&es); + destroy_group_rule(parent_group, DESTROY_GROUP_BY_CHILD, scanner); } else { @@ -2478,7 +2526,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro if(relation==NULL) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "update error, delete %s group %d error : compile id %d does not exisit.", + "update error, delete %s group %d form compile %d error, compile does not exist.", table->table_name[table->updating_name], db_group_rule->group_id, db_group_rule->parent_id); @@ -2488,13 +2536,13 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro if(group_rule==NULL) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "update error, delete %s group rule error : group id %d not in compile id %d." - ,table->table_name[table->updating_name] - ,db_group_rule->group_id - ,db_group_rule->parent_id); + "update error, delete %s group %d from compile %d error, target group does not in compile.", + table->table_name[table->updating_name], + db_group_rule->group_id, + db_group_rule->parent_id); return 0; } - if(relation->group_cnt==0) + if(relation->group_cnt==0 && relation->compile==NULL) { destroy_compile_group_relation(relation, scanner); } @@ -2531,7 +2579,7 @@ int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_ if(cg_relation==NULL || cg_relation->compile==NULL) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error, delete %s compile rule error : config id %d does not exist." + "update error, delete %s compile rule error : compile id %d does not exist." ,table->table_name[table->updating_name] ,compile_id); return -1; @@ -2542,7 +2590,7 @@ int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_ cg_relation->compile=NULL; pthread_rwlock_unlock(&(cg_relation->rwlock)); - if(cg_relation->group_cnt==0) + if(cg_relation->group_cnt==0&&cg_relation->compile==NULL) { destroy_compile_group_relation(cg_relation, scanner); } @@ -3880,10 +3928,11 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) { scanner=feather->scanner; } + MESA_handle_runtime_log(feather->logger, RLOG_LV_DEBUG, maat_module, "Maat table %s input: %s", table_name, line); ret=map_str2int(feather->map_tablename2id,table_name,&table_id); if(ret<0) { - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module ,"update warning,unknown table name %s",table_name); + MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s",table_name); return -1; } p_table=feather->p_table_info[table_id]; diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 8150230..5d45c85 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -83,7 +83,7 @@ static int get_group_seq(struct iris_description_t* iris_cfg) } else { - data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_GROUP 1"); + data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY %s 1", mr_group_id_var); sequence=(int)data_reply->integer-1; freeReplyObject(data_reply); } @@ -100,7 +100,7 @@ static int get_region_seq(struct iris_description_t* iris_cfg) } else { - data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_REGION 1"); + data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY %s 1", mr_region_id_var); sequence=(int)data_reply->integer-1; freeReplyObject(data_reply); } @@ -1098,6 +1098,7 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) } return 0; } +// redis_write_ctx is used by maat_redis_tool to write json to redis. int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger) { FILE* json_fp=NULL; diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 48114be..8013754 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -23,6 +23,8 @@ extern const char *maat_module; +#define mr_region_id_var "SEQUENCE_REGION" +#define mr_group_id_var "SEQUENCE_GROUP" #define MAX_TABLE_NUM 256 @@ -104,8 +106,7 @@ struct Maat_compile_rule const struct Maat_table_desc* ref_table; MAAT_RULE_EX_DATA* ads; }; -#define PARENT_TYPE_COMPILE 0 -#define PARENT_TYPE_GROUP 1 + struct db_group_rule_t { int group_id; @@ -140,7 +141,8 @@ struct Maat_group_inner int table_id; int region_boundary; int region_cnt; - int parent_ref_cnt; + int ref_by_parent_cnt; + int ref_by_children_cnt; char* group_name; int has_compile_neighbors; int vertex_id; @@ -303,6 +305,7 @@ struct Maat_scanner_t MESA_lqueue_head region_update_q; struct bool_matcher * bool_matcher_expr_compiler; scan_result_t *region_rslt_buff; + void* logger_ref; MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q int max_thread_num; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ef8db6a..02e8590 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,6 +1,9 @@ add_executable(maat_demo maat_demo.cpp) target_link_libraries(maat_demo maat_frame_shared) +add_executable(test_igraph test_igraph.cpp) +target_link_libraries(test_igraph igraph-static) + add_executable(test_maatframe test_maatframe.cpp) target_link_libraries(test_maatframe maat_frame_shared gtest) diff --git a/test/test_igraph.cpp b/test/test_igraph.cpp new file mode 100644 index 0000000..a4c4d24 --- /dev/null +++ b/test/test_igraph.cpp @@ -0,0 +1,62 @@ +#include "igraph/igraph.h" +#include + +void print_vector(igraph_vector_t *v, FILE *f) { + long int i; + for (i=0; i0); + + ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, v[0], v[1], -1); + assert(ret==IGRAPH_SUCCESS); + edge_num_before=igraph_ecount(&g); + ret=igraph_delete_edges(&g, es); + edge_num_after=igraph_ecount(&g); + assert(edge_num_before-edge_num_after==1); + assert(ret==IGRAPH_SUCCESS); + igraph_es_destroy(&es); + + ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, v[3], v[4], -1); + assert(ret==IGRAPH_SUCCESS); + edge_num_before=igraph_ecount(&g); + /* error test, no such edge to delete */ + igraph_set_error_handler(igraph_error_handler_ignore); + + ret=igraph_delete_edges(&g, es); + edge_num_after=igraph_ecount(&g); + assert(edge_num_before=edge_num_after); + assert(ret!=IGRAPH_SUCCESS); + igraph_es_destroy(&es); + + + igraph_destroy(&g); + + return 0; +} + diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 7cbb629..920dbf7 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -1475,6 +1475,7 @@ protected: static int linger_timeout; }; Maat_feather_t MaatCmdTest::_shared_feather; + void* MaatCmdTest::logger; int MaatCmdTest::linger_timeout; @@ -1760,7 +1761,7 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder) int table_id=0; - table_id=Maat_table_register(feather,table_name); + table_id=Maat_table_register(feather, table_name); ASSERT_GT(table_id, 0); int i=0, repeat_times=4; @@ -1792,8 +1793,121 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder) } return; } +#define MaatCmdTest_SubGroup 0 +TEST_F(MaatCmdTest, SubGroup) +{ +/* compile1->group1--group2->region1 + \ + \group3->region2 + compile2->group1 +*/ + const char* table_name="HTTP_URL"; + const char* group_table_name="GROUP"; + const char* compile_table_name="COMPILE"; + const char* scan_data1="www.v2ex.com/t/573028#程序员的核心竞争力是什么"; + const char* keyword1="程序员&核心竞争力"; + + const char* scan_data2="https://ask.leju.com/bj/detail/12189672562229248/?bi=tg&type=sina-pc&pos=index-dbtlwzl&wt_campaign=M_5CE750003F393&wt_source=PDPS_514ACACFD9E770"; +// const char* keyword2="ask.leju.com/bj/detail/12189672562229248/?bi=tg&type=sina-pc&pos=index-dbtlwzl&wt_campaign=M_5CE750003F393&wt_source=PDPS_514ACACFD9E770"; + const char* keyword2="ask.leju.com/b&/detail/12189672562229248/?&?bi=tg\\&type=sina-pc\\&&\\&pos=index-dbtlwzl\\&&\\&type=sina-pc\\&pos=index-dbtlwzl\\&"; + Maat_feather_t feather=MaatCmdTest::_shared_feather; + int table_id=0; + table_id=Maat_table_register(feather, table_name); + struct Maat_rule_t compile1, compile2; + struct Maat_group_t group1, group2, group3; + struct Maat_region_t region1,region2; + memset(&compile1, 0, sizeof(compile1)); + compile1.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1); + Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1); + + memset(&compile2, 0, sizeof(compile2)); + compile2.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1); + Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile2, compile_table_name, NULL, 1); + + + memset(&group1, 0, sizeof(group1)); + group1.group_id=Maat_cmd_get_new_group_id(feather); + group1.table_name=group_table_name; + group1.parent_id=compile1.config_id; + group1.parent_type=PARENT_TYPE_COMPILE; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1); + + group1.parent_id=compile2.config_id; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1); + + memset(&group2, 0, sizeof(group2)); + group2.group_id=Maat_cmd_get_new_group_id(feather); + group2.table_name=group_table_name; + group2.parent_id=group1.group_id; + group2.parent_type=PARENT_TYPE_GROUP; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2); + + memset(®ion1, 0, sizeof(region1)); + region1.region_id=Maat_cmd_get_new_region_id(feather); + region1.region_type=REGION_EXPR; + region1.table_name=table_name; + region1.expr_rule.keywords=keyword1; + region1.expr_rule.expr_type=EXPR_TYPE_AND; + Maat_command_raw_set_region(feather, MAAT_OP_ADD, ®ion1, group2.group_id); + sleep(1); + struct Maat_rule_t result[4]; + memset(&result, 0, sizeof(result)); + + scan_status_t mid=NULL; + int ret=0; + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1), + result, NULL, 4, + &mid, 0); + EXPECT_EQ(ret, 2); + EXPECT_EQ(result[0].config_id, compile2.config_id); + EXPECT_EQ(result[1].config_id, compile1.config_id); + Maat_clean_status(&mid); + + Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1); + sleep(1); + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1), + result, NULL, 4, + &mid, 0); + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, compile1.config_id); + Maat_clean_status(&mid); + + group1.parent_id=compile1.config_id; + Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1); + sleep(1); + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1), + result, NULL, 4, + &mid, 0); + EXPECT_EQ(ret, -2); + Maat_clean_status(&mid); + + memset(&group3, 0, sizeof(group3)); + group3.group_id=Maat_cmd_get_new_group_id(feather); + group3.parent_id=group1.group_id; + group3.parent_type=PARENT_TYPE_GROUP; + group3.table_name=group_table_name; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group3); + + memset(®ion2, 0, sizeof(region2)); + region2.region_id=Maat_cmd_get_new_region_id(feather); + region2.region_type=REGION_EXPR; + region2.table_name=table_name; + region2.expr_rule.keywords=keyword2; + region2.expr_rule.expr_type=EXPR_TYPE_AND; + Maat_command_raw_set_region(feather, MAAT_OP_ADD, ®ion2, group3.group_id); + + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1); + sleep(1); + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data2, strlen(scan_data2), + result, NULL, 4, + &mid, 0); + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, compile1.config_id); + Maat_clean_status(&mid); + +} TEST_F(MaatCmdTest, SetLines) { const int TEST_CMD_LINE_NUM=4;