gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1、增加Maat_command_raw_set_xx系列函数，可以操作sub-group、分组复用，增加sub-group增删的测试用例。 2、fix #13。

Browse Source
This commit is contained in:
zhengchao
2019-06-12 21:49:38 +08:00
parent a238b357d7
commit 0992c8a14b
9 changed files with 386 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
inc/Maat_command.h
View File

@@ -10,7 +10,11 @@ enum MAAT_OPERATION
MAAT_OP_ADD,
MAAT_OP_RENEW_TIMEOUT //Rule expire time is changed to now+cmd->expire_after
};
enum MAAT_GROUP_RELATION
{
PARENT_TYPE_COMPILE=0,
PARENT_TYPE_GROUP
};
enum MAAT_REGION_TYPE
{
REGION_EXPR,
@@ -104,8 +108,12 @@ struct Maat_region_t
};
struct Maat_group_t
{
int region_num;
const char* table_name;
int group_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number.
int parent_id;
int not_flag;
enum MAAT_GROUP_RELATION parent_type;
int region_num;
struct Maat_region_t *regions;
};
struct Maat_cmd_t
@@ -170,5 +178,14 @@ void Maat_cmd_key_free(struct Maat_cmd_key**keys, int number);
int Maat_cmd_key_select(Maat_feather_t feather, int label_id, struct Maat_cmd_key** keys);
int Maat_cmd_select(Maat_feather_t feather, int label_id, int * output_ids, unsigned int size);
int Maat_cmd_flushDB(Maat_feather_t feather);
int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num);
int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_region_t* region, int group_id);
int Maat_command_raw_set_group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_group_t* group);
int Maat_cmd_get_new_group_id(Maat_feather_t feather);
int Maat_cmd_get_new_region_id(Maat_feather_t feather);
#endif

14
src/entry/Maat_api.cpp
View File

@@ -181,7 +181,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
{
continue;
}
if(group_rule->compile_shortcut!=NULL&&group_rule->parent_ref_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM)
if(group_rule->compile_shortcut!=NULL&&group_rule->ref_by_parent_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM)
{
relation_arrary[shortcut_avilable_cnt]=(struct Maat_compile_group_relation*)(group_rule->compile_shortcut);
shortcut_avilable_cnt++;
@@ -662,12 +662,12 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
break;
case MAAT_OPT_JSON_FILE_PATH:
assert(_feather->input_mode==SOURCE_NONE);
ret=json2iris((const char*)value
,_feather->compile_tn,_feather->group_tn
,NULL
,_feather->json_ctx.iris_file
,sizeof(_feather->json_ctx.iris_file)
,_feather->logger);
ret=json2iris((const char*)value,
_feather->compile_tn,_feather->group_tn,
NULL,
_feather->json_ctx.iris_file,
sizeof(_feather->json_ctx.iris_file),
_feather->logger);
if(ret<0)
{
return -1;

130
src/entry/Maat_command.cpp
View File

@@ -242,13 +242,39 @@ int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq)
line[i]='0';
return 0;
}
void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int size)
void serialize_group(const struct Maat_group_t* p_group, enum MAAT_OPERATION op, char* buff, size_t sz)
{
UNUSED int ret=0;
if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD;
snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d", p_group->group_id,
p_group->parent_id,
op,
p_group->not_flag,
p_group->parent_type);
return;
}
void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int group_num, enum MAAT_OPERATION op, char* buff, size_t sz)
{
if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD;
const char* service_define=huge_service_defined?huge_service_defined:(strlen(p_m_rule->service_defined)?p_m_rule->service_defined:"null");
snprintf(buff, sz,"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t%d\t%d",
p_m_rule->config_id,
p_m_rule->service_id,
p_m_rule->action,
p_m_rule->do_blacklist,
p_m_rule->do_log,
service_define,
op,
group_num);
return;
}
void serialize_region(const struct Maat_region_t* p, int group_id, char* buff, size_t sz)
{
UNUSED size_t ret=0;
switch(p->region_type)
{
case REGION_IP:
ret=snprintf(buff,size,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1"
ret=snprintf(buff,sz,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1"
,p->region_id
,group_id
,p->ip_rule.addr_type
@@ -266,7 +292,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
case REGION_EXPR:
if(p->expr_rule.district==NULL)
{
ret=snprintf(buff,size,"%d\t%d\t%s\t%d\t%d\t%d\t1"
ret=snprintf(buff,sz,"%d\t%d\t%s\t%d\t%d\t%d\t1"
,p->region_id
,group_id
,p->expr_rule.keywords
@@ -276,7 +302,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
}
else //expr_plus
{
ret=snprintf(buff,size,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1"
ret=snprintf(buff,sz,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1"
,p->region_id
,group_id
,p->expr_rule.keywords
@@ -287,14 +313,14 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
}
break;
case REGION_INTERVAL:
ret=snprintf(buff,size,"%d\t%d\t%u\t%u\t1"
ret=snprintf(buff,sz,"%d\t%d\t%u\t%u\t1"
,p->region_id
,group_id
,p->interval_rule.low_boundary
,p->interval_rule.up_boundary);
break;
case REGION_DIGEST:
ret=snprintf(buff,size,"%d\t%d\t%llu\t%s\t%hd\t1"
ret=snprintf(buff,sz,"%d\t%d\t%llu\t%s\t%hd\t1"
,p->region_id
,group_id
,p->digest_rule.orgin_len
@@ -302,7 +328,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
,p->digest_rule.confidence_degree);
break;
case REGION_SIMILARITY:
ret=snprintf(buff,size,"%d\t%d\t%s\t%hd\t1"
ret=snprintf(buff,sz,"%d\t%d\t%s\t%hd\t1"
,p->region_id
,group_id
,p->similarity_rule.target
@@ -311,7 +337,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
default:
assert(0);
}
assert(ret<size);
assert(ret<sz);
return;
}
void empty_serial_rules(struct serial_rule_t* rule)
@@ -332,15 +358,15 @@ void empty_serial_rules(struct serial_rule_t* rule)
memset(rule,0,sizeof(struct serial_rule_t));
return;
}
void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout)
void set_serial_rule(struct serial_rule_t* rule, enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout)
{
memset(rule, 0, sizeof(struct serial_rule_t));
rule->op=op;
rule->rule_id=rule_id;
rule->label_id=label_id;
rule->timeout=timeout;
assert(strlen(table_name)<sizeof(rule->table_name));
memset(rule->table_name, 0, sizeof(rule->table_name));
memcpy(rule->table_name,table_name,strlen(table_name));
strncpy(rule->table_name, table_name, sizeof(rule->table_name));
if(line!=NULL)
{
rule->table_line=_maat_strdup(line);
@@ -875,7 +901,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
group_cmd=&(cmd->groups[grp_idx]);
group_cmd->group_id=group_inner->group_id;
if(group_inner->parent_ref_cnt>1)
if(group_inner->ref_by_parent_cnt>1)
{
continue;
}
@@ -900,7 +926,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
return 0;
}
int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,struct serial_rule_t* list, int size)
int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd, struct serial_rule_t* list, int size)
{
struct Maat_group_t* p_group=NULL;
struct Maat_region_t* p_region=NULL;
@@ -918,19 +944,12 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st
}
if(op==MAAT_OP_ADD)
{
snprintf(line,sizeof(line),"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t1\t%d",p_m_rule->config_id
,p_m_rule->service_id
,p_m_rule->action
,p_m_rule->do_blacklist
,p_m_rule->do_log
,(_cmd->huge_service_defined!=NULL)?(_cmd->huge_service_defined):(p_m_rule->service_defined)
,cmd->group_num);
set_serial_rule(list+rule_num,MAAT_OP_ADD,cmd->compile.config_id,cmd->label_id,feather->compile_tn,line,timeout);
serialize_compile(p_m_rule, _cmd->huge_service_defined, cmd->group_num, MAAT_OP_ADD, line, sizeof(line));
set_serial_rule(list+rule_num, MAAT_OP_ADD, cmd->compile.config_id, cmd->label_id, feather->compile_tn, line, timeout);
}
else
{
set_serial_rule(list+rule_num,op,cmd->compile.config_id,cmd->label_id,feather->compile_tn,NULL,timeout);
set_serial_rule(list+rule_num, op, cmd->compile.config_id, cmd->label_id, feather->compile_tn, NULL, timeout);
}
rule_num++;
for(i=0;i<cmd->group_num;i++)
@@ -943,9 +962,10 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st
p_group->group_id=feather->base_grp_seq;
feather->base_grp_seq++;
}
snprintf(line,sizeof(line),"%d\t%d\t1",p_group->group_id
,p_m_rule->config_id);
set_serial_rule(list+rule_num,MAAT_OP_ADD,p_group->group_id,0,feather->group_tn,line,timeout);
p_group->parent_id=p_m_rule->config_id;
p_group->parent_type=PARENT_TYPE_COMPILE;
serialize_group(p_group, MAAT_OP_ADD, line, sizeof(line));
set_serial_rule(list+rule_num, MAAT_OP_ADD, p_group->group_id, 0, feather->group_tn, line, timeout);
}
else
{
@@ -2374,7 +2394,7 @@ int Maat_cmd_commit(Maat_feather_t feather)
_feather->server_time=redis_server_time(write_ctx);
if(_feather->AUTO_NUMBERING_ON==1)
{
{
data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_region_id_var, new_region_num);
if(data_reply->type!=REDIS_REPLY_INTEGER)
{
@@ -2384,7 +2404,7 @@ int Maat_cmd_commit(Maat_feather_t feather)
}
_feather->base_rgn_seq=data_reply->integer-new_region_num;
freeReplyObject(data_reply);
data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_group_id_var, new_group_num);
if(data_reply->type!=REDIS_REPLY_INTEGER)
{
@@ -2448,6 +2468,19 @@ long long Maat_cmd_incrby(Maat_feather_t feather,const char* key, int increment)
}
freeReplyObject(data_reply);
return result;
}
int Maat_cmd_get_new_group_id(Maat_feather_t feather)
{
int group_id=0;
group_id=(int) Maat_cmd_incrby(feather, mr_group_id_var, 1);
return group_id;
}
int Maat_cmd_get_new_region_id(Maat_feather_t feather)
{
int region_id=0;
region_id=(int) Maat_cmd_incrby(feather, mr_region_id_var, 1);
return region_id;
}
void Maat_cmd_key_free(struct Maat_cmd_key**keys, int size)
{
@@ -2550,9 +2583,9 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger)
redisAppendCommand(ctx,"SET MAAT_VERSION %lld",maat_redis_version);
append_cmd_cnt++;
redisAppendCommand(ctx,"SET MAAT_PRE_VER %lld",maat_redis_version);
append_cmd_cnt++;
append_cmd_cnt++;
redisAppendCommand(ctx,"SET %s 1", mr_region_id_var);
append_cmd_cnt++;
append_cmd_cnt++;
redisAppendCommand(ctx,"SET %s 1", mr_group_id_var);
append_cmd_cnt++;
redisAppendCommand(ctx,"EXEC");
@@ -2574,6 +2607,41 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger)
);
}
return redis_transaction_success;
}
static int _Maat_command_set_one_line(struct _Maat_feather_t* _feather, enum MAAT_OPERATION op, int id, const char* table_name, const char* line)
{
redisContext* write_ctx=get_redis_ctx_for_write(_feather);
_feather->server_time=redis_server_time(write_ctx);
struct serial_rule_t s_rule;
set_serial_rule(&s_rule, op, id, 0, table_name, line, 0);
int transaction_success=0;
transaction_success=exec_serial_rule(write_ctx, &s_rule, 1, _feather->server_time, _feather->logger);
empty_serial_rules(&s_rule);
return transaction_success;
}
int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num)
{
struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather;
char line[MAX_TABLE_LINE_SIZE];
serialize_compile(compile, huge_service_defined, group_num, op, line, sizeof(line));
int ret=_Maat_command_set_one_line(_feather, op, compile->config_id, table_name, line);
return ret;
}
int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_region_t* region, int group_id)
{
struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather;
char line[MAX_TABLE_LINE_SIZE];
serialize_region(region, group_id, line, sizeof(line));
int ret=_Maat_command_set_one_line(_feather, op, region->region_id, region->table_name, line);
return ret;
}
int Maat_command_raw_set_group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_group_t* group)
{
struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather;
char line[MAX_TABLE_LINE_SIZE];
serialize_group(group, op, line, sizeof(line));
int ret=_Maat_command_set_one_line(_feather, op, group->group_id*1024*1204+group->parent_id, group->table_name, line);
return ret;
}
int Maat_cmd_flushDB(Maat_feather_t feather)
{

93
src/entry/Maat_rule.cpp
View File

@@ -814,14 +814,14 @@ int read_table_description(struct Maat_table_desc** p_table_info,int num,const c
conj_table=p_table_info[p->table_id];
if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Maat read table info %s line %d error:reach tableid %d conjunction upper limit."
,table_info_path,i,p->table_id);
goto invalid_table;
}
memcpy(conj_table->table_name[conj_table->conj_cnt],p->table_name[0],MAX_TABLE_NAME_LEN);
conj_table->conj_cnt++;
MESA_handle_runtime_log(logger, RLOG_LV_INFO,maat_module,
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
"Maat read table info %s:%d:conjunction %s with %s (id=%d,total=%d)."
,table_info_path,i,p->table_name[0]
,conj_table->table_name[0],conj_table->table_id,conj_table->conj_cnt);
@@ -847,7 +847,7 @@ struct Maat_group_inner* create_group_rule(int group_id, int table_id, struct Ma
group->group_id=group_id;
group->region_cnt=0;
group->region_boundary=0;
group->parent_ref_cnt=0;
group->ref_by_parent_cnt=0;
group->regions=dynamic_array_create(1,8);
group->compile_shortcut=NULL;
group->table_id=table_id;
@@ -868,7 +868,7 @@ void _destroy_group_rule(struct Maat_group_inner* group)
group->region_cnt=0;
group->region_boundary=0;
group->regions=NULL;
group->parent_ref_cnt=0;
group->ref_by_parent_cnt=0;
group->group_id=-1;
group->table_id=-1;
free(group->group_name);
@@ -879,8 +879,18 @@ void _destroy_group_rule(struct Maat_group_inner* group)
free(group);
}
size_t print_igraph_vector(igraph_vector_t *v, char* buff, size_t sz) {
long int i;
int printed=0;
for (i=0; i<igraph_vector_size(v); i++) {
printed+=snprintf(buff+printed, sz-printed, " %li", (long int) VECTOR(*v)[i]);
}
return printed;
}
#define DESTROY_GROUP_BY_REGION 0
#define DESTROY_GROUP_BY_PARENT 1
#define DESTROY_GROUP_BY_CHILD 2
void destroy_group_rule(struct Maat_group_inner* group_rule, int by_whom, struct Maat_scanner_t* scanner)
{
switch(by_whom)
@@ -888,16 +898,32 @@ void destroy_group_rule(struct Maat_group_inner* group_rule, int by_whom, struct
case DESTROY_GROUP_BY_REGION:
break;
case DESTROY_GROUP_BY_PARENT:
group_rule->parent_ref_cnt--;
group_rule->ref_by_parent_cnt--;
break;
case DESTROY_GROUP_BY_CHILD:
group_rule->ref_by_children_cnt--;
break;
default:
assert(0);
break;
}
if(group_rule->parent_ref_cnt==0&&group_rule->region_cnt==0)
igraph_vector_t v;
char buff[4096];
if(group_rule->ref_by_parent_cnt==0&&group_rule->ref_by_children_cnt==0&&group_rule->region_cnt==0)
{
HASH_delete_by_id(scanner->group_hash, group_rule->group_id);
HASH_delete_by_id(scanner->vertex_id2group, group_rule->vertex_id);
igraph_vector_init(&v, 8);
igraph_neighbors(&scanner->group_graph, &v, group_rule->vertex_id, IGRAPH_ALL);
if(igraph_vector_size(&v)>0)
{
print_igraph_vector(&v, buff, sizeof(buff));
MESA_handle_runtime_log(scanner->logger_ref, RLOG_LV_FATAL, maat_module,
"Del group %d exception, still reached by %s.",
group_rule->vertex_id, buff);
assert(0);
}
igraph_vector_destroy(&v);
//Calling _destroy_group_rule on garbage collection to free memory.
garbage_bagging(GARBAGE_GROUP_RULE, group_rule, scanner->tomb_ref);
@@ -1413,6 +1439,7 @@ struct Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
rulescan_set_param(scanner->region,RULESCAN_REGEX_GROUP,NULL,0);
}
scanner->tomb_ref=feather->garbage_q;
scanner->logger_ref=feather->logger;
scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num);
for(i=0;i<MAX_TABLE_NUM;i++)
@@ -1859,7 +1886,7 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa
}
relation->group_cnt++;
//member group->compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish.
if(a_rule_group->parent_ref_cnt==1&&relation->group_cnt==1)
if(a_rule_group->ref_by_parent_cnt==1&&relation->group_cnt==1)
{
a_rule_group->compile_shortcut=relation;
}
@@ -1867,7 +1894,7 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa
{
a_rule_group->compile_shortcut=NULL;
}
a_rule_group->parent_ref_cnt++;
a_rule_group->ref_by_parent_cnt++;
//update group's shortcut when compile has more than one group.
if(relation->group_cnt!=1)
{
@@ -1900,7 +1927,7 @@ struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relati
}
if(group_rule->group_id==group_id)
{
group_rule->parent_ref_cnt--;
group_rule->ref_by_parent_cnt--;
dynamic_array_write(relation->groups,i,NULL);
if(relation->not_flag[i]==1)
{
@@ -2387,6 +2414,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
struct Maat_group_inner* group_rule=NULL, *parent_group=NULL;
struct Maat_compile_group_relation*compile_rule=NULL;
int ret=0;
igraph_integer_t edge_id;
group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id);
if(group_rule==NULL)
{
@@ -2400,7 +2428,19 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
{
parent_group=create_group_rule(db_group_rule->parent_id, table->table_id, scanner);
}
group_rule->parent_ref_cnt++;
group_rule->ref_by_parent_cnt++;
parent_group->ref_by_children_cnt++;
ret=igraph_get_eid(&scanner->group_graph, &edge_id, group_rule->vertex_id, parent_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0);
if(edge_id>0)//if the edge was not found and error is false, then -1 will be assigned to eid.
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"update error, add sub group: %s %d to group %d error, sub group exist.",
table->table_name[table->updating_name],
db_group_rule->group_id,
db_group_rule->parent_id);
return -1;
}
//igraph allow add multiple edges between two vetex, igraph_delete_edges removes one edge per call.
igraph_add_edge(&scanner->group_graph, group_rule->vertex_id, parent_group->vertex_id);
}
else
@@ -2422,7 +2462,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
return -1;
}
}
return 0;
return 1;
}
int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner_t *scanner, void* logger)
{
@@ -2430,6 +2470,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
struct Maat_group_inner* group_rule=NULL, *parent_group=NULL;
igraph_es_t es;
int ret=0;
igraph_integer_t edge_num_before=0, edge_num_after=0;
if(db_group_rule->parent_type==PARENT_TYPE_GROUP)
@@ -2454,15 +2495,21 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
db_group_rule->parent_id);
return 0;
}
edge_num_before=igraph_ecount(&scanner->group_graph);
// The edges between the given pairs of vertices will be included in the edge selection.
//The vertex pairs must be given as the arguments of the function call, the third argument
//is the first vertex of the first edge, the fourth argument is the second vertex of the
//first edge, the fifth is the first vertex of the second edge and so on. The last element
//of the argument list must be -1 to denote the end of the argument list.
//https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small
igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group_rule->vertex_id, parent_group->vertex_id, -1);
ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group_rule->vertex_id, parent_group->vertex_id, -1);
assert(ret==IGRAPH_SUCCESS);
// ignore no such edge to abort().
igraph_set_error_handler(igraph_error_handler_ignore);
ret=igraph_delete_edges(&scanner->group_graph, es);
if(ret!=IGRAPH_SUCCESS)
edge_num_after=igraph_ecount(&scanner->group_graph);
if(ret!=IGRAPH_SUCCESS||edge_num_before-edge_num_after!=1)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module ,
"update error, delete %s group %d from parent group %d error, not such relation before.",
@@ -2471,6 +2518,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
db_group_rule->parent_id);
}
igraph_es_destroy(&es);
destroy_group_rule(parent_group, DESTROY_GROUP_BY_CHILD, scanner);
}
else
{
@@ -2478,7 +2526,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
if(relation==NULL)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"update error, delete %s group %d error : compile id %d does not exisit.",
"update error, delete %s group %d form compile %d error, compile does not exist.",
table->table_name[table->updating_name],
db_group_rule->group_id,
db_group_rule->parent_id);
@@ -2488,13 +2536,13 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
if(group_rule==NULL)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"update error, delete %s group rule error : group id %d not in compile id %d."
,table->table_name[table->updating_name]
,db_group_rule->group_id
,db_group_rule->parent_id);
"update error, delete %s group %d from compile %d error, target group does not in compile.",
table->table_name[table->updating_name],
db_group_rule->group_id,
db_group_rule->parent_id);
return 0;
}
if(relation->group_cnt==0)
if(relation->group_cnt==0 && relation->compile==NULL)
{
destroy_compile_group_relation(relation, scanner);
}
@@ -2531,7 +2579,7 @@ int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_
if(cg_relation==NULL || cg_relation->compile==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error, delete %s compile rule error : config id %d does not exist."
"update error, delete %s compile rule error : compile id %d does not exist."
,table->table_name[table->updating_name]
,compile_id);
return -1;
@@ -2542,7 +2590,7 @@ int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_
cg_relation->compile=NULL;
pthread_rwlock_unlock(&(cg_relation->rwlock));
if(cg_relation->group_cnt==0)
if(cg_relation->group_cnt==0&&cg_relation->compile==NULL)
{
destroy_compile_group_relation(cg_relation, scanner);
}
@@ -3880,10 +3928,11 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
{
scanner=feather->scanner;
}
MESA_handle_runtime_log(feather->logger, RLOG_LV_DEBUG, maat_module, "Maat table %s input: %s", table_name, line);
ret=map_str2int(feather->map_tablename2id,table_name,&table_id);
if(ret<0)
{
MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module ,"update warning,unknown table name %s",table_name);
MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s",table_name);
return -1;
}
p_table=feather->p_table_info[table_id];

5
src/entry/json2iris.cpp
View File

@@ -83,7 +83,7 @@ static int get_group_seq(struct iris_description_t* iris_cfg)
}
else
{
data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_GROUP 1");
data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY %s 1", mr_group_id_var);
sequence=(int)data_reply->integer-1;
freeReplyObject(data_reply);
}
@@ -100,7 +100,7 @@ static int get_region_seq(struct iris_description_t* iris_cfg)
}
else
{
data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_REGION 1");
data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY %s 1", mr_region_id_var);
sequence=(int)data_reply->integer-1;
freeReplyObject(data_reply);
}
@@ -1098,6 +1098,7 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
}
return 0;
}
// redis_write_ctx is used by maat_redis_tool to write json to redis.
int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger)
{
FILE* json_fp=NULL;

9
src/inc_internal/Maat_rule_internal.h
View File

@@ -23,6 +23,8 @@
extern const char *maat_module;
#define mr_region_id_var "SEQUENCE_REGION"
#define mr_group_id_var "SEQUENCE_GROUP"
#define MAX_TABLE_NUM 256
@@ -104,8 +106,7 @@ struct Maat_compile_rule
const struct Maat_table_desc* ref_table;
MAAT_RULE_EX_DATA* ads;
};
#define PARENT_TYPE_COMPILE 0
#define PARENT_TYPE_GROUP 1
struct db_group_rule_t
{
int group_id;
@@ -140,7 +141,8 @@ struct Maat_group_inner
int table_id;
int region_boundary;
int region_cnt;
int parent_ref_cnt;
int ref_by_parent_cnt;
int ref_by_children_cnt;
char* group_name;
int has_compile_neighbors;
int vertex_id;
@@ -303,6 +305,7 @@ struct Maat_scanner_t
MESA_lqueue_head region_update_q;
struct bool_matcher * bool_matcher_expr_compiler;
scan_result_t *region_rslt_buff;
void* logger_ref;
MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q
int max_thread_num;

3
test/CMakeLists.txt
View File

@@ -1,6 +1,9 @@
add_executable(maat_demo maat_demo.cpp)
target_link_libraries(maat_demo maat_frame_shared)
add_executable(test_igraph test_igraph.cpp)
target_link_libraries(test_igraph igraph-static)
add_executable(test_maatframe test_maatframe.cpp)
target_link_libraries(test_maatframe maat_frame_shared gtest)

62
test/test_igraph.cpp Normal file
View File

@@ -0,0 +1,62 @@
#include "igraph/igraph.h"
#include <assert.h>
void print_vector(igraph_vector_t *v, FILE *f) {
long int i;
for (i=0; i<igraph_vector_size(v); i++) {
fprintf(f, " %li", (long int) VECTOR(*v)[i]);
}
fprintf(f, "\n");
}
int main() {
igraph_t g;
int ret;
ret=igraph_empty(&g, 0, IGRAPH_DIRECTED);
igraph_es_t es;
igraph_integer_t edge_num_before=0, edge_num_after=0;
int v[10];
int i=0;
for(i=0; i<sizeof(v)/sizeof(int); i++)
{
v[i]=i;
igraph_add_vertices(&g, 1, NULL); //Add 1 vertice.
}
igraph_add_edge(&g, v[0], v[1]);
igraph_add_edge(&g, v[0], v[1]);
igraph_add_edge(&g, v[2], v[3]);
int edge_id=0;
ret=igraph_get_eid(&g, &edge_id, v[2], v[3], IGRAPH_DIRECTED, 0);
assert(edge_id>0);
ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, v[0], v[1], -1);
assert(ret==IGRAPH_SUCCESS);
edge_num_before=igraph_ecount(&g);
ret=igraph_delete_edges(&g, es);
edge_num_after=igraph_ecount(&g);
assert(edge_num_before-edge_num_after==1);
assert(ret==IGRAPH_SUCCESS);
igraph_es_destroy(&es);
ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, v[3], v[4], -1);
assert(ret==IGRAPH_SUCCESS);
edge_num_before=igraph_ecount(&g);
/* error test, no such edge to delete */
igraph_set_error_handler(igraph_error_handler_ignore);
ret=igraph_delete_edges(&g, es);
edge_num_after=igraph_ecount(&g);
assert(edge_num_before=edge_num_after);
assert(ret!=IGRAPH_SUCCESS);
igraph_es_destroy(&es);
igraph_destroy(&g);
return 0;
}

116
test/test_maatframe.cpp
View File

@@ -1475,6 +1475,7 @@ protected:
static int linger_timeout;
};
Maat_feather_t MaatCmdTest::_shared_feather;
void* MaatCmdTest::logger;
int MaatCmdTest::linger_timeout;
@@ -1760,7 +1761,7 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder)
int table_id=0;
table_id=Maat_table_register(feather,table_name);
table_id=Maat_table_register(feather, table_name);
ASSERT_GT(table_id, 0);
int i=0, repeat_times=4;
@@ -1792,8 +1793,121 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder)
}
return;
}
#define MaatCmdTest_SubGroup 0
TEST_F(MaatCmdTest, SubGroup)
{
/* compile1->group1--group2->region1
\
\group3->region2
compile2->group1
*/
const char* table_name="HTTP_URL";
const char* group_table_name="GROUP";
const char* compile_table_name="COMPILE";
const char* scan_data1="www.v2ex.com/t/573028#程序员的核心竞争力是什么";
const char* keyword1="程序员&核心竞争力";
const char* scan_data2="https://ask.leju.com/bj/detail/12189672562229248/?bi=tg&type=sina-pc&pos=index-dbtlwzl&wt_campaign=M_5CE750003F393&wt_source=PDPS_514ACACFD9E770";
// const char* keyword2="ask.leju.com/bj/detail/12189672562229248/?bi=tg&type=sina-pc&pos=index-dbtlwzl&wt_campaign=M_5CE750003F393&wt_source=PDPS_514ACACFD9E770";
const char* keyword2="ask.leju.com/b&/detail/12189672562229248/?&?bi=tg\\&type=sina-pc\\&&\\&pos=index-dbtlwzl\\&&\\&type=sina-pc\\&pos=index-dbtlwzl\\&";
Maat_feather_t feather=MaatCmdTest::_shared_feather;
int table_id=0;
table_id=Maat_table_register(feather, table_name);
struct Maat_rule_t compile1, compile2;
struct Maat_group_t group1, group2, group3;
struct Maat_region_t region1,region2;
memset(&compile1, 0, sizeof(compile1));
compile1.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1);
memset(&compile2, 0, sizeof(compile2));
compile2.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile2, compile_table_name, NULL, 1);
memset(&group1, 0, sizeof(group1));
group1.group_id=Maat_cmd_get_new_group_id(feather);
group1.table_name=group_table_name;
group1.parent_id=compile1.config_id;
group1.parent_type=PARENT_TYPE_COMPILE;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
group1.parent_id=compile2.config_id;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
memset(&group2, 0, sizeof(group2));
group2.group_id=Maat_cmd_get_new_group_id(feather);
group2.table_name=group_table_name;
group2.parent_id=group1.group_id;
group2.parent_type=PARENT_TYPE_GROUP;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2);
memset(&region1, 0, sizeof(region1));
region1.region_id=Maat_cmd_get_new_region_id(feather);
region1.region_type=REGION_EXPR;
region1.table_name=table_name;
region1.expr_rule.keywords=keyword1;
region1.expr_rule.expr_type=EXPR_TYPE_AND;
Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region1, group2.group_id);
sleep(1);
struct Maat_rule_t result[4];
memset(&result, 0, sizeof(result));
scan_status_t mid=NULL;
int ret=0;
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1),
result, NULL, 4,
&mid, 0);
EXPECT_EQ(ret, 2);
EXPECT_EQ(result[0].config_id, compile2.config_id);
EXPECT_EQ(result[1].config_id, compile1.config_id);
Maat_clean_status(&mid);
Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1);
sleep(1);
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1),
result, NULL, 4,
&mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, compile1.config_id);
Maat_clean_status(&mid);
group1.parent_id=compile1.config_id;
Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1);
sleep(1);
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1),
result, NULL, 4,
&mid, 0);
EXPECT_EQ(ret, -2);
Maat_clean_status(&mid);
memset(&group3, 0, sizeof(group3));
group3.group_id=Maat_cmd_get_new_group_id(feather);
group3.parent_id=group1.group_id;
group3.parent_type=PARENT_TYPE_GROUP;
group3.table_name=group_table_name;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group3);
memset(&region2, 0, sizeof(region2));
region2.region_id=Maat_cmd_get_new_region_id(feather);
region2.region_type=REGION_EXPR;
region2.table_name=table_name;
region2.expr_rule.keywords=keyword2;
region2.expr_rule.expr_type=EXPR_TYPE_AND;
Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region2, group3.group_id);
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
sleep(1);
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data2, strlen(scan_data2),
result, NULL, 4,
&mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, compile1.config_id);
Maat_clean_status(&mid);
}
TEST_F(MaatCmdTest, SetLines)
{
const int TEST_CMD_LINE_NUM=4;