gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

初步完成编码，待解决配置结构化哈希的多线程安全访问。

Browse Source
This commit is contained in:
zhengchao
2017-07-03 19:54:47 +08:00
parent 7e68a46292
commit 63b1a9b13e
5 changed files with 596 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
inc/Maat_command.h
View File

@@ -51,10 +51,10 @@ enum MAAT_ADDR_DIRECTION
struct Maat_rgn_str_t struct Maat_rgn_str_t
{ {
const char *keywords; const char *keywords;
const char *district; const char *district;// optional for expr_plus, otherwise set to NULL.
enum MAAT_EXPR_TYPE expr_type; enum MAAT_EXPR_TYPE expr_type;
enum MAAT_MATCH_METHOD match_method; enum MAAT_MATCH_METHOD match_method;
enum MAAT_CASE_TYPE case_type; enum MAAT_CASE_TYPE hex_bin;
}; };
struct Maat_rgn_addr_t struct Maat_rgn_addr_t
{ {
@@ -94,7 +94,7 @@ struct Maat_region_t
union union
{ {
struct Maat_rgn_str_t expr_rule; struct Maat_rgn_str_t expr_rule;
struct Maat_rgn_addr_t addr_rule; struct Maat_rgn_addr_t ip_rule;
struct Maat_rgn_intv_t interval_rule; struct Maat_rgn_intv_t interval_rule;
struct Maat_rgn_digest_t digest_rule; struct Maat_rgn_digest_t digest_rule;
struct Maat_rgn_sim_t similarity_rule; struct Maat_rgn_sim_t similarity_rule;
@@ -104,14 +104,12 @@ struct Maat_group_t
{ {
int region_num; int region_num;
int group_id; //Any, maat will assigned one. int group_id; //Any, maat will assigned one.
char* table_name;//optional, if not specified, maat will assigned to the first GROUP table in table_info.conf.
char* group_name;//optional, for group reuse. char* group_name;//optional, for group reuse.
struct Maat_region_t *regions; struct Maat_region_t *regions;
}; };
struct Maat_command_t struct Maat_command_t
{ {
struct Maat_rule_t compile;// for MAAT_OP_DEL, only compile.config_id is necessary. struct Maat_rule_t compile;// for MAAT_OP_DEL, only compile.config_id is necessary.
char* table_name; //optional, if not specified, maat will assigned to the first COMPILE table in table_info.conf.
int group_num; // for MAAT_OP_DEL, Any. int group_num; // for MAAT_OP_DEL, Any.
struct Maat_group_t* groups;// for MAAT_OP_DEL, SET to NULL. struct Maat_group_t* groups;// for MAAT_OP_DEL, SET to NULL.
}; };

31
src/entry/Maat_api.cpp
View File

@@ -106,21 +106,21 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
int r_in_c_cnt=0; int r_in_c_cnt=0;
int shortcut_avilable_cnt=0; int shortcut_avilable_cnt=0;
void* bool_matcher=feather->scanner->expr_compiler; void* bool_matcher=feather->scanner->expr_compiler;
struct _Maat_group_rule_t* group_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL;
struct _Maat_compile_rule_t* array_mi_rule[MAX_SCANNER_HIT_NUM]; struct _Maat_compile_inner_t* array_mi_rule[MAX_SCANNER_HIT_NUM];
struct _Maat_compile_rule_t* _mi_rule=NULL; struct _Maat_compile_inner_t* _mi_rule=NULL;
int region_pos[MAX_SCANNER_HIT_NUM]; int region_pos[MAX_SCANNER_HIT_NUM];
_mid->cur_hit_cnt=0; _mid->cur_hit_cnt=0;
for(i=0;i<region_hit_num;i++) for(i=0;i<region_hit_num;i++)
{ {
group_rule=*(struct _Maat_group_rule_t**)((char*)region_hit+region_type_size*i+group_offset); group_rule=*(struct _Maat_group_inner_t**)((char*)region_hit+region_type_size*i+group_offset);
if(group_rule->group_id<0) if(group_rule->group_id<0)
{ {
continue; continue;
} }
if(group_rule->compile_shortcut!=NULL&&group_rule->ref_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM) if(group_rule->compile_shortcut!=NULL&&group_rule->ref_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM)
{ {
array_mi_rule[shortcut_avilable_cnt]=(struct _Maat_compile_rule_t*)(group_rule->compile_shortcut); array_mi_rule[shortcut_avilable_cnt]=(struct _Maat_compile_inner_t*)(group_rule->compile_shortcut);
shortcut_avilable_cnt++; shortcut_avilable_cnt++;
} }
_mid->cur_hit_id[_mid->cur_hit_cnt]=group_rule->group_id; _mid->cur_hit_id[_mid->cur_hit_cnt]=group_rule->group_id;
@@ -183,23 +183,24 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
return result_cnt; return result_cnt;
} }
int exprid2region_id(struct _Maat_group_rule_t* group_rule,int expr_id,int* district_id) int exprid2region_id(struct _Maat_group_inner_t* group_rule,int expr_id,int* district_id)
{ {
int i=0,region_id=-1; int i=0,j=0,region_id=-1;
struct _Maat_region_rule_t* region_rule=NULL; struct _Maat_region_inner_t* region_rule=NULL;
assert(group_rule->group_id>=0); assert(group_rule->group_id>=0);
pthread_mutex_lock(&(group_rule->mutex)); pthread_mutex_lock(&(group_rule->mutex));
for(i=0;i<group_rule->region_boundary;i++) for(i=0;i<group_rule->region_boundary;i++)
{ {
region_rule=(struct _Maat_region_rule_t*)dynamic_array_read(group_rule->region_rules, i); region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group_rule->regions, i);
if(region_rule==NULL) if(region_rule==NULL)
{ {
continue; continue;
} }
if(region_rule->expr_id==expr_id) if(expr_id>=region_rule->expr_id_lb&&expr_id<=region_rule->expr_id_ub)
{ {
region_id=region_rule->region_id; region_id=region_rule->region_id;
*district_id=region_rule->district_id; *district_id=region_rule->district_id;
break;
} }
} }
pthread_mutex_unlock(&(group_rule->mutex)); pthread_mutex_unlock(&(group_rule->mutex));
@@ -207,13 +208,13 @@ int exprid2region_id(struct _Maat_group_rule_t* group_rule,int expr_id,int* dist
} }
int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num) int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num)
{ {
struct _Maat_group_rule_t* group_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL;
int i=0; int i=0;
int district_id=-1,region_id=-1; int district_id=-1,region_id=-1;
int ret_region_num=region_hit_num; int ret_region_num=region_hit_num;
while(i<ret_region_num) while(i<ret_region_num)
{ {
group_rule=(struct _Maat_group_rule_t*)(region_hit[i].tag); group_rule=(struct _Maat_group_inner_t*)(region_hit[i].tag);
region_id=exprid2region_id(group_rule, region_hit[i].expr_id,&district_id); region_id=exprid2region_id(group_rule, region_hit[i].expr_id,&district_id);
if(region_id>0&&district_id!=_mid->district_id) if(region_id>0&&district_id!=_mid->district_id)
{ {
@@ -309,7 +310,7 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi
memset(r_in_c_flag,0,sizeof(r_in_c_flag)); memset(r_in_c_flag,0,sizeof(r_in_c_flag));
memset(region_pos,0,sizeof(region_pos)); memset(region_pos,0,sizeof(region_pos));
struct _Maat_group_rule_t* group_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL;
//for each hitted compile cfg,find its region_ids //for each hitted compile cfg,find its region_ids
for(i=0;i<compile_cnt&&i<detail_num;i++) for(i=0;i<compile_cnt&&i<detail_num;i++)
{ {
@@ -321,7 +322,7 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi
{ {
pos=region_pos[j]; pos=region_pos[j];
r_in_c_flag[pos]=1; r_in_c_flag[pos]=1;
group_rule=(struct _Maat_group_rule_t*)(region_hit[pos].tag); group_rule=(struct _Maat_group_inner_t*)(region_hit[pos].tag);
region_id=exprid2region_id(group_rule,region_hit[pos].expr_id,&district_id); region_id=exprid2region_id(group_rule,region_hit[pos].expr_id,&district_id);
if(region_id<0) if(region_id<0)
{ {
@@ -341,7 +342,7 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi
{ {
if(r_in_c_flag[k]==0) if(r_in_c_flag[k]==0)
{ {
group_rule=(struct _Maat_group_rule_t*)(region_hit[k].tag); group_rule=(struct _Maat_group_inner_t*)(region_hit[k].tag);
hit_detail[j].config_id=-2; hit_detail[j].config_id=-2;
hit_detail[j].hit_region_cnt=1; hit_detail[j].hit_region_cnt=1;
hit_detail[j].region_pos[0].region_id=exprid2region_id(group_rule,region_hit[k].expr_id,&district_id); hit_detail[j].region_pos[0].region_id=exprid2region_id(group_rule,region_hit[k].expr_id,&district_id);

641
src/entry/Maat_command.cpp
View File

@@ -9,18 +9,14 @@ const char* maat_redis_monitor="MAAT_REDIS_MONITOR";
const char* maat_redis_command="MAAT_REDIS_COMMAND"; const char* maat_redis_command="MAAT_REDIS_COMMAND";
const char* rm_key_prefix[2]={"OBSOLETE_RULE","EFFECTIVE_RULE"}; const char* rm_key_prefix[2]={"OBSOLETE_RULE","EFFECTIVE_RULE"};
const char* redis_OK_reply="+OK\r\n";
const char* PREFIX_COMPILE_INDEX="INDEX_COMPILE";
const char* PREFIX_GROUP_INDEX="INDEX_GROUP";
const char* PREFIX_REGION_INDEX="INDEX_REGION";
const int MAAT_REDIS_SYNC_TIME=30*60;
struct serial_rule_t //rm= Redis Maat struct serial_rule_t //rm= Redis Maat
{ {
int op;//0: delete, 1: add. enum MAAT_OPERATION op;//0: delete, 1: add.
int rule_id; int rule_id;
enum MAAT_TABLE_TYPE table_type; char table_name[256];
char* table_name[256];
char* table_line; char* table_line;
}; };
@@ -29,11 +25,9 @@ struct _Maat_cmd_t
struct Maat_command_t cmd; struct Maat_command_t cmd;
enum MAAT_OPERATION op; enum MAAT_OPERATION op;
int ref_cnt; int ref_cnt;
int group_ref_cnt[MAAT_MAX_EXPR_ITEM_NUM];
int group_rgn_cnt[MAAT_MAX_EXPR_ITEM_NUM];
struct _Maat_cmd_t* next; struct _Maat_cmd_t* next;
}; };
enum MAAT_TABLE_TYPE region_table_type(const struct Maat_region_t* p) enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p)
{ {
enum MAAT_TABLE_TYPE ret=0; enum MAAT_TABLE_TYPE ret=0;
switch(p->region_type) switch(p->region_type)
@@ -65,9 +59,76 @@ enum MAAT_TABLE_TYPE region_table_type(const struct Maat_region_t* p)
} }
return ret; return ret;
} }
void free_serial_rules(void* p) void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int size)
{
int ret=0;
switch(p->region_type)
{
case REGION_IP:
ret=snprintf(buff,size,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1"
,p->ip_rule.region_id
,group_id
,p->ip_rule.addr_type
,p->ip_rule.src_ip
,p->ip_rule.mask_src_ip
,p->ip_rule.src_port
,p->ip_rule.mask_src_port
,p->ip_rule.dst_ip
,p->ip_rule.mask_dst_ip
,p->ip_rule.dst_port
,p->ip_rule.mask_dst_port
,p->ip_rule.protocol
,p->ip_rule.direction);
break;
case REGION_EXPR:
if(p->expr_rule.district==NULL)
{
ret=snprintf(buff,size,"%d\t%d\t%s\t%d\t%d\t%d\t1"
,p->region_id
,group_id
,p->expr_rule.keywords
,p->expr_rule.expr_type
,p->expr_rule.match_method
,p->expr_rule.hex_bin);
}
else //expr_plus
{
ret=snprintf(buff,size,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1"
,p->region_id
,group_id
,p->expr_rule.keywords
,p->expr_rule.district
,p->expr_rule.expr_type
,p->expr_rule.match_method
,p->expr_rule.hex_bin);
}
break;
case REGION_INTERVAL:
ret=snprintf(buff,size,"%d\t%d\t%u\t%u\t1"
,p->region_id
,group_id
,p->interval_rule.low_boundary
,p->interval_rule.up_boundary);
break;
case REGION_DIGEST:
ret=snprintf(buff,size,"%d\t%d\t%llu\t%s\t%hd\t1"
,p->region_id
,group_id
,p->digest_rule.orgin_len
,p->digest_rule.digest_string
,p->digest_rule.confidence_degree);
break;
case REGION_SIMILARITY://not support yet
assert(0);
break;
default:
assert(0);
}
assert(ret<size);
return;
}
void free_serial_rules(struct serial_rule_t* rule)
{ {
struct serial_rule_t* rule=(struct serial_rule_t*)p;
if(rule->table_line!=NULL) if(rule->table_line!=NULL)
{ {
free(rule->table_line); free(rule->table_line);
@@ -76,11 +137,10 @@ void free_serial_rules(void* p)
free(rule); free(rule);
return; return;
} }
void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,enum MAAT_TABLE_TYPE table_type,int id,const char* table_name,const char* line) void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,const char* table_name,const char* line)
{ {
rule->op=op; rule->op=op;
rule->rule_id=id; rule->rule_id=rule_id;
rule->table_type=table_type;
assert(srtlen(table_name)<sizeof(rule->table_name)); assert(srtlen(table_name)<sizeof(rule->table_name));
memcpy(rule->table_name,table_name,strlen(table_name)); memcpy(rule->table_name,table_name,strlen(table_name));
if(line!=NULL) if(line!=NULL)
@@ -97,7 +157,7 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
int rule_id; int rule_id;
long long version_in_redis=0; long long version_in_redis=0;
int i=0,ret=0,retry=0; int i=0,ret=0,retry=0;
struct serial_rule_t *p_rm_rule=NULL; struct serial_rule_t *s_rule=NULL;
if(version==0) if(version==0)
{ {
@@ -166,26 +226,26 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
} }
freeReplyObject(data_reply); freeReplyObject(data_reply);
data_reply=NULL; data_reply=NULL;
p_rm_rule=(struct serial_rule_t*)calloc(ctrl_reply->elements,sizeof(struct serial_rule_t)); s_rule=(struct serial_rule_t*)calloc(ctrl_reply->elements,sizeof(struct serial_rule_t));
for(i=0;i<ctrl_reply->elements;i++) for(i=0;i<ctrl_reply->elements;i++)
{ {
assert(ctrl_reply->element[i]->type==REDIS_REPLY_STRING); assert(ctrl_reply->element[i]->type==REDIS_REPLY_STRING);
ret=sscanf(ctrl_reply->element[i]->str,"%s,%s,%d",op_str,p_rm_rule.table_name,&(p_rm_rule[i].rule_id)); ret=sscanf(ctrl_reply->element[i]->str,"%s,%s,%d",op_str,s_rule.table_name,&(s_rule[i].rule_id));
assert(ret==3); assert(ret==3);
if(strncmp(op_str,"ADD")==0) if(strncmp(op_str,"ADD")==0)
{ {
p_rm_rule[i].op=1; s_rule[i].op=MAAT_OP_ADD;
} }
else if(strncmp(op_str,"DEL")==0) else if(strncmp(op_str,"DEL")==0)
{ {
p_rm_rule[i].op=0; s_rule[i].op=MAAT_OP_DEL;
} }
else else
{ {
assert(0); assert(0);
} }
} }
*list=p_rm_rule; *list=s_rule;
*update_type=CM_UPDATE_TYPE_INC; *update_type=CM_UPDATE_TYPE_INC;
freeReplyObject(ctrl_reply); freeReplyObject(ctrl_reply);
return i; return i;
@@ -194,15 +254,15 @@ FULL_UPDATE:
"Initiate full udpate from version %d to %lld.",version,version_in_redis); "Initiate full udpate from version %d to %lld.",version,version_in_redis);
data_reply=(redisReply*)redisCommand(c, "KEYS EFFECTIVE_RULE:*"); data_reply=(redisReply*)redisCommand(c, "KEYS EFFECTIVE_RULE:*");
assert(data_reply->type==REDIS_REPLY_ARRAY); assert(data_reply->type==REDIS_REPLY_ARRAY);
p_rm_rule=(struct serial_rule_t*)calloc(ctrl_reply->elements,sizeof(struct serial_rule_t)); s_rule=(struct serial_rule_t*)calloc(ctrl_reply->elements,sizeof(struct serial_rule_t));
for(i=0;i<data_reply->elements;i++) for(i=0;i<data_reply->elements;i++)
{ {
assert(ctrl_reply->element[i]->type==REDIS_REPLY_STRING); assert(ctrl_reply->element[i]->type==REDIS_REPLY_STRING);
ret=sscanf(ctrl_reply->element[i]->str,"EFFECTIVE_RULE:%s,%d",p_rm_rule[i].table_name,&(p_rm_rule[i].rule_id)); ret=sscanf(ctrl_reply->element[i]->str,"EFFECTIVE_RULE:%s,%d",s_rule[i].table_name,&(s_rule[i].rule_id));
p_rm_rule[i].op=1; s_rule[i].op=1;
assert(ret==2); assert(ret==2);
} }
*list=p_rm_rule; *list=s_rule;
*update_type=CM_UPDATE_TYPE_FULL; *update_type=CM_UPDATE_TYPE_FULL;
freeReplyObject(ctrl_reply); freeReplyObject(ctrl_reply);
ctrl_reply=NULL; ctrl_reply=NULL;
@@ -216,14 +276,16 @@ void redis_monitor_traverse(unsigned int version,redisContext *c,
void (*finish)(void*),//u_para void (*finish)(void*),//u_para
void* u_para, void* u_para,
const unsigned char* dec_key, const unsigned char* dec_key,
void* logger) _Maat_feather_t* feather)
{ {
redisReply* ctrl_reply=NULL,*data_reply=NULL; redisReply* ctrl_reply=NULL,*data_reply=NULL;
int rule_num=0,i=0; int rule_num=0,i=0,table_id=0;
int ret=0; int ret=0;
struct serial_rule_t* rule_list=NULL; struct serial_rule_t* rule_list=NULL;
int update_type=0; int update_type=0;
unsigned int new_version=0; unsigned int new_version=0;
enum MAAT_TABLE_TYPE table_type=0;
void* logger=feather->logger;
rule_num=get_rm_key_list(version, c, &rule_list, logger,&new_version, &update_type); rule_num=get_rm_key_list(version, c, &rule_list, logger,&new_version, &update_type);
if(rule_num==0) if(rule_num==0)
{ {
@@ -241,6 +303,16 @@ void redis_monitor_traverse(unsigned int version,redisContext *c,
start(new_version,update_type); start(new_version,update_type);
for(i=0;i<data_reply->elements;i++) for(i=0;i<data_reply->elements;i++)
{ {
if(rule_list[i].op==MAAT_OP_DEL)
{
ret=feather->map_tablename2id(rule_list[i].table_name,&table_id);
if(ret<0)
{
continue;
}
table_type=feather->p_table_info[table_id].table_type;
invalidate_line(data_reply->element[i].str,table_type);
}
update(rule_list[i].table_name,data_reply->element[i].str,u_para); update(rule_list[i].table_name,data_reply->element[i].str,u_para);
} }
finish(u_para); finish(u_para);
@@ -259,17 +331,9 @@ struct Maat_command_t* Maat_create_comand(const struct Maat_rule_t* rule, int gr
return (struct Maat_command_t*)_cmd; return (struct Maat_command_t*)_cmd;
} }
void Maat_cmd_set_group(struct Maat_command_t* cmd,int which_group,int region_num,const char* table_name,const char* group_name) void Maat_cmd_set_group(struct Maat_command_t* cmd,int which_group,int region_num,const char* group_name)
{ {
assert(which_group<cmd->group_num); assert(which_group<cmd->group_num);
if(table_name!=NULL)
{
if(cmd->groups[which_group].table_name!=NULL)
{
free(cmd->groups[which_group].table_name);
}
cmd->groups[which_group].table_name=_maat_strdup(table_name);
}
if(group_name!=NULL) if(group_name!=NULL)
{ {
if(cmd->groups[which_group].group_name!=NULL) if(cmd->groups[which_group].group_name!=NULL)
@@ -292,10 +356,10 @@ void Maat_copy_region(struct Maat_region_t* dst,const struct Maat_region_t* src)
switch(dst->region_type) switch(dst->region_type)
{ {
case REGION_IP: case REGION_IP:
dst->addr_rule.src_ip=_maat_strdup(src->addr_rule.src_ip); dst->ip_rule.src_ip=_maat_strdup(src->ip_rule.src_ip);
dst->addr_rule.mask_src_ip=_maat_strdup(src->addr_rule.mask_src_ip); dst->ip_rule.mask_src_ip=_maat_strdup(src->ip_rule.mask_src_ip);
dst->addr_rule.dst_ip=_maat_strdup(src->addr_rule.dst_ip); dst->ip_rule.dst_ip=_maat_strdup(src->ip_rule.dst_ip);
dst->addr_rule.mask_src_ip=_maat_strdup(src->addr_rule.mask_src_ip); dst->ip_rule.mask_src_ip=_maat_strdup(src->ip_rule.mask_src_ip);
break; break;
case REGION_EXPR: case REGION_EXPR:
dst->expr_rule.keywords=_maat_strdup(src->expr_rule.keywords); dst->expr_rule.keywords=_maat_strdup(src->expr_rule.keywords);
@@ -321,10 +385,10 @@ void Maat_empty_region(struct Maat_region_t* p)
switch(p->region_type) switch(p->region_type)
{ {
case REGION_IP: case REGION_IP:
free(p->addr_rule.src_ip); free(p->ip_rule.src_ip);
free(p->addr_rule.mask_src_ip); free(p->ip_rule.mask_src_ip);
free(p->addr_rule.dst_ip); free(p->ip_rule.dst_ip);
free(p->addr_rule.mask_src_ip); free(p->ip_rule.mask_src_ip);
break; break;
case REGION_EXPR: case REGION_EXPR:
free(p->expr_rule.keywords); free(p->expr_rule.keywords);
@@ -371,7 +435,6 @@ void Maat_free_command(struct Maat_command_t* cmd)
{ {
Maat_empty_region(&(cmd.groups[i].regions[j])); Maat_empty_region(&(cmd.groups[i].regions[j]));
} }
free(cmd.groups[i].table_name);
free(cmd.groups[i].group_name); free(cmd.groups[i].group_name);
free(cmd.groups[i].regions); free(cmd.groups[i].regions);
cmd.groups[i].regions=NULL; cmd.groups[i].regions=NULL;
@@ -384,7 +447,8 @@ void Maat_free_command(struct Maat_command_t* cmd)
} }
int Maat_format_command(struct Maat_command_t* rule, char* buffer, int size) int Maat_format_command(struct Maat_command_t* rule, char* buffer, int size)
{ {
//TODO
return 0;
} }
int Maat_command(Maat_feather_t feather,struct Maat_command_t* raw_rule,enum MAAT_OPERATION op) int Maat_command(Maat_feather_t feather,struct Maat_command_t* raw_rule,enum MAAT_OPERATION op)
{ {
@@ -397,28 +461,26 @@ int Maat_command(Maat_feather_t feather,struct Maat_command_t* raw_rule,enum MAA
ret=Maat_commit_command(feather); ret=Maat_commit_command(feather);
return ret; return ret;
} }
//functioned as strdup, for dictator compatible.
char* _maat_strdup(const char* s) int Maat_append_command(Maat_feather_t feather,struct Maat_command_t* cmd,enum MAAT_OPERATION op)
{
char*d=NULL;
if(s==NULL)
{
return NULL;
}
d=(char*)malloc(strlen(s)+1);
memcpy(d,s,strlen(s)+1));
return d;
}
void Maat_append_command(Maat_feather_t feather,struct Maat_command_t* cmd,enum MAAT_OPERATION op)
{ {
_Maat_feather_t* _feather=(_Maat_feather_t*)feather; _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_cmd_t* _cmd=(struct _Maat_cmd_t*)cmd; struct _Maat_cmd_t* _cmd=(struct _Maat_cmd_t*)cmd;
int i=0, ret=0;
_cmd->ref_cnt++; _cmd->ref_cnt++;
_cmd->op=op; _cmd->op=op;
assert(op==MAAT_OP_DEL||op==MAAT_OP_ADD); assert(op==MAAT_OP_DEL||op==MAAT_OP_ADD);
if(cmd->table_name==NULL) if(op==MAAT_OP_DEL)
{ {
cmd->table_name=_maat_strdup(_feather->compile_tn); ret=reconstruct_cmd(_feather, _cmd)
}
else
{
ret=fix_table_name(_feather, cmd);
}
if(ret<0)
{
return -1;
} }
if(_feather->cmd_num==0) if(_feather->cmd_num==0)
{ {
@@ -483,104 +545,89 @@ void invalidate_line(char* line, enum MAAT_TABLE_TYPE type)
line[i]='0'; line[i]='0';
return; return;
} }
int calculate_serial_rule_num(struct Maat_command_t* cmd) int calculate_serial_rule_num(struct _Maat_cmd_t* _cmd,int * new_region_cnt, int* new_group_cnt)
{ {
int serial_num=0; int serial_num=0;
int i=0,j=0; int i=0;
struct Maat_command_t* cmd=&(_cmd->cmd);
serial_num++;//compile rule serial_num++;//compile rule
for(i=0;i<cmd->group_num;i++) for(i=0;i<cmd->group_num;i++)
{ {
serial_num++; serial_num++;
for(j=0;j<cmd->groups[i].region_num;j++) if(cmd->groups[i].regions==NULL)
{ {
serial_num++; continue;
}
if(_cmd->op==MAAT_OP_ADD)
{
*new_region_cnt+=cmd->groups[i].region_num;
*new_group_cnt++;
} }
} }
return serial_num; return serial_num;
} }
int reconstruct_cmd(redisContext *ctx,struct _Maat_cmd_t* _cmd,void* logger) int reconstruct_cmd(struct _Maat_feather_t feather, struct _Maat_cmd_t* _cmd)
{ {
int redis_ret=REDIS_ERR,ret=0; int ret=0;
int i=0,j=0,grp_idx=0; int i=0,j=0,grp_idx=0;
redisReply* compile_reply=NULL,*group_reply=NULL,*region_reply=NULL;
long long group_ref_cnt=0; long long group_ref_cnt=0;
char table_name[MAX_TABLE_NAME_LEN];
struct Maat_command_t* cmd=&(_cmd->cmd); struct Maat_command_t* cmd=&(_cmd->cmd);
struct Maat_group_t* p_group=NULL; struct Maat_group_t* group_cmd=NULL;
struct Maat_region_t* p_region=NULL; struct Maat_region_t* region_cmd=NULL;
struct _Maat_compile_inner_t *compile_inner=NULL;
struct _Maat_group_inner_t* group_inner=NULL;
struct _Maat_region_inner_t* region_inner=NULL;
void* logger=feather->logger;
int config_id=cmd->compile.config_id; int config_id=cmd->compile.config_id;
compile_reply=redisCommand(ctx,"HKEYS %s:%d",PREFIX_COMPILE_INDEX,config_id); compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id);
if(compile_reply==NULL) if(compile_inner==NULL)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_command MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_command
,"%s:%d not found." ,"config %d not exist."
,PREFIX_COMPILE_INDEX,config_id); ,config_id);
return -1; return -1;
} }
cmd->group_num=compile_reply->elements; cmd->group_num=compile_inner->group_cnt;
assert(cmd->compile.declare_grp_num==cmd->group_num);
assert(cmd->groups==NULL); assert(cmd->groups==NULL);
cmd->groups=calloc(sizeof(struct Maat_group_t),cmd->group_num); cmd->groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),cmd->group_num);
for(i=0;i<compile_reply->elements;i++) for(i=0;i<compile_inner->group_boundary;i++)
{ {
if(strncmp(compile_reply->element[i].str,"GROUP:",strlen("GROUP:"))!=0) group_inner=(struct _Maat_group_inner_t*)dynamic_array_read(compile_inner->groups,i);
if(group_inner==NULL)
{ {
continue; continue;
} }
p_group=&(cmd->groups[grp_idx]); group_cmd=&(cmd->groups[grp_idx]);
ret=sscanf(compile_reply->element[i].str,"GROUP:%s,%d",table_name,&(p_group->group_id)); group_cmd->group_id=group_inner->group_id;
assert(ret==2);
p_group->table_name=_maat_strdup(table_name);
group_reply=redisCommand(ctx,"HINCRBY %s:%s,%d REF_CNT -1",PREFIX_GROUP_INDEX if(group_inner->ref_cnt>0)
,p_group->table_name
,p_group->group_id);
freeReplyObject(group_reply);
group_reply=redisCommand(ctx,"HGET %s:%s,%d REF_CNT",PREFIX_GROUP_INDEX
,p_group->table_name
,p_group->group_id);
group_ref_cnt=group_reply->integer;
freeReplyObject(group_reply);
_cmd->group_ref_cnt[grp_idx]=group_ref_cnt;
grp_idx++;
if(group_ref_cnt>0)
{ {
continue; continue;
} }
group_reply=redisCommand(ctx,"HKEYS %s:%s,%d",PREFIX_GROUP_INDEX group_cmd->region_num=group_inner->region_cnt;
,p_group->table_name group_cmd->regions=(struct Maat_region_t*)calloc(sizeof(struct Maat_region_t),group_cmd->region_num);
,p_group->group_id); for(j=0;j<group_inner->region_boundary;j++)
p_group->region_num=group_reply->elements;
p_group->regions=(struct Maat_region_t*)calloc(sizeof(struct Maat_region_t),p_group->region_num);
for(j=0;j<group_reply->elements;j++)
{ {
if(strncmp(group_reply->element[j].str,"REGION:",strlen("REGION:"))!=0) region_inner=(struct _Maat_region_inner_t*)dynamic_array_read(group_inner->regions,i);
if(region_inner==NULL)
{ {
continue; continue;
} }
p_region=&(p_group->regions[p_group->region_num]); region_cmd=&(group_cmd->regions[group_cmd->region_num]);
ret=sscanf(group_reply->element[j].str,"REGION:%s,%d",table_name,&(p_region->region_id)); region_cmd->table_name=_maat_strdup(feather->p_table_info[region_inner->table_id].table_name[0]);
assert(ret==2); region_cmd->region_id=region_inner->table_id;
p_region->table_name=_maat_strdup(table_name); //NOTICE: a table_type stored in region_type.
region_reply=redisCommand(ctx,"HGET %s %s,%d",PREFIX_REGION_INDEX region_cmd->region_type=region_inner->table_type;
,p_region->table_name group_cmd->region_num++;
,p_region->region_id);
p_region->region_type=region_reply->integer;
p_group->region_num++;
freeReplyObject(region_reply);
} }
freeReplyObject(group_reply); grp_idx++;
group_reply=NULL;
} }
freeReplyObject(compile_reply);
compile_reply=NULL;
return 0; return 0;
} }
int build_serial_rule_from_redis(redisContext *ctx,struct Maat_command_t* cmd,struct serial_rule_t* list, int size) int build_serial_rule_from_redis(_Maat_feather_t *feather,redisContext *ctx,struct Maat_command_t* cmd,struct serial_rule_t* list, int size)
{ {
struct Maat_group_t* p_group=NULL; struct Maat_group_t* p_group=NULL;
struct Maat_region_t* p_region=NULL; struct Maat_region_t* p_region=NULL;
@@ -588,21 +635,14 @@ int build_serial_rule_from_redis(redisContext *ctx,struct Maat_command_t* cmd,st
redisReply* data_reply=NULL; redisReply* data_reply=NULL;
int rule_num=0; int rule_num=0;
int i=0,j=0; int i=0,j=0;
data_reply=redisCommand(ctx,"GET %s:%s,%d",rm_key_prefix[MAAT_OP_ADD]
,cmd->table_name set_serial_rule(list+rule_num,MAAT_OP_DEL,cmd->compile.config_id,feather->compile_tn,NULL);
,cmd->compile.config_id);
invalidate_line(data_reply->str, TABLE_TYPE_COMPILE);
set_serial_rule(list+rule_num,MAAT_OP_DEL,TABLE_TYPE_COMPILE,cmd->compile.config_id,cmd->table_name,data_reply->str);
rule_num++; rule_num++;
freeReplyObject(data_reply);
for(i=0;i<cmd->group_num;i++) for(i=0;i<cmd->group_num;i++)
{ {
p_group=&(cmd->groups[i]); p_group=&(cmd->groups[i]);
data_reply=redisCommand(ctx,"GET %s:%s,%d",rm_key_prefix[MAAT_OP_ADD]
,p_group->table_name set_serial_rule(list+rule_num,MAAT_OP_DEL,p_group->group_id,feather->group_tn,NULL);
,p_group->group_id);
invalidate_line(data_reply->str, TABLE_TYPE_GROUP);
set_serial_rule(list+rule_num,MAAT_OP_DEL,TABLE_TYPE_GROUP,p_group->group_id,p_group->table_name,data_reply->str);
rule_num++; rule_num++;
freeReplyObject(data_reply); freeReplyObject(data_reply);
for(j=0;j<p_group->region_num;j++) for(j=0;j<p_group->region_num;j++)
@@ -611,38 +651,87 @@ int build_serial_rule_from_redis(redisContext *ctx,struct Maat_command_t* cmd,st
data_reply=redisCommand(ctx,"GET %s:%s,%d",rm_key_prefix[MAAT_OP_ADD] data_reply=redisCommand(ctx,"GET %s:%s,%d",rm_key_prefix[MAAT_OP_ADD]
,p_region->table_name ,p_region->table_name
,p_region->region_id); ,p_region->region_id);
table_type=region_table_type(p_region); table_type=type_region2table(p_region);
invalidate_line(data_reply->str, table_type); invalidate_line(data_reply->str, (enum MAAT_TABLE_TYPE)(p_region->region_type));
set_serial_rule(list+rule_num,MAAT_OP_DEL,p_region->region_id,p_region->table_name,data_reply->str); set_serial_rule(list+rule_num,MAAT_OP_DEL,p_region->region_id,p_region->table_name,data_reply->str);
rule_num++; rule_num++;
freeReplyObject(data_reply); freeReplyObject(data_reply);
} }
} }
assert(rule_num<size); assert(rule_num<size);
return 0; return rule_num;
} }
int build_serial_rule_from_cmd(struct Maat_command_t* cmd,struct serial_rule_t* list, int size) int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_t* _cmd,struct serial_rule_t* list, int size)
{ {
struct Maat_group_t* p_group=NULL; struct Maat_group_t* p_group=NULL;
struct Maat_region_t* p_region=NULL; struct Maat_region_t* p_region=NULL;
struct Maat_rule_t* p_m_rule=NULL; struct Maat_rule_t* p_m_rule=NULL;
struct Maat_command_t* cmd=&(_cmd->cmd);
enum MAAT_OPERATION op=_cmd->op;
redisContext* ctx=feather->redis_write_ctx;
redisReply* data_reply=NULL; redisReply* data_reply=NULL;
int rule_num=0,i=0; int rule_num=0,i=0,j=0;
p_m_rule=&(cmd->compile); p_m_rule=&(cmd->compile);
char line[1024]; char line[1024];
snprintf(line,sizeof(line),"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t1\t%d",p_m_rule->config_id if(op===MAAT_OP_ADD)
,p_m_rule->service_id {
,p_m_rule->action snprintf(line,sizeof(line),"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t1\t%d",p_m_rule->config_id
,p_m_rule->do_blacklist ,p_m_rule->service_id
,p_m_rule->do_log ,p_m_rule->action
,p_m_rule->service_defined ,p_m_rule->do_blacklist
,p_m_rule->declare_grp_num); ,p_m_rule->do_log
set_serial_rule(list+rule_num,MAAT_OP_ADD,cmd->compile.config_id,cmd->table_name,line); ,p_m_rule->service_defined
,cmd->group_num);
set_serial_rule(list+rule_num,MAAT_OP_ADD,cmd->compile.config_id,feather->compile_tn,line);
}
else
{
set_serial_rule(list+rule_num,MAAT_OP_DEL,cmd->compile.config_id,feather->compile_tn,NULL);
}
rule_num++; rule_num++;
data_reply=redisCommand(ctx,"GET %s:%s,%d",rm_key_prefix[MAAT_OP_ADD] for(i=0;cmd->group_num;i++)
,cmd->table_name {
,cmd->compile.config_id); p_group=&(cmd->groups[i]);
return 0; if(op===MAAT_OP_ADD)
{
p_group->group_id=feather->base_grp_seq;
feather->base_grp_seq++;
snprintf(line,sizeof(line),"%d\t%d\t1",p_group->group_id
,p_m_rule->config_id);
set_serial_rule(list+rule_num,MAAT_OP_ADD,p_group->group_id,feather->group_tn,line);
}
else
{
set_serial_rule(list+rule_num,MAAT_OP_DEL,p_group->group_id,feather->group_tn,NULL);
}
rule_num++;
if(p_group->regions==NULL)//group reuse.
{
continue;
}
for(j=0;j<p_group->region_num;j++)
{
if(op===MAAT_OP_ADD)
{
p_group->regions[j].region_id=feather->base_rgn_seq;
feather->base_rgn_seq++;
serialize_region(p_group->regions+j, p_group->group_id, line, sizeof(line));
set_serial_rule(list+rule_num,MAAT_OP_ADD
,p_group->regions[j].region_id,p_group->regions[j].table_name,line);
}
else
{
set_serial_rule(list+rule_num,MAAT_OP_DEL
,p_group->regions[j].region_id,p_group->regions[j].table_name,NULL);
}
rule_num++;
}
}
assert(rule_num<size);
return rule_num;
} }
int mr_transaction_success(redisReply* data_reply) int mr_transaction_success(redisReply* data_reply)
{ {
@@ -656,142 +745,43 @@ int mr_transaction_success(redisReply* data_reply)
} }
return 1; return 1;
} }
const int MAAT_REDIS_SYNC_TIME=30*60; int fix_table_name(_Maat_feather_t* feather,struct Maat_command_t* cmd)
int del_cmd_from_redis(redisContext *ctx,struct _Maat_cmd_t* _cmd,void* logger)
{ {
int serial_num=0; int i=0,j=0,ret=0;
int i=0; char *table_name=NULL;
int retry=0,ret=0; int table_id=0;
long long maat_redis_version=0,group_num=0; enum MAAT_TABLE_TYPE table_type;
struct serial_rule_t *s_rule=NULL; for(i=0;i<cmd->group_num;i++)
struct Maat_command_t* cmd=&(_cmd->cmd);
int redis_ret=REDIS_ERR,ret=0;
redisReply* data_reply=NULL;
reconstruct_cmd(ctx, _cmd, logger);
serial_num=calculate_serial_rule_num(cmd);
s_rule=(struct serial_rule_t*)calloc(sizeof(struct serial_rule_t),serial_num);
ret=build_serial_rule_from_redis(ctx, cmd, s_rule, serial_num);
retry=0;
while(1)
{ {
for(j=0;j<cmd->groups[i].region_num;j++)
data_reply=redisCommand(ctx, "WATCH MAAT_VERSION");
freeReplyObject(data_reply);
data_reply=redisCommand(ctx, "GET MAAT_VERSION");
freeReplyObject(data_reply);
maat_redis_version=data_reply->integer;
maat_redis_version++;
data_reply=redisCommand(ctx,"MULTI");
freeReplyObject(data_reply);
for(i=0;i<serial_num;i++)
{ {
data_reply=redisCommand(ctx,"DEL %s:%s,%d",rm_key_prefix[MAAT_OP_ADD] table_name=cmd->groups[i].regions[j].table_name;
,s_rule[i].table_name ret=map_str2int(feather->map_tablename2id, table_name, &table_id);
,s_rule[i].rule_id); if(ret<0)
freeReplyObject(data_reply); {
MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module
,"Unknown table %s of Maat_command_t[%d]->group[%d]->region[%d]."
,table_name,cmd->compile.config_id,i,j);
data_reply=redisCommand(ctx,"SET %s:%s,%d \"%s\"",rm_key_prefix[MAAT_OP_DEL] return -1;
,s_rule[i].table_name }
,s_rule[i].rule_id table_type=type_region2table(&(cmd->groups[i].regions[j]));
,s_rule[i].table_line); if(table_type!=feather->p_table_info[i].table_type)
freeReplyObject(data_reply); {
data_reply=redisCommand(ctx,"EXPIRE %s:%s,%d %d",rm_key_prefix[MAAT_OP_DEL] MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module
,s_rule[i].table_name ,"Table %s not support region type %d of Maat_command_t[%d]->group[%d]->region[%d]."
,s_rule[i].rule_id ,table_name
,MAAT_REDIS_SYNC_TIME); ,cmd->groups[i].regions[j].region_type
freeReplyObject(data_reply); ,cmd->compile.config_id,i,j);
//NX: Don't update already exisiting elements. Always add new elements. return -1;
data_reply=redisCommand(ctx,"ZADD NX DEL,%s,%d %d",s_rule[i].table_name }
,s_rule[i].rule_id free(cmd->groups[i].regions[j].table_name);
,maat_redis_version); cmd->groups[i].regions[j].table_name=_maat_strdup(feather->p_table_info[i].table_name[0]);
freeReplyObject(data_reply);
data_reply=redisCommand(ctx,"INCRBY MAAT_VERSION 1");
freeReplyObject(data_reply);
}
data_reply=redisCommand(ctx,"EXEC");
if(1==mr_transaction_success(data_reply))
{
freeReplyObject(data_reply);
break;
}
else
{
retry++;
assert(retry<5);
freeReplyObject(data_reply);
} }
} }
for(i=0;i<serial_num;i++) return 0;
{
free_serial_rules(s_rule+i);
s_rule[i]=NULL;
}
free(s_rule);
} }
int add_cmd_to_redis(redisContext *ctx,struct _Maat_cmd_t* _cmd,void* logger)
{
int ret=0,retry=0,i=0;
redisReply* data_reply=NULL;
long long maat_redis_version=0;
struct Maat_command_t* cmd=&(_cmd->cmd);
struct serial_rule_t *s_rule=NULL;
int serial_num=calculate_serial_rule_num(cmd);
s_rule=(struct serial_rule_t*)calloc(sizeof(struct serial_rule_t),serial_num);
ret=build_serial_rule_from_cmd(ctx, cmd, s_rule, serial_num);
retry=0;
while(1)
{
data_reply=redisCommand(ctx, "WATCH MAAT_VERSION");
freeReplyObject(data_reply);
data_reply=redisCommand(ctx, "GET MAAT_VERSION");
freeReplyObject(data_reply);
maat_redis_version=data_reply->integer;
maat_redis_version++;
data_reply=redisCommand(ctx,"MULTI");
freeReplyObject(data_reply);
for(i=0;i<serial_num;i++)
{
data_reply=redisCommand(ctx,"SET %s:%s,%d \"%s\"",rm_key_prefix[MAAT_OP_ADD]
,s_rule[i].table_name
,s_rule[i].rule_id
,s_rule[i].table_line);
freeReplyObject(data_reply);
//NX: Don't update already exisiting elements. Always add new elements.
data_reply=redisCommand(ctx,"ZADD NX ADD,%s,%d %d",s_rule[i].table_name
,s_rule[i].rule_id
,maat_redis_version);
freeReplyObject(data_reply);
}
data_reply=redisCommand(ctx,"EXEC");
if(1==mr_transaction_success(data_reply))
{
freeReplyObject(data_reply);
break;
}
else
{
retry++;
assert(retry<5);
freeReplyObject(data_reply);
}
}
for(i=0;i<serial_num;i++)
{
free_serial_rules(s_rule+i);
s_rule[i]=NULL;
}
free(s_rule);
}
int Maat_commit_command(Maat_feather_t feather) int Maat_commit_command(Maat_feather_t feather)
{ {
_Maat_feather_t* _feather=(_Maat_feather_t*)feather; _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
@@ -802,8 +792,15 @@ int Maat_commit_command(Maat_feather_t feather)
timeout.tv_sec=0; timeout.tv_sec=0;
timeout.tv_usec=100*1000; // 100 ms timeout.tv_usec=100*1000; // 100 ms
int ret=0,i=0,redis_ret=REDIS_ERR,retry=0; int ret=0,i=0,redis_ret=REDIS_ERR,retry=0;
long long maat_redis_version=0,region_seq=0,group_seq=0; long long maat_redis_version=0,new_region_num=0,new_group_num=0;
int serial_rule_num=0,serial_rule_idx=0;
struct _Maat_cmd_t* p=NULL,*n=NULL; struct _Maat_cmd_t* p=NULL,*n=NULL;
redisContext* ctx=NULL;
redisReply* data_reply=NULL;
struct serial_rule_t* s_rule=NULL;
if(_feather->redis_write_ctx==NULL) if(_feather->redis_write_ctx==NULL)
{ {
_feather->redis_write_ctx=redisConnectWithTimeout(_feather->redis_ip, _feather->redis_port,timeout); _feather->redis_write_ctx=redisConnectWithTimeout(_feather->redis_ip, _feather->redis_port,timeout);
@@ -816,19 +813,94 @@ int Maat_commit_command(Maat_feather_t feather)
goto error_out; goto error_out;
} }
} }
p=feather->cmd_qhead; ctx=_feather->redis_write_ctx;
for(i=0;i<feather->cmd_num;i++)
for(i=0,p=feather->cmd_qhead;i<feather->cmd_num;i++)
{ {
p=p->next; p=p->next;
if(p->op==MAAT_OP_DEL) serial_rule_num+=calculate_serial_rule_num(&(p->cmd), &new_region_num, &new_group_num);
}
data_reply=redisCommand(ctx,"INCRBY SEQUENCE_REGION %d",new_region_num);
_feather->base_rgn_seq=data_reply->integer-new_region_num;
freeReplyObject(data_reply);
data_reply=redisCommand(ctx,"INCRBY SEQUENCE_GROUP %d",new_group_num);
_feather->base_rgn_seq=data_reply->integer-new_group_num;
freeReplyObject(data_reply);
s_rule=(struct serial_rule_t*)calloc(sizeof(struct serial_rule_t),serial_rule_num);
for(i=0,p=feather->cmd_qhead;i<feather->cmd_num;i++)
{
p=p->next;
serial_rule_idx+=build_serial_rule(feather,p,s_rule, serial_rule_num-serial_rule_idx);
}
assert(serial_rule_idx==serial_rule_num);
while(1)
{
data_reply=redisCommand(ctx, "WATCH MAAT_VERSION");
freeReplyObject(data_reply);
data_reply=redisCommand(ctx, "GET MAAT_VERSION");
freeReplyObject(data_reply);
maat_redis_version=data_reply->integer;
maat_redis_version++;
data_reply=redisCommand(ctx,"MULTI");
freeReplyObject(data_reply);
for(i=0;i<serial_rule_num;i++)
{ {
del_cmd_from_redis(_feather->redis_write_ctx, p, _feather->logger); if(s_rule[i].op==MAAT_OP_ADD)
{
redisAppendCommand(ctx,"SET %s:%s,%d \"%s\"",rm_key_prefix[MAAT_OP_ADD]
,s_rule[i].table_name
,s_rule[i].rule_id
,s_rule[i].table_line);
//NX: Don't update already exisiting elements. Always add new elements.
redisAppendCommand(ctx,"ZADD NX ADD,%s,%d %d",s_rule[i].table_name
,s_rule[i].rule_id
,maat_redis_version);
}
else
{
redisAppendCommand(ctx,"RENAME %s:%s,%d %s:%s,%d"
,rm_key_prefix[MAAT_OP_ADD]
,s_rule[i].table_name
,s_rule[i].rule_id
,rm_key_prefix[MAAT_OP_DEL]
,s_rule[i].table_name
,s_rule[i].rule_id
);
redisAppendCommand(ctx,"EXPIRE %s:%s,%d %d",rm_key_prefix[MAAT_OP_DEL]
,s_rule[i].table_name
,s_rule[i].rule_id
,MAAT_REDIS_SYNC_TIME);
//NX: Don't update already exisiting elements. Always add new elements.
redisAppendCommand(ctx,"ZADD NX DEL,%s,%d %d",s_rule[i].table_name
,s_rule[i].rule_id
,maat_redis_version);
}
}
redisAppendCommand(ctx,"INCRBY MAAT_VERSION 1");
redisAppendCommand(ctx,"EXEC");
redisGetReply(ctx, &data_reply);
if(1==mr_transaction_success(data_reply))
{
freeReplyObject(data_reply);
break;
} }
else else
{ {
add_cmd_to_redis(_feather->redis_write_ctx, p, _feather->logger); retry++;
assert(retry<5);
freeReplyObject(data_reply);
} }
} }
error_out: error_out:
p=_feather->cmd_qhead; p=_feather->cmd_qhead;
for(i=0;i<_feather->cmd_num;i++) for(i=0;i<_feather->cmd_num;i++)
@@ -839,6 +911,13 @@ error_out:
} }
_feather->cmd_qhead=_feather->cmd_qtail=NULL; _feather->cmd_qhead=_feather->cmd_qtail=NULL;
_feather->cmd_num=0; _feather->cmd_num=0;
for(i=0;i<serial_rule_num;i++)
{
free_serial_rules(s_rule+i);
s_rule[i]=NULL;
}
free(s_rule);
return ret; return ret;
} }

331
src/entry/Maat_rule.cpp
View File

@@ -62,6 +62,18 @@ int hex2bin(char *hex,int hex_len,char *binary,int size)
binary[resultlen]='\0'; binary[resultlen]='\0';
return resultlen; return resultlen;
} }
//functioned as strdup, for dictator compatible.
char* _maat_strdup(const char* s)
{
char*d=NULL;
if(s==NULL)
{
return NULL;
}
d=(char*)malloc(strlen(s)+1);
memcpy(d,s,strlen(s)+1));
return d;
}
int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type) int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type)
{ {
switch(expr_type) switch(expr_type)
@@ -622,19 +634,21 @@ error_jump:
map_destroy(string2int_map); map_destroy(string2int_map);
return table_cnt; return table_cnt;
} }
struct _Maat_group_rule_t* create_group_rule(int group_id) struct _Maat_group_inner_t* create_group_rule(int group_id)
{ {
struct _Maat_group_rule_t* group=(struct _Maat_group_rule_t*)malloc(sizeof(struct _Maat_group_rule_t)); struct _Maat_group_inner_t* group=(struct _Maat_group_inner_t*)malloc(sizeof(struct _Maat_group_inner_t));
group->group_id=group_id; group->group_id=group_id;
group->region_cnt=0; group->region_cnt=0;
group->region_boundary=0; group->region_boundary=0;
group->ref_cnt=0; group->ref_cnt=0;
group->region_rules=dynamic_array_create(1,8); group->regions=dynamic_array_create(1,8);
group->compile_shortcut=NULL; group->compile_shortcut=NULL;
group->table_id=0;
group->group_name=NULL;
pthread_mutex_init(&(group->mutex), NULL); pthread_mutex_init(&(group->mutex), NULL);
return group; return group;
} }
void destroy_group_rule(struct _Maat_group_rule_t* group) void destroy_group_rule(struct _Maat_group_inner_t* group)
{ {
if(group->ref_cnt>0||group->region_cnt>0) if(group->ref_cnt>0||group->region_cnt>0)
@@ -642,25 +656,28 @@ void destroy_group_rule(struct _Maat_group_rule_t* group)
return; return;
} }
dynamic_array_destroy(group->region_rules,free); dynamic_array_destroy(group->regions,free);
group->region_cnt=0; group->region_cnt=0;
group->region_boundary=0; group->region_boundary=0;
group->region_rules=NULL; group->regions=NULL;
group->ref_cnt=0; group->ref_cnt=0;
group->group_id=-1; group->group_id=-1;
group->table_id=-1;
free(group->group_name);
group->group_name=NULL;
pthread_mutex_destroy(&(group->mutex)); pthread_mutex_destroy(&(group->mutex));
free(group); free(group);
} }
void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bool_expr_t* a_set) void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set)
{ {
int i=0,j=0; int i=0,j=0;
a_set->bool_expr_id=(void*)compile_rule; a_set->bool_expr_id=(void*)compile_rule;
struct _Maat_group_rule_t*group=NULL; struct _Maat_group_inner_t*group=NULL;
assert(MAAT_MAX_EXPR_ITEM_NUM<=MAX_ITEMS_PER_BOOL_EXPR); assert(compile_rule->group_cnt<=MAX_ITEMS_PER_BOOL_EXPR);
for(i=0,j=0;i<MAAT_MAX_EXPR_ITEM_NUM&&j<MAX_ITEMS_PER_BOOL_EXPR;i++) for(i=0,j=0;i<compile_rule->group_boundary&&j<MAX_ITEMS_PER_BOOL_EXPR;i++)
{ {
group=(struct _Maat_group_rule_t*)dynamic_array_read(compile_rule->groups,i); group=(struct _Maat_group_inner_t*)dynamic_array_read(compile_rule->groups,i);
if(group==NULL) if(group==NULL)
{ {
continue; continue;
@@ -674,7 +691,7 @@ void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bo
void walk_compile_hash(const uchar * key, uint size, void * data, void * user) void walk_compile_hash(const uchar * key, uint size, void * data, void * user)
{ {
universal_bool_expr_t* one_set=NULL; universal_bool_expr_t* one_set=NULL;
struct _Maat_compile_rule_t* compile_rule=(struct _Maat_compile_rule_t*)data; struct _Maat_compile_inner_t* compile_rule=(struct _Maat_compile_inner_t*)data;
MESA_lqueue_head update_q=(MESA_lqueue_head)user; MESA_lqueue_head update_q=(MESA_lqueue_head)user;
if(compile_rule->db_c_rule==NULL) if(compile_rule->db_c_rule==NULL)
{ {
@@ -746,24 +763,26 @@ void EMPTY_FREE(void*p)
{ {
return; return;
} }
struct _Maat_compile_rule_t * create_compile_rule(int compile_id) struct _Maat_compile_inner_t * create_compile_rule(int compile_id)
{ {
struct _Maat_compile_rule_t* p=NULL; struct _Maat_compile_inner_t* p=NULL;
p=(struct _Maat_compile_rule_t*)calloc(sizeof(struct _Maat_compile_rule_t),1); p=(struct _Maat_compile_inner_t*)calloc(sizeof(struct _Maat_compile_inner_t),1);
p->compile_id=compile_id; p->compile_id=compile_id;
p->group_cnt=0; p->group_cnt=0;
p->table_id=0;
p->group_boundary=1;
p->groups=dynamic_array_create(1, 1); p->groups=dynamic_array_create(1, 1);
pthread_rwlock_init(&(p->rwlock), NULL); pthread_rwlock_init(&(p->rwlock), NULL);
return p; return p;
} }
void destroy_compile_rule(struct _Maat_compile_rule_t * p) void destroy_compile_rule(struct _Maat_compile_inner_t * p)
{ {
int i=0; int i=0;
struct _Maat_compile_rule_t* p_group=NULL; struct _Maat_compile_inner_t* p_group=NULL;
assert(p->group_cnt==0); assert(p->group_cnt==0);
for(i=0;i<MAAT_MAX_EXPR_ITEM_NUM;i++) for(i=0;i<p->group_boundary;i++)
{ {
p_group=(struct _Maat_compile_rule_t*)dynamic_array_read(p->groups,i); p_group=(struct _Maat_compile_inner_t*)dynamic_array_read(p->groups,i);
assert(p_group==NULL); assert(p_group==NULL);
} }
p->compile_id=-1; p->compile_id=-1;
@@ -901,7 +920,7 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule)
return; return;
} }
GIE_digest_t* create_digest_rule(int id,short op,unsigned long long origin_len,const char* digest, GIE_digest_t* create_digest_rule(int id,short op,unsigned long long origin_len,const char* digest,
short cfds_lvl,struct _Maat_group_rule_t* tag) short cfds_lvl,struct _Maat_group_inner_t* tag)
{ {
GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1); GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1);
int digest_len=strlen(digest); int digest_len=strlen(digest);
@@ -1293,133 +1312,176 @@ void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*log
update_array=NULL; update_array=NULL;
return; return;
} }
struct _Maat_group_rule_t* add_region_to_group(struct _Maat_group_rule_t* group,int region_id,int district_id,int expr_id,enum MAAT_TABLE_TYPE region_type) struct _Maat_group_inner_t* add_region_to_group(struct _Maat_group_inner_t* group,int table_id,int region_id,int district_id,int expr_id,enum MAAT_TABLE_TYPE region_type)
{ {
struct _Maat_region_rule_t* region_rule=(struct _Maat_region_rule_t*)malloc(sizeof(struct _Maat_region_rule_t)); int i=0;
region_rule->region_id=region_id; struct _Maat_region_inner_t* region_rule=NULL;
region_rule->expr_id=expr_id;
region_rule->district_id=district_id;
region_rule->region_type=region_type;
pthread_mutex_lock(&(group->mutex));
dynamic_array_write(group->region_rules,group->region_boundary,region_rule);
group->region_cnt++;
group->region_boundary++;
pthread_mutex_unlock(&(group->mutex));
return group;
}
void cancel_last_region_from_group(struct _Maat_group_rule_t* group,int region_id,int expr_id)
{
struct _Maat_region_rule_t* region_rule=NULL;
pthread_mutex_lock(&(group->mutex));
group->region_boundary--;
region_rule=(struct _Maat_region_rule_t*)dynamic_array_read(group->region_rules,group->region_boundary);
assert(region_rule->expr_id==expr_id&&region_rule->region_id==region_id);
free(region_rule);
dynamic_array_write(group->region_rules,group->region_boundary,NULL);
group->region_cnt--;
pthread_mutex_unlock(&(group->mutex));
return;
}
unsigned int del_region_from_group(struct _Maat_group_rule_t* group,int region_id,unsigned int *output_expr_id,int output_size)
{
int i=0,j=0;
struct _Maat_region_rule_t* region_rule=NULL;
pthread_mutex_lock(&(group->mutex));
for(i=0;i<group->region_boundary;i++) for(i=0;i<group->region_boundary;i++)
{ {
region_rule=(struct _Maat_region_rule_t*)dynamic_array_read(group->region_rules, i); region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group->regions, i);
if(region_rule==NULL) if(region_rule==NULL)
{ {
continue; continue;
} }
if(region_rule->region_id==region_id) if(region_rule->region_id==region_id)
{ {
dynamic_array_write(group->region_rules, i, NULL); break;
output_expr_id[j]=region_rule->expr_id; }
j++; }
if(i==group->region_boundary)//new region
{
region_rule=(struct _Maat_region_inner_t*)malloc(sizeof(struct _Maat_region_inner_t));
region_rule->region_id=region_id;
region_rule->expr_id_cnt=1;
region_rule->expr_id_ub=region_rule->expr_id_lb=expr_id;
region_rule->district_id=district_id;
region_rule->table_type=region_type;
region_rule->table_id=table_id;
pthread_mutex_lock(&(group->mutex));
dynamic_array_write(group->regions,group->region_boundary,region_rule);
group->region_cnt++;
group->region_boundary++;
pthread_mutex_unlock(&(group->mutex));
}
else
{
assert(expr_id==region_rule->expr_id_ub+1);
region_rule->expr_id_ub=expr_id;
region_rule->expr_id_cnt++;
}
return group;
}
void cancel_last_region_from_group(struct _Maat_group_inner_t* group,int region_id,int expr_id)
{
struct _Maat_region_inner_t* region_rule=NULL;
pthread_mutex_lock(&(group->mutex));
region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group->regions,group->region_boundary);
assert(region_rule->expr_id_ub==expr_id&&region_rule->region_id==region_id);
if(region_rule->expr_id_cnt==1)
{
free(region_rule);
dynamic_array_write(group->regions,group->region_boundary,NULL);
group->region_cnt--;
group->region_boundary--;
}
else
{
region_rule->expr_id_ub--;
region_rule->region_cnt--;
}
pthread_mutex_unlock(&(group->mutex));
return;
}
unsigned int del_region_from_group(struct _Maat_group_inner_t* group,int region_id,unsigned int *output_expr_id,int output_size)
{
int i=0,j=0;
struct _Maat_region_inner_t* region_rule=NULL;
pthread_mutex_lock(&(group->mutex));
for(i=0;i<group->region_boundary;i++)
{
region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group->regions, i);
if(region_rule==NULL)
{
continue;
}
if(region_rule->region_id==region_id)
{
dynamic_array_write(group->regions, i, NULL);
for(j=0;j<region_rule->expr_id_cnt;j++)
{
output_expr_id[j]=region_rule->expr_id_lb+j;
}
assert(j<=output_size); assert(j<=output_size);
region_rule->region_id=0;
free(region_rule); free(region_rule);
region_rule=NULL; region_rule=NULL;
group->region_cnt--; group->region_cnt--;
break;
} }
} }
pthread_mutex_unlock(&(group->mutex)); pthread_mutex_unlock(&(group->mutex));
return j; return j;
} }
int add_group_to_compile(struct _Maat_compile_rule_t*a_compile_rule,struct _Maat_group_rule_t* a_rule_group) int add_group_to_compile(struct _Maat_compile_inner_t*a_compile_rule,struct _Maat_group_inner_t* a_rule_group)
{ {
int i=0,ret=-1; int i=0,ret=-1;
struct _Maat_group_rule_t* p=NULL; int write_pos=-1;
struct _Maat_group_inner_t* p=NULL;
pthread_rwlock_wrlock(&(a_compile_rule->rwlock)); pthread_rwlock_wrlock(&(a_compile_rule->rwlock));
if(a_compile_rule->db_c_rule!=NULL if(a_compile_rule->db_c_rule!=NULL
&&a_compile_rule->group_cnt>=a_compile_rule->db_c_rule->declare_grp_num &&a_compile_rule->group_cnt>=a_compile_rule->db_c_rule->declare_grp_num
&&a_compile_rule->db_c_rule->declare_grp_num!=0) &&a_compile_rule->db_c_rule->declare_grp_num!=0)
{ {
ret=-1; ret=-1;
goto error_out;
}
for(i=0;i<a_compile_rule->group_boundary;i++)
{
p=(struct _Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i);
if(p==NULL)
{
write_pos=i;
}
else
{
if(p->group_id==a_rule_group->group_id)//duplicate group
{
ret=-1;
goto error_out;
}
}
}
if(write_pos<0&&a_compile_rule->group_boundary==MAX_EXPR_ITEM_NUM)
{
ret=-1;
goto error_out;
}
if(write_pos<0)
{
write_pos=a_compile_rule->group_boundary;
a_compile_rule->group_boundary++;
}
dynamic_array_write(a_compile_rule->groups,write_pos, a_rule_group);
a_compile_rule->group_cnt++;
a_rule_group->ref_cnt++;
//member group->compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish.
if(a_rule_group->ref_cnt==1&&a_compile_rule->group_cnt==1)
{
a_rule_group->compile_shortcut=a_compile_rule;
} }
else else
{ {
for(i=0;i<MAAT_MAX_EXPR_ITEM_NUM;i++) a_rule_group->compile_shortcut=NULL;
{ }
p=(struct _Maat_group_rule_t*)dynamic_array_read(a_compile_rule->groups,i);
if(p==NULL)
{
dynamic_array_write(a_compile_rule->groups,i, a_rule_group);
a_compile_rule->group_cnt++;
a_rule_group->ref_cnt++;
//variable compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish.
if(a_rule_group->ref_cnt==1&&a_compile_rule->group_cnt==1)
{
a_rule_group->compile_shortcut=a_compile_rule;
}
else
{
a_rule_group->compile_shortcut=NULL;
}
ret=1;
break;
}
else
{
if(p->group_id==a_rule_group->group_id)//duplicate group
{
ret=-1;
} //update group's shortcut when compile has more than one group.
} if(a_compile_rule->group_cnt!=1)
} {
if(i==MAAT_MAX_EXPR_ITEM_NUM) for(i=0;i<a_compile_rule->group_boundary;i++)
{ {
ret=-1; p=(struct _Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i);
} if(p!=NULL)
//update group's shortcut when compile has more than one group.
if(a_compile_rule->group_cnt!=1)
{
for(i=0;i<MAAT_MAX_EXPR_ITEM_NUM;i++)
{ {
p=(struct _Maat_group_rule_t*)dynamic_array_read(a_compile_rule->groups,i); p->compile_shortcut=NULL;
if(p!=NULL)
{
p->compile_shortcut=NULL;
}
} }
} }
} }
ret=1;
error_out:
pthread_rwlock_unlock(&(a_compile_rule->rwlock)); pthread_rwlock_unlock(&(a_compile_rule->rwlock));
return ret; return ret;
} }
struct _Maat_group_rule_t* del_group_from_compile(struct _Maat_compile_rule_t*a_compile_rule,int group_id) struct _Maat_group_inner_t* del_group_from_compile(struct _Maat_compile_inner_t*a_compile_rule,int group_id)
{ {
int i=0; int i=0;
struct _Maat_group_rule_t* group_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL;
pthread_rwlock_wrlock(&(a_compile_rule->rwlock)); pthread_rwlock_wrlock(&(a_compile_rule->rwlock));
for(i=0;i<MAAT_MAX_EXPR_ITEM_NUM;i++) for(i=0;i<MAAT_MAX_EXPR_ITEM_NUM;i++)
{ {
group_rule=(struct _Maat_group_rule_t*)dynamic_array_read(a_compile_rule->groups,i); group_rule=(struct _Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i);
if(group_rule==NULL) if(group_rule==NULL)
{ {
continue; continue;
@@ -1493,7 +1555,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
int expr_id=0,district_id=-1; int expr_id=0,district_id=-1;
scan_rule_t*p_rule=NULL; scan_rule_t*p_rule=NULL;
struct _Maat_group_rule_t* group_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL;
enum MAAT_CHARSET dst_charset=CHARSET_NONE; enum MAAT_CHARSET dst_charset=CHARSET_NONE;
char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM]; char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM];
int key_left_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1},key_right_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1}; int key_left_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1},key_right_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1};
@@ -1504,14 +1566,14 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
} }
int sub_expr_cnt=0; int sub_expr_cnt=0;
struct op_expr_t *op_expr=NULL; struct op_expr_t *op_expr=NULL;
struct _Maat_group_rule_t* u_para=NULL; struct _Maat_group_inner_t* u_para=NULL;
if(table->table_type==TABLE_TYPE_EXPR_PLUS) if(table->table_type==TABLE_TYPE_EXPR_PLUS)
{ {
assert(strlen(db_rule->district)>0); assert(strlen(db_rule->district)>0);
district_id=get_district_id(scanner, db_rule->district); district_id=get_district_id(scanner, db_rule->district);
} }
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id); group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id);
if(group_rule==NULL) if(group_rule==NULL)
{ {
group_rule=create_group_rule(db_rule->group_id); group_rule=create_group_rule(db_rule->group_id);
@@ -1574,7 +1636,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
break; break;
case EXPR_TYPE_REGEX://it's easy,no need to charset convert case EXPR_TYPE_REGEX://it's easy,no need to charset convert
expr_id=scanner->exprid_generator++; expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule,db_rule->region_id,district_id,expr_id,TABLE_TYPE_EXPR); u_para=add_region_to_group(group_rule,table->table_id,db_rule->region_id,district_id,expr_id,TABLE_TYPE_EXPR);
if(u_para==NULL) if(u_para==NULL)
{ {
return -1; return -1;
@@ -1641,7 +1703,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
break; break;
} }
expr_id=scanner->exprid_generator++; expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule, db_rule->region_id,district_id,expr_id, table->table_type); u_para=add_region_to_group(group_rule,table->table_id, db_rule->region_id,district_id,expr_id, table->table_type);
if(u_para==NULL)//duplicate if(u_para==NULL)//duplicate
{ {
return -1; return -1;
@@ -1719,7 +1781,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
else else
{ {
expr_id=scanner->exprid_generator++; expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule, db_rule->region_id,district_id,expr_id, table->table_type); u_para=add_region_to_group(group_rule, table->table_id, db_rule->region_id,district_id,expr_id, table->table_type);
if(u_para==NULL) if(u_para==NULL)
{ {
return -1; return -1;
@@ -1752,13 +1814,13 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
} }
int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger) int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_group_rule_t* group_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL;
scan_rule_t* p_rule=NULL; scan_rule_t* p_rule=NULL;
struct op_expr_t* op_expr=NULL; struct op_expr_t* op_expr=NULL;
struct _Maat_group_rule_t* u_para=NULL; struct _Maat_group_inner_t* u_para=NULL;
int expr_id=0,district_id=-1; int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id); group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id);
if(group_rule==NULL) if(group_rule==NULL)
{ {
group_rule=create_group_rule(db_ip_rule->group_id); group_rule=create_group_rule(db_ip_rule->group_id);
@@ -1766,7 +1828,7 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule
} }
expr_id=scanner->exprid_generator++; expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule,db_ip_rule->region_id,district_id,expr_id,TABLE_TYPE_IP); u_para=add_region_to_group(group_rule, table->table_id,db_ip_rule->region_id,district_id,expr_id,TABLE_TYPE_IP);
if(u_para==NULL) if(u_para==NULL)
{ {
return -1; return -1;
@@ -1784,20 +1846,20 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule
} }
int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger) int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_group_rule_t* group_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL;
scan_rule_t* p_rule=NULL; scan_rule_t* p_rule=NULL;
struct op_expr_t* op_expr=NULL; struct op_expr_t* op_expr=NULL;
struct _Maat_group_rule_t* u_para=NULL; struct _Maat_group_inner_t* u_para=NULL;
int expr_id=0,district_id=-1; int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id); group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id);
if(group_rule==NULL) if(group_rule==NULL)
{ {
group_rule=create_group_rule(intval_rule->group_id); group_rule=create_group_rule(intval_rule->group_id);
HASH_add_by_id(scanner->group_hash, intval_rule->group_id, group_rule); HASH_add_by_id(scanner->group_hash, intval_rule->group_id, group_rule);
} }
expr_id=scanner->exprid_generator++; expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule,intval_rule->region_id,district_id,expr_id,TABLE_TYPE_INTERVAL); u_para=add_region_to_group(group_rule, table->table_id,intval_rule->region_id,district_id,expr_id,TABLE_TYPE_INTERVAL);
if(u_para==NULL) if(u_para==NULL)
{ {
return -1; return -1;
@@ -1815,19 +1877,19 @@ int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* in
} }
int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db_digest_rule,struct _Maat_scanner_t *scanner,void* logger) int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db_digest_rule,struct _Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_group_rule_t* group_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL;
GIE_digest_t* digest_rule=NULL; GIE_digest_t* digest_rule=NULL;
struct _Maat_group_rule_t* u_para=NULL; struct _Maat_group_inner_t* u_para=NULL;
int expr_id=0,district_id=-1; int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id);
if(group_rule==NULL) if(group_rule==NULL)
{ {
group_rule=create_group_rule(db_digest_rule->group_id); group_rule=create_group_rule(db_digest_rule->group_id);
HASH_add_by_id(scanner->group_hash, db_digest_rule->group_id, group_rule); HASH_add_by_id(scanner->group_hash, db_digest_rule->group_id, group_rule);
} }
expr_id=scanner->exprid_generator++; expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule,db_digest_rule->region_id,expr_id,district_id,TABLE_TYPE_DIGEST); u_para=add_region_to_group(group_rule,table->table_id,db_digest_rule->region_id,expr_id,district_id,TABLE_TYPE_DIGEST);
if(u_para==NULL) if(u_para==NULL)
{ {
return -1; return -1;
@@ -1845,10 +1907,10 @@ int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id,
int i=0; int i=0;
unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0}; unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0};
int expr_num=0; int expr_num=0;
struct _Maat_group_rule_t* group_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL;
struct op_expr_t* op_expr=NULL; struct op_expr_t* op_expr=NULL;
GIE_digest_t* digest_rule=NULL; GIE_digest_t* digest_rule=NULL;
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(maat_scanner->group_hash, group_id); group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(maat_scanner->group_hash, group_id);
if(group_rule==NULL) if(group_rule==NULL)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
@@ -1909,18 +1971,20 @@ int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id,
} }
int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_group_rule_t* group_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL;
struct _Maat_compile_rule_t*compile_rule=NULL; struct _Maat_compile_inner_t*compile_rule=NULL;
int ret=0; int ret=0;
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id);
if(group_rule==NULL) if(group_rule==NULL)
{ {
group_rule=create_group_rule(db_group_rule->group_id); group_rule=create_group_rule(db_group_rule->group_id);
group_rule->table_id=table->table_id;
ret=HASH_add_by_id(scanner->group_hash, db_group_rule->group_id,group_rule); ret=HASH_add_by_id(scanner->group_hash, db_group_rule->group_id,group_rule);
assert(ret>=0); assert(ret>=0);
} }
compile_rule=(struct _Maat_compile_rule_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id); compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id);
if(compile_rule==NULL) if(compile_rule==NULL)
{ {
compile_rule=create_compile_rule(db_group_rule->compile_id); compile_rule=create_compile_rule(db_group_rule->compile_id);
@@ -1942,9 +2006,9 @@ int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_g
} }
void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_compile_rule_t*compile_rule=NULL; struct _Maat_compile_inner_t*compile_rule=NULL;
struct _Maat_group_rule_t* group_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL;
compile_rule=(struct _Maat_compile_rule_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id); compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id);
if(compile_rule==NULL) if(compile_rule==NULL)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
@@ -1980,12 +2044,13 @@ void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_
} }
int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_compile_rule_t *compile_rule=NULL; struct _Maat_compile_inner_t *compile_rule=NULL;
struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head); struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head);
compile_rule=(struct _Maat_compile_rule_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id); compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id);
if(compile_rule==NULL) if(compile_rule==NULL)
{ {
compile_rule=create_compile_rule(p_maat_rule_head->config_id); compile_rule=create_compile_rule(p_maat_rule_head->config_id);
compile_rule->table_id=table->table_id;
HASH_add_by_id(scanner->compile_hash,p_maat_rule_head->config_id,compile_rule); HASH_add_by_id(scanner->compile_hash,p_maat_rule_head->config_id,compile_rule);
} }
if(compile_rule->db_c_rule!=NULL)//duplicate config if(compile_rule->db_c_rule!=NULL)//duplicate config
@@ -1998,8 +2063,8 @@ int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t*
} }
int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_compile_rule_t *compile_rule=NULL; struct _Maat_compile_inner_t *compile_rule=NULL;
compile_rule=(struct _Maat_compile_rule_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id); compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id);
if(compile_rule==NULL) if(compile_rule==NULL)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,

21
src/entry/Maat_rule_internal.h
View File

@@ -140,7 +140,6 @@ struct _head_Maat_rule_t
}; };
struct db_compile_rule_t struct db_compile_rule_t
{ {
// Maat_rule_t m_rule_head;
struct _head_Maat_rule_t m_rule_head;// fix len of Maat_rule_t struct _head_Maat_rule_t m_rule_head;// fix len of Maat_rule_t
char* service_defined; char* service_defined;
long long effective_range; long long effective_range;
@@ -163,14 +162,16 @@ struct op_expr_t
int rule_type; int rule_type;
}; };
struct _Maat_region_rule_t struct _Maat_region_inner_t
{ {
int region_id; int region_id;
int expr_id; int expr_id_cnt;
int district_id; int district_id;
int expr_id_lb;
int expr_id_ub;
enum MAAT_TABLE_TYPE region_type; enum MAAT_TABLE_TYPE region_type;
}; };
struct _Maat_group_rule_t struct _Maat_group_inner_t
{ {
int group_id; int group_id;
int region_boundary; int region_boundary;
@@ -180,10 +181,11 @@ struct _Maat_group_rule_t
void* compile_shortcut; void* compile_shortcut;
pthread_mutex_t mutex; pthread_mutex_t mutex;
}; };
struct _Maat_compile_rule_t struct _Maat_compile_inner_t
{ {
struct db_compile_rule_t *db_c_rule; struct db_compile_rule_t *db_c_rule;
dynamic_array_t *groups; dynamic_array_t *groups;
int group_boundary;
int group_cnt; int group_cnt;
int compile_id;//equal to db_c_rule->m_rule.config_id int compile_id;//equal to db_c_rule->m_rule.config_id
pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read
@@ -374,6 +376,7 @@ struct _Maat_feather_t
int cmd_num; int cmd_num;
struct _Maat_cmd_t* cmd_qhead, *cmd_qtail; struct _Maat_cmd_t* cmd_qhead, *cmd_qtail;
pthread_mutex_t redis_write_lock; //protect redis_write_ctx pthread_mutex_t redis_write_lock; //protect redis_write_ctx
long long base_rgn_seq,base_grp_seq;
//for stat>>>> //for stat>>>>
screen_stat_handle_t stat_handle; screen_stat_handle_t stat_handle;
int total_stat_id; int total_stat_id;
@@ -431,7 +434,13 @@ void maat_stat_init(struct _Maat_feather_t* feather);
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num); void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num);
void maat_stat_output(struct _Maat_feather_t* feather); void maat_stat_output(struct _Maat_feather_t* feather);
void redis_monitor_traverse(unsigned int version,redisContext *c,
void (*start)(unsigned int ,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para
void (*update)(const char* ,const char*,void* ),//table name ,line ,u_para
void (*finish)(void*),//u_para
void* u_para,
const unsigned char* dec_key,
_Maat_feather_t* feather);
#endif #endif