diff --git a/inc/Maat_command.h b/inc/Maat_command.h index 9e6dfec..1d824cb 100644 --- a/inc/Maat_command.h +++ b/inc/Maat_command.h @@ -51,10 +51,10 @@ enum MAAT_ADDR_DIRECTION struct Maat_rgn_str_t { const char *keywords; - const char *district; + const char *district;// optional for expr_plus, otherwise set to NULL. enum MAAT_EXPR_TYPE expr_type; enum MAAT_MATCH_METHOD match_method; - enum MAAT_CASE_TYPE case_type; + enum MAAT_CASE_TYPE hex_bin; }; struct Maat_rgn_addr_t { @@ -94,7 +94,7 @@ struct Maat_region_t union { struct Maat_rgn_str_t expr_rule; - struct Maat_rgn_addr_t addr_rule; + struct Maat_rgn_addr_t ip_rule; struct Maat_rgn_intv_t interval_rule; struct Maat_rgn_digest_t digest_rule; struct Maat_rgn_sim_t similarity_rule; @@ -104,14 +104,12 @@ struct Maat_group_t { int region_num; int group_id; //Any, maat will assigned one. - char* table_name;//optional, if not specified, maat will assigned to the first GROUP table in table_info.conf. char* group_name;//optional, for group reuse. struct Maat_region_t *regions; }; struct Maat_command_t { struct Maat_rule_t compile;// for MAAT_OP_DEL, only compile.config_id is necessary. - char* table_name; //optional, if not specified, maat will assigned to the first COMPILE table in table_info.conf. int group_num; // for MAAT_OP_DEL, Any. struct Maat_group_t* groups;// for MAAT_OP_DEL, SET to NULL. }; diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 542ecc1..9521dd8 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -106,21 +106,21 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int int r_in_c_cnt=0; int shortcut_avilable_cnt=0; void* bool_matcher=feather->scanner->expr_compiler; - struct _Maat_group_rule_t* group_rule=NULL; - struct _Maat_compile_rule_t* array_mi_rule[MAX_SCANNER_HIT_NUM]; - struct _Maat_compile_rule_t* _mi_rule=NULL; + struct _Maat_group_inner_t* group_rule=NULL; + struct _Maat_compile_inner_t* array_mi_rule[MAX_SCANNER_HIT_NUM]; + struct _Maat_compile_inner_t* _mi_rule=NULL; int region_pos[MAX_SCANNER_HIT_NUM]; _mid->cur_hit_cnt=0; for(i=0;igroup_id<0) { continue; } if(group_rule->compile_shortcut!=NULL&&group_rule->ref_cnt==1&&shortcut_avilable_cntcompile_shortcut); + array_mi_rule[shortcut_avilable_cnt]=(struct _Maat_compile_inner_t*)(group_rule->compile_shortcut); shortcut_avilable_cnt++; } _mid->cur_hit_id[_mid->cur_hit_cnt]=group_rule->group_id; @@ -183,23 +183,24 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int return result_cnt; } -int exprid2region_id(struct _Maat_group_rule_t* group_rule,int expr_id,int* district_id) +int exprid2region_id(struct _Maat_group_inner_t* group_rule,int expr_id,int* district_id) { - int i=0,region_id=-1; - struct _Maat_region_rule_t* region_rule=NULL; + int i=0,j=0,region_id=-1; + struct _Maat_region_inner_t* region_rule=NULL; assert(group_rule->group_id>=0); pthread_mutex_lock(&(group_rule->mutex)); for(i=0;iregion_boundary;i++) { - region_rule=(struct _Maat_region_rule_t*)dynamic_array_read(group_rule->region_rules, i); + region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group_rule->regions, i); if(region_rule==NULL) { continue; } - if(region_rule->expr_id==expr_id) + if(expr_id>=region_rule->expr_id_lb&&expr_id<=region_rule->expr_id_ub) { region_id=region_rule->region_id; *district_id=region_rule->district_id; + break; } } pthread_mutex_unlock(&(group_rule->mutex)); @@ -207,13 +208,13 @@ int exprid2region_id(struct _Maat_group_rule_t* group_rule,int expr_id,int* dist } int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num) { - struct _Maat_group_rule_t* group_rule=NULL; + struct _Maat_group_inner_t* group_rule=NULL; int i=0; int district_id=-1,region_id=-1; int ret_region_num=region_hit_num; while(i0&&district_id!=_mid->district_id) { @@ -309,7 +310,7 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi memset(r_in_c_flag,0,sizeof(r_in_c_flag)); memset(region_pos,0,sizeof(region_pos)); - struct _Maat_group_rule_t* group_rule=NULL; + struct _Maat_group_inner_t* group_rule=NULL; //for each hitted compile cfg,find its region_ids for(i=0;iregion_type) @@ -65,9 +59,76 @@ enum MAAT_TABLE_TYPE region_table_type(const struct Maat_region_t* p) } return ret; } -void free_serial_rules(void* p) +void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int size) +{ + int ret=0; + switch(p->region_type) + { + case REGION_IP: + ret=snprintf(buff,size,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1" + ,p->ip_rule.region_id + ,group_id + ,p->ip_rule.addr_type + ,p->ip_rule.src_ip + ,p->ip_rule.mask_src_ip + ,p->ip_rule.src_port + ,p->ip_rule.mask_src_port + ,p->ip_rule.dst_ip + ,p->ip_rule.mask_dst_ip + ,p->ip_rule.dst_port + ,p->ip_rule.mask_dst_port + ,p->ip_rule.protocol + ,p->ip_rule.direction); + break; + case REGION_EXPR: + if(p->expr_rule.district==NULL) + { + ret=snprintf(buff,size,"%d\t%d\t%s\t%d\t%d\t%d\t1" + ,p->region_id + ,group_id + ,p->expr_rule.keywords + ,p->expr_rule.expr_type + ,p->expr_rule.match_method + ,p->expr_rule.hex_bin); + } + else //expr_plus + { + ret=snprintf(buff,size,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1" + ,p->region_id + ,group_id + ,p->expr_rule.keywords + ,p->expr_rule.district + ,p->expr_rule.expr_type + ,p->expr_rule.match_method + ,p->expr_rule.hex_bin); + } + break; + case REGION_INTERVAL: + ret=snprintf(buff,size,"%d\t%d\t%u\t%u\t1" + ,p->region_id + ,group_id + ,p->interval_rule.low_boundary + ,p->interval_rule.up_boundary); + break; + case REGION_DIGEST: + ret=snprintf(buff,size,"%d\t%d\t%llu\t%s\t%hd\t1" + ,p->region_id + ,group_id + ,p->digest_rule.orgin_len + ,p->digest_rule.digest_string + ,p->digest_rule.confidence_degree); + break; + case REGION_SIMILARITY://not support yet + assert(0); + break; + default: + assert(0); + } + assert(rettable_line!=NULL) { free(rule->table_line); @@ -76,11 +137,10 @@ void free_serial_rules(void* p) free(rule); return; } -void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,enum MAAT_TABLE_TYPE table_type,int id,const char* table_name,const char* line) +void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,const char* table_name,const char* line) { rule->op=op; - rule->rule_id=id; - rule->table_type=table_type; + rule->rule_id=rule_id; assert(srtlen(table_name)table_name)); memcpy(rule->table_name,table_name,strlen(table_name)); if(line!=NULL) @@ -97,7 +157,7 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** int rule_id; long long version_in_redis=0; int i=0,ret=0,retry=0; - struct serial_rule_t *p_rm_rule=NULL; + struct serial_rule_t *s_rule=NULL; if(version==0) { @@ -166,26 +226,26 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** } freeReplyObject(data_reply); data_reply=NULL; - p_rm_rule=(struct serial_rule_t*)calloc(ctrl_reply->elements,sizeof(struct serial_rule_t)); + s_rule=(struct serial_rule_t*)calloc(ctrl_reply->elements,sizeof(struct serial_rule_t)); for(i=0;ielements;i++) { assert(ctrl_reply->element[i]->type==REDIS_REPLY_STRING); - ret=sscanf(ctrl_reply->element[i]->str,"%s,%s,%d",op_str,p_rm_rule.table_name,&(p_rm_rule[i].rule_id)); + ret=sscanf(ctrl_reply->element[i]->str,"%s,%s,%d",op_str,s_rule.table_name,&(s_rule[i].rule_id)); assert(ret==3); if(strncmp(op_str,"ADD")==0) { - p_rm_rule[i].op=1; + s_rule[i].op=MAAT_OP_ADD; } else if(strncmp(op_str,"DEL")==0) { - p_rm_rule[i].op=0; + s_rule[i].op=MAAT_OP_DEL; } else { assert(0); } } - *list=p_rm_rule; + *list=s_rule; *update_type=CM_UPDATE_TYPE_INC; freeReplyObject(ctrl_reply); return i; @@ -194,15 +254,15 @@ FULL_UPDATE: "Initiate full udpate from version %d to %lld.",version,version_in_redis); data_reply=(redisReply*)redisCommand(c, "KEYS EFFECTIVE_RULE:*"); assert(data_reply->type==REDIS_REPLY_ARRAY); - p_rm_rule=(struct serial_rule_t*)calloc(ctrl_reply->elements,sizeof(struct serial_rule_t)); + s_rule=(struct serial_rule_t*)calloc(ctrl_reply->elements,sizeof(struct serial_rule_t)); for(i=0;ielements;i++) { assert(ctrl_reply->element[i]->type==REDIS_REPLY_STRING); - ret=sscanf(ctrl_reply->element[i]->str,"EFFECTIVE_RULE:%s,%d",p_rm_rule[i].table_name,&(p_rm_rule[i].rule_id)); - p_rm_rule[i].op=1; + ret=sscanf(ctrl_reply->element[i]->str,"EFFECTIVE_RULE:%s,%d",s_rule[i].table_name,&(s_rule[i].rule_id)); + s_rule[i].op=1; assert(ret==2); } - *list=p_rm_rule; + *list=s_rule; *update_type=CM_UPDATE_TYPE_FULL; freeReplyObject(ctrl_reply); ctrl_reply=NULL; @@ -216,14 +276,16 @@ void redis_monitor_traverse(unsigned int version,redisContext *c, void (*finish)(void*),//u_para void* u_para, const unsigned char* dec_key, - void* logger) + _Maat_feather_t* feather) { redisReply* ctrl_reply=NULL,*data_reply=NULL; - int rule_num=0,i=0; + int rule_num=0,i=0,table_id=0; int ret=0; struct serial_rule_t* rule_list=NULL; int update_type=0; unsigned int new_version=0; + enum MAAT_TABLE_TYPE table_type=0; + void* logger=feather->logger; rule_num=get_rm_key_list(version, c, &rule_list, logger,&new_version, &update_type); if(rule_num==0) { @@ -241,6 +303,16 @@ void redis_monitor_traverse(unsigned int version,redisContext *c, start(new_version,update_type); for(i=0;ielements;i++) { + if(rule_list[i].op==MAAT_OP_DEL) + { + ret=feather->map_tablename2id(rule_list[i].table_name,&table_id); + if(ret<0) + { + continue; + } + table_type=feather->p_table_info[table_id].table_type; + invalidate_line(data_reply->element[i].str,table_type); + } update(rule_list[i].table_name,data_reply->element[i].str,u_para); } finish(u_para); @@ -259,17 +331,9 @@ struct Maat_command_t* Maat_create_comand(const struct Maat_rule_t* rule, int gr return (struct Maat_command_t*)_cmd; } -void Maat_cmd_set_group(struct Maat_command_t* cmd,int which_group,int region_num,const char* table_name,const char* group_name) +void Maat_cmd_set_group(struct Maat_command_t* cmd,int which_group,int region_num,const char* group_name) { assert(which_groupgroup_num); - if(table_name!=NULL) - { - if(cmd->groups[which_group].table_name!=NULL) - { - free(cmd->groups[which_group].table_name); - } - cmd->groups[which_group].table_name=_maat_strdup(table_name); - } if(group_name!=NULL) { if(cmd->groups[which_group].group_name!=NULL) @@ -292,10 +356,10 @@ void Maat_copy_region(struct Maat_region_t* dst,const struct Maat_region_t* src) switch(dst->region_type) { case REGION_IP: - dst->addr_rule.src_ip=_maat_strdup(src->addr_rule.src_ip); - dst->addr_rule.mask_src_ip=_maat_strdup(src->addr_rule.mask_src_ip); - dst->addr_rule.dst_ip=_maat_strdup(src->addr_rule.dst_ip); - dst->addr_rule.mask_src_ip=_maat_strdup(src->addr_rule.mask_src_ip); + dst->ip_rule.src_ip=_maat_strdup(src->ip_rule.src_ip); + dst->ip_rule.mask_src_ip=_maat_strdup(src->ip_rule.mask_src_ip); + dst->ip_rule.dst_ip=_maat_strdup(src->ip_rule.dst_ip); + dst->ip_rule.mask_src_ip=_maat_strdup(src->ip_rule.mask_src_ip); break; case REGION_EXPR: dst->expr_rule.keywords=_maat_strdup(src->expr_rule.keywords); @@ -321,10 +385,10 @@ void Maat_empty_region(struct Maat_region_t* p) switch(p->region_type) { case REGION_IP: - free(p->addr_rule.src_ip); - free(p->addr_rule.mask_src_ip); - free(p->addr_rule.dst_ip); - free(p->addr_rule.mask_src_ip); + free(p->ip_rule.src_ip); + free(p->ip_rule.mask_src_ip); + free(p->ip_rule.dst_ip); + free(p->ip_rule.mask_src_ip); break; case REGION_EXPR: free(p->expr_rule.keywords); @@ -371,7 +435,6 @@ void Maat_free_command(struct Maat_command_t* cmd) { Maat_empty_region(&(cmd.groups[i].regions[j])); } - free(cmd.groups[i].table_name); free(cmd.groups[i].group_name); free(cmd.groups[i].regions); cmd.groups[i].regions=NULL; @@ -384,7 +447,8 @@ void Maat_free_command(struct Maat_command_t* cmd) } int Maat_format_command(struct Maat_command_t* rule, char* buffer, int size) { - + //TODO + return 0; } int Maat_command(Maat_feather_t feather,struct Maat_command_t* raw_rule,enum MAAT_OPERATION op) { @@ -397,28 +461,26 @@ int Maat_command(Maat_feather_t feather,struct Maat_command_t* raw_rule,enum MAA ret=Maat_commit_command(feather); return ret; } -//functioned as strdup, for dictator compatible. -char* _maat_strdup(const char* s) -{ - char*d=NULL; - if(s==NULL) - { - return NULL; - } - d=(char*)malloc(strlen(s)+1); - memcpy(d,s,strlen(s)+1)); - return d; -} -void Maat_append_command(Maat_feather_t feather,struct Maat_command_t* cmd,enum MAAT_OPERATION op) + +int Maat_append_command(Maat_feather_t feather,struct Maat_command_t* cmd,enum MAAT_OPERATION op) { _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_cmd_t* _cmd=(struct _Maat_cmd_t*)cmd; + int i=0, ret=0; _cmd->ref_cnt++; _cmd->op=op; assert(op==MAAT_OP_DEL||op==MAAT_OP_ADD); - if(cmd->table_name==NULL) + if(op==MAAT_OP_DEL) { - cmd->table_name=_maat_strdup(_feather->compile_tn); + ret=reconstruct_cmd(_feather, _cmd) + } + else + { + ret=fix_table_name(_feather, cmd); + } + if(ret<0) + { + return -1; } if(_feather->cmd_num==0) { @@ -483,104 +545,89 @@ void invalidate_line(char* line, enum MAAT_TABLE_TYPE type) line[i]='0'; return; } -int calculate_serial_rule_num(struct Maat_command_t* cmd) +int calculate_serial_rule_num(struct _Maat_cmd_t* _cmd,int * new_region_cnt, int* new_group_cnt) { int serial_num=0; - int i=0,j=0; + int i=0; + struct Maat_command_t* cmd=&(_cmd->cmd); serial_num++;//compile rule for(i=0;igroup_num;i++) { serial_num++; - for(j=0;jgroups[i].region_num;j++) + if(cmd->groups[i].regions==NULL) { - serial_num++; + continue; + } + if(_cmd->op==MAAT_OP_ADD) + { + *new_region_cnt+=cmd->groups[i].region_num; + *new_group_cnt++; } } return serial_num; } -int reconstruct_cmd(redisContext *ctx,struct _Maat_cmd_t* _cmd,void* logger) +int reconstruct_cmd(struct _Maat_feather_t feather, struct _Maat_cmd_t* _cmd) { - int redis_ret=REDIS_ERR,ret=0; + int ret=0; int i=0,j=0,grp_idx=0; - redisReply* compile_reply=NULL,*group_reply=NULL,*region_reply=NULL; long long group_ref_cnt=0; - char table_name[MAX_TABLE_NAME_LEN]; struct Maat_command_t* cmd=&(_cmd->cmd); - struct Maat_group_t* p_group=NULL; - struct Maat_region_t* p_region=NULL; - + struct Maat_group_t* group_cmd=NULL; + struct Maat_region_t* region_cmd=NULL; + struct _Maat_compile_inner_t *compile_inner=NULL; + struct _Maat_group_inner_t* group_inner=NULL; + struct _Maat_region_inner_t* region_inner=NULL; + void* logger=feather->logger; + int config_id=cmd->compile.config_id; - compile_reply=redisCommand(ctx,"HKEYS %s:%d",PREFIX_COMPILE_INDEX,config_id); - if(compile_reply==NULL) + compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id); + if(compile_inner==NULL) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_command - ,"%s:%d not found." - ,PREFIX_COMPILE_INDEX,config_id); + ,"config %d not exist." + ,config_id); return -1; } - cmd->group_num=compile_reply->elements; - assert(cmd->compile.declare_grp_num==cmd->group_num); + cmd->group_num=compile_inner->group_cnt; assert(cmd->groups==NULL); - cmd->groups=calloc(sizeof(struct Maat_group_t),cmd->group_num); - for(i=0;ielements;i++) + cmd->groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),cmd->group_num); + for(i=0;igroup_boundary;i++) { - if(strncmp(compile_reply->element[i].str,"GROUP:",strlen("GROUP:"))!=0) + group_inner=(struct _Maat_group_inner_t*)dynamic_array_read(compile_inner->groups,i); + if(group_inner==NULL) { continue; } - p_group=&(cmd->groups[grp_idx]); - ret=sscanf(compile_reply->element[i].str,"GROUP:%s,%d",table_name,&(p_group->group_id)); - assert(ret==2); - p_group->table_name=_maat_strdup(table_name); + group_cmd=&(cmd->groups[grp_idx]); + group_cmd->group_id=group_inner->group_id; - group_reply=redisCommand(ctx,"HINCRBY %s:%s,%d REF_CNT -1",PREFIX_GROUP_INDEX - ,p_group->table_name - ,p_group->group_id); - freeReplyObject(group_reply); - group_reply=redisCommand(ctx,"HGET %s:%s,%d REF_CNT",PREFIX_GROUP_INDEX - ,p_group->table_name - ,p_group->group_id); - group_ref_cnt=group_reply->integer; - freeReplyObject(group_reply); - _cmd->group_ref_cnt[grp_idx]=group_ref_cnt; - grp_idx++; - if(group_ref_cnt>0) + if(group_inner->ref_cnt>0) { continue; } - group_reply=redisCommand(ctx,"HKEYS %s:%s,%d",PREFIX_GROUP_INDEX - ,p_group->table_name - ,p_group->group_id); - p_group->region_num=group_reply->elements; - p_group->regions=(struct Maat_region_t*)calloc(sizeof(struct Maat_region_t),p_group->region_num); - for(j=0;jelements;j++) + group_cmd->region_num=group_inner->region_cnt; + group_cmd->regions=(struct Maat_region_t*)calloc(sizeof(struct Maat_region_t),group_cmd->region_num); + for(j=0;jregion_boundary;j++) { - if(strncmp(group_reply->element[j].str,"REGION:",strlen("REGION:"))!=0) + region_inner=(struct _Maat_region_inner_t*)dynamic_array_read(group_inner->regions,i); + if(region_inner==NULL) { continue; } - p_region=&(p_group->regions[p_group->region_num]); - ret=sscanf(group_reply->element[j].str,"REGION:%s,%d",table_name,&(p_region->region_id)); - assert(ret==2); - p_region->table_name=_maat_strdup(table_name); - region_reply=redisCommand(ctx,"HGET %s %s,%d",PREFIX_REGION_INDEX - ,p_region->table_name - ,p_region->region_id); - p_region->region_type=region_reply->integer; - p_group->region_num++; - - freeReplyObject(region_reply); + region_cmd=&(group_cmd->regions[group_cmd->region_num]); + region_cmd->table_name=_maat_strdup(feather->p_table_info[region_inner->table_id].table_name[0]); + region_cmd->region_id=region_inner->table_id; + //NOTICE: a table_type stored in region_type. + region_cmd->region_type=region_inner->table_type; + group_cmd->region_num++; } - freeReplyObject(group_reply); - group_reply=NULL; + grp_idx++; } - freeReplyObject(compile_reply); - compile_reply=NULL; return 0; } -int build_serial_rule_from_redis(redisContext *ctx,struct Maat_command_t* cmd,struct serial_rule_t* list, int size) +int build_serial_rule_from_redis(_Maat_feather_t *feather,redisContext *ctx,struct Maat_command_t* cmd,struct serial_rule_t* list, int size) { struct Maat_group_t* p_group=NULL; struct Maat_region_t* p_region=NULL; @@ -588,21 +635,14 @@ int build_serial_rule_from_redis(redisContext *ctx,struct Maat_command_t* cmd,st redisReply* data_reply=NULL; int rule_num=0; int i=0,j=0; - data_reply=redisCommand(ctx,"GET %s:%s,%d",rm_key_prefix[MAAT_OP_ADD] - ,cmd->table_name - ,cmd->compile.config_id); - invalidate_line(data_reply->str, TABLE_TYPE_COMPILE); - set_serial_rule(list+rule_num,MAAT_OP_DEL,TABLE_TYPE_COMPILE,cmd->compile.config_id,cmd->table_name,data_reply->str); + + set_serial_rule(list+rule_num,MAAT_OP_DEL,cmd->compile.config_id,feather->compile_tn,NULL); rule_num++; - freeReplyObject(data_reply); for(i=0;igroup_num;i++) { p_group=&(cmd->groups[i]); - data_reply=redisCommand(ctx,"GET %s:%s,%d",rm_key_prefix[MAAT_OP_ADD] - ,p_group->table_name - ,p_group->group_id); - invalidate_line(data_reply->str, TABLE_TYPE_GROUP); - set_serial_rule(list+rule_num,MAAT_OP_DEL,TABLE_TYPE_GROUP,p_group->group_id,p_group->table_name,data_reply->str); + + set_serial_rule(list+rule_num,MAAT_OP_DEL,p_group->group_id,feather->group_tn,NULL); rule_num++; freeReplyObject(data_reply); for(j=0;jregion_num;j++) @@ -611,38 +651,87 @@ int build_serial_rule_from_redis(redisContext *ctx,struct Maat_command_t* cmd,st data_reply=redisCommand(ctx,"GET %s:%s,%d",rm_key_prefix[MAAT_OP_ADD] ,p_region->table_name ,p_region->region_id); - table_type=region_table_type(p_region); - invalidate_line(data_reply->str, table_type); + table_type=type_region2table(p_region); + invalidate_line(data_reply->str, (enum MAAT_TABLE_TYPE)(p_region->region_type)); set_serial_rule(list+rule_num,MAAT_OP_DEL,p_region->region_id,p_region->table_name,data_reply->str); rule_num++; freeReplyObject(data_reply); } } assert(rule_numcmd); + enum MAAT_OPERATION op=_cmd->op; + redisContext* ctx=feather->redis_write_ctx; + redisReply* data_reply=NULL; - int rule_num=0,i=0; + int rule_num=0,i=0,j=0; p_m_rule=&(cmd->compile); char line[1024]; - snprintf(line,sizeof(line),"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t1\t%d",p_m_rule->config_id - ,p_m_rule->service_id - ,p_m_rule->action - ,p_m_rule->do_blacklist - ,p_m_rule->do_log - ,p_m_rule->service_defined - ,p_m_rule->declare_grp_num); - set_serial_rule(list+rule_num,MAAT_OP_ADD,cmd->compile.config_id,cmd->table_name,line); + if(op===MAAT_OP_ADD) + { + snprintf(line,sizeof(line),"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t1\t%d",p_m_rule->config_id + ,p_m_rule->service_id + ,p_m_rule->action + ,p_m_rule->do_blacklist + ,p_m_rule->do_log + ,p_m_rule->service_defined + ,cmd->group_num); + set_serial_rule(list+rule_num,MAAT_OP_ADD,cmd->compile.config_id,feather->compile_tn,line); + + } + else + { + set_serial_rule(list+rule_num,MAAT_OP_DEL,cmd->compile.config_id,feather->compile_tn,NULL); + } rule_num++; - data_reply=redisCommand(ctx,"GET %s:%s,%d",rm_key_prefix[MAAT_OP_ADD] - ,cmd->table_name - ,cmd->compile.config_id); - return 0; + for(i=0;cmd->group_num;i++) + { + p_group=&(cmd->groups[i]); + if(op===MAAT_OP_ADD) + { + p_group->group_id=feather->base_grp_seq; + feather->base_grp_seq++; + snprintf(line,sizeof(line),"%d\t%d\t1",p_group->group_id + ,p_m_rule->config_id); + set_serial_rule(list+rule_num,MAAT_OP_ADD,p_group->group_id,feather->group_tn,line); + } + else + { + set_serial_rule(list+rule_num,MAAT_OP_DEL,p_group->group_id,feather->group_tn,NULL); + } + rule_num++; + if(p_group->regions==NULL)//group reuse. + { + continue; + } + for(j=0;jregion_num;j++) + { + if(op===MAAT_OP_ADD) + { + p_group->regions[j].region_id=feather->base_rgn_seq; + feather->base_rgn_seq++; + serialize_region(p_group->regions+j, p_group->group_id, line, sizeof(line)); + set_serial_rule(list+rule_num,MAAT_OP_ADD + ,p_group->regions[j].region_id,p_group->regions[j].table_name,line); + } + else + { + set_serial_rule(list+rule_num,MAAT_OP_DEL + ,p_group->regions[j].region_id,p_group->regions[j].table_name,NULL); + + } + rule_num++; + } + } + assert(rule_numcmd); - int redis_ret=REDIS_ERR,ret=0; - redisReply* data_reply=NULL; - reconstruct_cmd(ctx, _cmd, logger); - serial_num=calculate_serial_rule_num(cmd); - s_rule=(struct serial_rule_t*)calloc(sizeof(struct serial_rule_t),serial_num); - ret=build_serial_rule_from_redis(ctx, cmd, s_rule, serial_num); - - - retry=0; - while(1) + int i=0,j=0,ret=0; + char *table_name=NULL; + int table_id=0; + enum MAAT_TABLE_TYPE table_type; + for(i=0;igroup_num;i++) { - - data_reply=redisCommand(ctx, "WATCH MAAT_VERSION"); - freeReplyObject(data_reply); - data_reply=redisCommand(ctx, "GET MAAT_VERSION"); - freeReplyObject(data_reply); - maat_redis_version=data_reply->integer; - maat_redis_version++; - data_reply=redisCommand(ctx,"MULTI"); - freeReplyObject(data_reply); - for(i=0;igroups[i].region_num;j++) { - data_reply=redisCommand(ctx,"DEL %s:%s,%d",rm_key_prefix[MAAT_OP_ADD] - ,s_rule[i].table_name - ,s_rule[i].rule_id); - freeReplyObject(data_reply); - - data_reply=redisCommand(ctx,"SET %s:%s,%d \"%s\"",rm_key_prefix[MAAT_OP_DEL] - ,s_rule[i].table_name - ,s_rule[i].rule_id - ,s_rule[i].table_line); - freeReplyObject(data_reply); - data_reply=redisCommand(ctx,"EXPIRE %s:%s,%d %d",rm_key_prefix[MAAT_OP_DEL] - ,s_rule[i].table_name - ,s_rule[i].rule_id - ,MAAT_REDIS_SYNC_TIME); - freeReplyObject(data_reply); - //NX: Don't update already exisiting elements. Always add new elements. - data_reply=redisCommand(ctx,"ZADD NX DEL,%s,%d %d",s_rule[i].table_name - ,s_rule[i].rule_id - ,maat_redis_version); - freeReplyObject(data_reply); + table_name=cmd->groups[i].regions[j].table_name; + ret=map_str2int(feather->map_tablename2id, table_name, &table_id); + if(ret<0) + { + MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module + ,"Unknown table %s of Maat_command_t[%d]->group[%d]->region[%d]." + ,table_name,cmd->compile.config_id,i,j); - - data_reply=redisCommand(ctx,"INCRBY MAAT_VERSION 1"); - freeReplyObject(data_reply); - } - - data_reply=redisCommand(ctx,"EXEC"); - if(1==mr_transaction_success(data_reply)) - { - freeReplyObject(data_reply); - break; - } - else - { - retry++; - assert(retry<5); - freeReplyObject(data_reply); + return -1; + } + table_type=type_region2table(&(cmd->groups[i].regions[j])); + if(table_type!=feather->p_table_info[i].table_type) + { + MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module + ,"Table %s not support region type %d of Maat_command_t[%d]->group[%d]->region[%d]." + ,table_name + ,cmd->groups[i].regions[j].region_type + ,cmd->compile.config_id,i,j); + return -1; + } + free(cmd->groups[i].regions[j].table_name); + cmd->groups[i].regions[j].table_name=_maat_strdup(feather->p_table_info[i].table_name[0]); } } - for(i=0;icmd); - struct serial_rule_t *s_rule=NULL; - int serial_num=calculate_serial_rule_num(cmd); - s_rule=(struct serial_rule_t*)calloc(sizeof(struct serial_rule_t),serial_num); - ret=build_serial_rule_from_cmd(ctx, cmd, s_rule, serial_num); - retry=0; - while(1) - { - - data_reply=redisCommand(ctx, "WATCH MAAT_VERSION"); - freeReplyObject(data_reply); - data_reply=redisCommand(ctx, "GET MAAT_VERSION"); - freeReplyObject(data_reply); - maat_redis_version=data_reply->integer; - maat_redis_version++; - data_reply=redisCommand(ctx,"MULTI"); - freeReplyObject(data_reply); - for(i=0;iredis_write_ctx==NULL) { _feather->redis_write_ctx=redisConnectWithTimeout(_feather->redis_ip, _feather->redis_port,timeout); @@ -816,19 +813,94 @@ int Maat_commit_command(Maat_feather_t feather) goto error_out; } } - p=feather->cmd_qhead; - for(i=0;icmd_num;i++) + ctx=_feather->redis_write_ctx; + + for(i=0,p=feather->cmd_qhead;icmd_num;i++) { p=p->next; - if(p->op==MAAT_OP_DEL) + serial_rule_num+=calculate_serial_rule_num(&(p->cmd), &new_region_num, &new_group_num); + } + + data_reply=redisCommand(ctx,"INCRBY SEQUENCE_REGION %d",new_region_num); + _feather->base_rgn_seq=data_reply->integer-new_region_num; + freeReplyObject(data_reply); + + data_reply=redisCommand(ctx,"INCRBY SEQUENCE_GROUP %d",new_group_num); + _feather->base_rgn_seq=data_reply->integer-new_group_num; + freeReplyObject(data_reply); + + s_rule=(struct serial_rule_t*)calloc(sizeof(struct serial_rule_t),serial_rule_num); + + for(i=0,p=feather->cmd_qhead;icmd_num;i++) + { + p=p->next; + serial_rule_idx+=build_serial_rule(feather,p,s_rule, serial_rule_num-serial_rule_idx); + } + assert(serial_rule_idx==serial_rule_num); + + while(1) + { + data_reply=redisCommand(ctx, "WATCH MAAT_VERSION"); + freeReplyObject(data_reply); + data_reply=redisCommand(ctx, "GET MAAT_VERSION"); + freeReplyObject(data_reply); + maat_redis_version=data_reply->integer; + maat_redis_version++; + data_reply=redisCommand(ctx,"MULTI"); + freeReplyObject(data_reply); + for(i=0;iredis_write_ctx, p, _feather->logger); + if(s_rule[i].op==MAAT_OP_ADD) + { + redisAppendCommand(ctx,"SET %s:%s,%d \"%s\"",rm_key_prefix[MAAT_OP_ADD] + ,s_rule[i].table_name + ,s_rule[i].rule_id + ,s_rule[i].table_line); + + //NX: Don't update already exisiting elements. Always add new elements. + redisAppendCommand(ctx,"ZADD NX ADD,%s,%d %d",s_rule[i].table_name + ,s_rule[i].rule_id + ,maat_redis_version); + } + else + { + + redisAppendCommand(ctx,"RENAME %s:%s,%d %s:%s,%d" + ,rm_key_prefix[MAAT_OP_ADD] + ,s_rule[i].table_name + ,s_rule[i].rule_id + ,rm_key_prefix[MAAT_OP_DEL] + ,s_rule[i].table_name + ,s_rule[i].rule_id + ); + + redisAppendCommand(ctx,"EXPIRE %s:%s,%d %d",rm_key_prefix[MAAT_OP_DEL] + ,s_rule[i].table_name + ,s_rule[i].rule_id + ,MAAT_REDIS_SYNC_TIME); + + //NX: Don't update already exisiting elements. Always add new elements. + redisAppendCommand(ctx,"ZADD NX DEL,%s,%d %d",s_rule[i].table_name + ,s_rule[i].rule_id + ,maat_redis_version); + } + } + redisAppendCommand(ctx,"INCRBY MAAT_VERSION 1"); + redisAppendCommand(ctx,"EXEC"); + redisGetReply(ctx, &data_reply); + if(1==mr_transaction_success(data_reply)) + { + freeReplyObject(data_reply); + break; } else { - add_cmd_to_redis(_feather->redis_write_ctx, p, _feather->logger); - } + retry++; + assert(retry<5); + freeReplyObject(data_reply); + } } + error_out: p=_feather->cmd_qhead; for(i=0;i<_feather->cmd_num;i++) @@ -839,6 +911,13 @@ error_out: } _feather->cmd_qhead=_feather->cmd_qtail=NULL; _feather->cmd_num=0; + + for(i=0;igroup_id=group_id; group->region_cnt=0; group->region_boundary=0; group->ref_cnt=0; - group->region_rules=dynamic_array_create(1,8); + group->regions=dynamic_array_create(1,8); group->compile_shortcut=NULL; + group->table_id=0; + group->group_name=NULL; pthread_mutex_init(&(group->mutex), NULL); return group; } -void destroy_group_rule(struct _Maat_group_rule_t* group) +void destroy_group_rule(struct _Maat_group_inner_t* group) { if(group->ref_cnt>0||group->region_cnt>0) @@ -642,25 +656,28 @@ void destroy_group_rule(struct _Maat_group_rule_t* group) return; } - dynamic_array_destroy(group->region_rules,free); + dynamic_array_destroy(group->regions,free); group->region_cnt=0; group->region_boundary=0; - group->region_rules=NULL; + group->regions=NULL; group->ref_cnt=0; group->group_id=-1; + group->table_id=-1; + free(group->group_name); + group->group_name=NULL; pthread_mutex_destroy(&(group->mutex)); free(group); } -void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bool_expr_t* a_set) +void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set) { int i=0,j=0; a_set->bool_expr_id=(void*)compile_rule; - struct _Maat_group_rule_t*group=NULL; - assert(MAAT_MAX_EXPR_ITEM_NUM<=MAX_ITEMS_PER_BOOL_EXPR); - for(i=0,j=0;igroup_cnt<=MAX_ITEMS_PER_BOOL_EXPR); + for(i=0,j=0;igroup_boundary&&jgroups,i); + group=(struct _Maat_group_inner_t*)dynamic_array_read(compile_rule->groups,i); if(group==NULL) { continue; @@ -674,7 +691,7 @@ void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bo void walk_compile_hash(const uchar * key, uint size, void * data, void * user) { universal_bool_expr_t* one_set=NULL; - struct _Maat_compile_rule_t* compile_rule=(struct _Maat_compile_rule_t*)data; + struct _Maat_compile_inner_t* compile_rule=(struct _Maat_compile_inner_t*)data; MESA_lqueue_head update_q=(MESA_lqueue_head)user; if(compile_rule->db_c_rule==NULL) { @@ -746,24 +763,26 @@ void EMPTY_FREE(void*p) { return; } -struct _Maat_compile_rule_t * create_compile_rule(int compile_id) +struct _Maat_compile_inner_t * create_compile_rule(int compile_id) { - struct _Maat_compile_rule_t* p=NULL; - p=(struct _Maat_compile_rule_t*)calloc(sizeof(struct _Maat_compile_rule_t),1); + struct _Maat_compile_inner_t* p=NULL; + p=(struct _Maat_compile_inner_t*)calloc(sizeof(struct _Maat_compile_inner_t),1); p->compile_id=compile_id; p->group_cnt=0; + p->table_id=0; + p->group_boundary=1; p->groups=dynamic_array_create(1, 1); pthread_rwlock_init(&(p->rwlock), NULL); return p; } -void destroy_compile_rule(struct _Maat_compile_rule_t * p) +void destroy_compile_rule(struct _Maat_compile_inner_t * p) { int i=0; - struct _Maat_compile_rule_t* p_group=NULL; + struct _Maat_compile_inner_t* p_group=NULL; assert(p->group_cnt==0); - for(i=0;igroup_boundary;i++) { - p_group=(struct _Maat_compile_rule_t*)dynamic_array_read(p->groups,i); + p_group=(struct _Maat_compile_inner_t*)dynamic_array_read(p->groups,i); assert(p_group==NULL); } p->compile_id=-1; @@ -901,7 +920,7 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule) return; } GIE_digest_t* create_digest_rule(int id,short op,unsigned long long origin_len,const char* digest, - short cfds_lvl,struct _Maat_group_rule_t* tag) + short cfds_lvl,struct _Maat_group_inner_t* tag) { GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1); int digest_len=strlen(digest); @@ -1293,133 +1312,176 @@ void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*log update_array=NULL; return; } -struct _Maat_group_rule_t* add_region_to_group(struct _Maat_group_rule_t* group,int region_id,int district_id,int expr_id,enum MAAT_TABLE_TYPE region_type) +struct _Maat_group_inner_t* add_region_to_group(struct _Maat_group_inner_t* group,int table_id,int region_id,int district_id,int expr_id,enum MAAT_TABLE_TYPE region_type) { - struct _Maat_region_rule_t* region_rule=(struct _Maat_region_rule_t*)malloc(sizeof(struct _Maat_region_rule_t)); - region_rule->region_id=region_id; - region_rule->expr_id=expr_id; - region_rule->district_id=district_id; - region_rule->region_type=region_type; - pthread_mutex_lock(&(group->mutex)); - dynamic_array_write(group->region_rules,group->region_boundary,region_rule); - group->region_cnt++; - group->region_boundary++; - pthread_mutex_unlock(&(group->mutex)); - return group; -} -void cancel_last_region_from_group(struct _Maat_group_rule_t* group,int region_id,int expr_id) -{ - struct _Maat_region_rule_t* region_rule=NULL; - pthread_mutex_lock(&(group->mutex)); - group->region_boundary--; - region_rule=(struct _Maat_region_rule_t*)dynamic_array_read(group->region_rules,group->region_boundary); - assert(region_rule->expr_id==expr_id&®ion_rule->region_id==region_id); - free(region_rule); - dynamic_array_write(group->region_rules,group->region_boundary,NULL); - group->region_cnt--; - pthread_mutex_unlock(&(group->mutex)); - return; -} -unsigned int del_region_from_group(struct _Maat_group_rule_t* group,int region_id,unsigned int *output_expr_id,int output_size) -{ - int i=0,j=0; - struct _Maat_region_rule_t* region_rule=NULL; - pthread_mutex_lock(&(group->mutex)); + int i=0; + struct _Maat_region_inner_t* region_rule=NULL; for(i=0;iregion_boundary;i++) { - region_rule=(struct _Maat_region_rule_t*)dynamic_array_read(group->region_rules, i); + region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group->regions, i); if(region_rule==NULL) { continue; } if(region_rule->region_id==region_id) { - dynamic_array_write(group->region_rules, i, NULL); - output_expr_id[j]=region_rule->expr_id; - j++; - assert(j<=output_size); - + break; + } + } + if(i==group->region_boundary)//new region + { + region_rule=(struct _Maat_region_inner_t*)malloc(sizeof(struct _Maat_region_inner_t)); + region_rule->region_id=region_id; + region_rule->expr_id_cnt=1; + region_rule->expr_id_ub=region_rule->expr_id_lb=expr_id; + region_rule->district_id=district_id; + region_rule->table_type=region_type; + region_rule->table_id=table_id; + pthread_mutex_lock(&(group->mutex)); + dynamic_array_write(group->regions,group->region_boundary,region_rule); + group->region_cnt++; + group->region_boundary++; + pthread_mutex_unlock(&(group->mutex)); + } + else + { + assert(expr_id==region_rule->expr_id_ub+1); + region_rule->expr_id_ub=expr_id; + region_rule->expr_id_cnt++; + } + return group; +} +void cancel_last_region_from_group(struct _Maat_group_inner_t* group,int region_id,int expr_id) +{ + struct _Maat_region_inner_t* region_rule=NULL; + pthread_mutex_lock(&(group->mutex)); + region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group->regions,group->region_boundary); + assert(region_rule->expr_id_ub==expr_id&®ion_rule->region_id==region_id); + if(region_rule->expr_id_cnt==1) + { + free(region_rule); + dynamic_array_write(group->regions,group->region_boundary,NULL); + group->region_cnt--; + group->region_boundary--; + } + else + { + region_rule->expr_id_ub--; + region_rule->region_cnt--; + } + pthread_mutex_unlock(&(group->mutex)); + return; +} +unsigned int del_region_from_group(struct _Maat_group_inner_t* group,int region_id,unsigned int *output_expr_id,int output_size) +{ + int i=0,j=0; + struct _Maat_region_inner_t* region_rule=NULL; + pthread_mutex_lock(&(group->mutex)); + for(i=0;iregion_boundary;i++) + { + region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group->regions, i); + if(region_rule==NULL) + { + continue; + } + if(region_rule->region_id==region_id) + { + dynamic_array_write(group->regions, i, NULL); + for(j=0;jexpr_id_cnt;j++) + { + output_expr_id[j]=region_rule->expr_id_lb+j; + } + assert(j<=output_size); + region_rule->region_id=0; free(region_rule); region_rule=NULL; - group->region_cnt--; + break; } } pthread_mutex_unlock(&(group->mutex)); return j; } -int add_group_to_compile(struct _Maat_compile_rule_t*a_compile_rule,struct _Maat_group_rule_t* a_rule_group) +int add_group_to_compile(struct _Maat_compile_inner_t*a_compile_rule,struct _Maat_group_inner_t* a_rule_group) { int i=0,ret=-1; - struct _Maat_group_rule_t* p=NULL; + int write_pos=-1; + struct _Maat_group_inner_t* p=NULL; pthread_rwlock_wrlock(&(a_compile_rule->rwlock)); if(a_compile_rule->db_c_rule!=NULL &&a_compile_rule->group_cnt>=a_compile_rule->db_c_rule->declare_grp_num &&a_compile_rule->db_c_rule->declare_grp_num!=0) { ret=-1; + goto error_out; + } + + for(i=0;igroup_boundary;i++) + { + p=(struct _Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i); + if(p==NULL) + { + write_pos=i; + } + else + { + if(p->group_id==a_rule_group->group_id)//duplicate group + { + ret=-1; + goto error_out; + } + } + } + if(write_pos<0&&a_compile_rule->group_boundary==MAX_EXPR_ITEM_NUM) + { + ret=-1; + goto error_out; + } + if(write_pos<0) + { + write_pos=a_compile_rule->group_boundary; + a_compile_rule->group_boundary++; + } + dynamic_array_write(a_compile_rule->groups,write_pos, a_rule_group); + a_compile_rule->group_cnt++; + a_rule_group->ref_cnt++; + //member group->compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish. + if(a_rule_group->ref_cnt==1&&a_compile_rule->group_cnt==1) + { + a_rule_group->compile_shortcut=a_compile_rule; } else { - for(i=0;icompile_shortcut=NULL; + } + + //update group's shortcut when compile has more than one group. + if(a_compile_rule->group_cnt!=1) + { + for(i=0;igroup_boundary;i++) { - p=(struct _Maat_group_rule_t*)dynamic_array_read(a_compile_rule->groups,i); - if(p==NULL) + p=(struct _Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i); + if(p!=NULL) { - dynamic_array_write(a_compile_rule->groups,i, a_rule_group); - a_compile_rule->group_cnt++; - a_rule_group->ref_cnt++; - //variable compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish. - if(a_rule_group->ref_cnt==1&&a_compile_rule->group_cnt==1) - { - a_rule_group->compile_shortcut=a_compile_rule; - } - else - { - a_rule_group->compile_shortcut=NULL; - } - ret=1; - break; - } - else - { - if(p->group_id==a_rule_group->group_id)//duplicate group - { - ret=-1; - - } - } - } - if(i==MAAT_MAX_EXPR_ITEM_NUM) - { - ret=-1; - } - //update group's shortcut when compile has more than one group. - if(a_compile_rule->group_cnt!=1) - { - for(i=0;igroups,i); - if(p!=NULL) - { - p->compile_shortcut=NULL; - } + p->compile_shortcut=NULL; } } } + ret=1; +error_out: pthread_rwlock_unlock(&(a_compile_rule->rwlock)); return ret; } -struct _Maat_group_rule_t* del_group_from_compile(struct _Maat_compile_rule_t*a_compile_rule,int group_id) +struct _Maat_group_inner_t* del_group_from_compile(struct _Maat_compile_inner_t*a_compile_rule,int group_id) { int i=0; - struct _Maat_group_rule_t* group_rule=NULL; + struct _Maat_group_inner_t* group_rule=NULL; pthread_rwlock_wrlock(&(a_compile_rule->rwlock)); for(i=0;igroups,i); + group_rule=(struct _Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i); if(group_rule==NULL) { continue; @@ -1493,7 +1555,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule int expr_id=0,district_id=-1; scan_rule_t*p_rule=NULL; - struct _Maat_group_rule_t* group_rule=NULL; + struct _Maat_group_inner_t* group_rule=NULL; enum MAAT_CHARSET dst_charset=CHARSET_NONE; char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM]; int key_left_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1},key_right_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1}; @@ -1504,14 +1566,14 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule } int sub_expr_cnt=0; struct op_expr_t *op_expr=NULL; - struct _Maat_group_rule_t* u_para=NULL; + struct _Maat_group_inner_t* u_para=NULL; if(table->table_type==TABLE_TYPE_EXPR_PLUS) { assert(strlen(db_rule->district)>0); district_id=get_district_id(scanner, db_rule->district); } - group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id); + group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id); if(group_rule==NULL) { group_rule=create_group_rule(db_rule->group_id); @@ -1574,7 +1636,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule break; case EXPR_TYPE_REGEX://it's easy,no need to charset convert expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule,db_rule->region_id,district_id,expr_id,TABLE_TYPE_EXPR); + u_para=add_region_to_group(group_rule,table->table_id,db_rule->region_id,district_id,expr_id,TABLE_TYPE_EXPR); if(u_para==NULL) { return -1; @@ -1641,7 +1703,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule break; } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule, db_rule->region_id,district_id,expr_id, table->table_type); + u_para=add_region_to_group(group_rule,table->table_id, db_rule->region_id,district_id,expr_id, table->table_type); if(u_para==NULL)//duplicate { return -1; @@ -1719,7 +1781,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule else { expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule, db_rule->region_id,district_id,expr_id, table->table_type); + u_para=add_region_to_group(group_rule, table->table_id, db_rule->region_id,district_id,expr_id, table->table_type); if(u_para==NULL) { return -1; @@ -1752,13 +1814,13 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule } int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger) { - struct _Maat_group_rule_t* group_rule=NULL; + struct _Maat_group_inner_t* group_rule=NULL; scan_rule_t* p_rule=NULL; struct op_expr_t* op_expr=NULL; - struct _Maat_group_rule_t* u_para=NULL; + struct _Maat_group_inner_t* u_para=NULL; int expr_id=0,district_id=-1; - group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id); + group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id); if(group_rule==NULL) { group_rule=create_group_rule(db_ip_rule->group_id); @@ -1766,7 +1828,7 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule,db_ip_rule->region_id,district_id,expr_id,TABLE_TYPE_IP); + u_para=add_region_to_group(group_rule, table->table_id,db_ip_rule->region_id,district_id,expr_id,TABLE_TYPE_IP); if(u_para==NULL) { return -1; @@ -1784,20 +1846,20 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule } int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger) { - struct _Maat_group_rule_t* group_rule=NULL; + struct _Maat_group_inner_t* group_rule=NULL; scan_rule_t* p_rule=NULL; struct op_expr_t* op_expr=NULL; - struct _Maat_group_rule_t* u_para=NULL; + struct _Maat_group_inner_t* u_para=NULL; int expr_id=0,district_id=-1; - group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id); + group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id); if(group_rule==NULL) { group_rule=create_group_rule(intval_rule->group_id); HASH_add_by_id(scanner->group_hash, intval_rule->group_id, group_rule); } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule,intval_rule->region_id,district_id,expr_id,TABLE_TYPE_INTERVAL); + u_para=add_region_to_group(group_rule, table->table_id,intval_rule->region_id,district_id,expr_id,TABLE_TYPE_INTERVAL); if(u_para==NULL) { return -1; @@ -1815,19 +1877,19 @@ int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* in } int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db_digest_rule,struct _Maat_scanner_t *scanner,void* logger) { - struct _Maat_group_rule_t* group_rule=NULL; + struct _Maat_group_inner_t* group_rule=NULL; GIE_digest_t* digest_rule=NULL; - struct _Maat_group_rule_t* u_para=NULL; + struct _Maat_group_inner_t* u_para=NULL; int expr_id=0,district_id=-1; - group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); + group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); if(group_rule==NULL) { group_rule=create_group_rule(db_digest_rule->group_id); HASH_add_by_id(scanner->group_hash, db_digest_rule->group_id, group_rule); } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule,db_digest_rule->region_id,expr_id,district_id,TABLE_TYPE_DIGEST); + u_para=add_region_to_group(group_rule,table->table_id,db_digest_rule->region_id,expr_id,district_id,TABLE_TYPE_DIGEST); if(u_para==NULL) { return -1; @@ -1845,10 +1907,10 @@ int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id, int i=0; unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0}; int expr_num=0; - struct _Maat_group_rule_t* group_rule=NULL; + struct _Maat_group_inner_t* group_rule=NULL; struct op_expr_t* op_expr=NULL; GIE_digest_t* digest_rule=NULL; - group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(maat_scanner->group_hash, group_id); + group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(maat_scanner->group_hash, group_id); if(group_rule==NULL) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , @@ -1909,18 +1971,20 @@ int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id, } int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) { - struct _Maat_group_rule_t* group_rule=NULL; - struct _Maat_compile_rule_t*compile_rule=NULL; + struct _Maat_group_inner_t* group_rule=NULL; + struct _Maat_compile_inner_t*compile_rule=NULL; int ret=0; - group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); + group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); if(group_rule==NULL) { group_rule=create_group_rule(db_group_rule->group_id); + group_rule->table_id=table->table_id; + ret=HASH_add_by_id(scanner->group_hash, db_group_rule->group_id,group_rule); assert(ret>=0); } - compile_rule=(struct _Maat_compile_rule_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id); + compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id); if(compile_rule==NULL) { compile_rule=create_compile_rule(db_group_rule->compile_id); @@ -1942,9 +2006,9 @@ int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_g } void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) { - struct _Maat_compile_rule_t*compile_rule=NULL; - struct _Maat_group_rule_t* group_rule=NULL; - compile_rule=(struct _Maat_compile_rule_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id); + struct _Maat_compile_inner_t*compile_rule=NULL; + struct _Maat_group_inner_t* group_rule=NULL; + compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id); if(compile_rule==NULL) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , @@ -1980,12 +2044,13 @@ void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_ } int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) { - struct _Maat_compile_rule_t *compile_rule=NULL; + struct _Maat_compile_inner_t *compile_rule=NULL; struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head); - compile_rule=(struct _Maat_compile_rule_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id); + compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id); if(compile_rule==NULL) { compile_rule=create_compile_rule(p_maat_rule_head->config_id); + compile_rule->table_id=table->table_id; HASH_add_by_id(scanner->compile_hash,p_maat_rule_head->config_id,compile_rule); } if(compile_rule->db_c_rule!=NULL)//duplicate config @@ -1998,8 +2063,8 @@ int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* } int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) { - struct _Maat_compile_rule_t *compile_rule=NULL; - compile_rule=(struct _Maat_compile_rule_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id); + struct _Maat_compile_inner_t *compile_rule=NULL; + compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id); if(compile_rule==NULL) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , diff --git a/src/entry/Maat_rule_internal.h b/src/entry/Maat_rule_internal.h index 579edaa..19ad084 100644 --- a/src/entry/Maat_rule_internal.h +++ b/src/entry/Maat_rule_internal.h @@ -140,7 +140,6 @@ struct _head_Maat_rule_t }; struct db_compile_rule_t { -// Maat_rule_t m_rule_head; struct _head_Maat_rule_t m_rule_head;// fix len of Maat_rule_t char* service_defined; long long effective_range; @@ -163,14 +162,16 @@ struct op_expr_t int rule_type; }; -struct _Maat_region_rule_t +struct _Maat_region_inner_t { int region_id; - int expr_id; + int expr_id_cnt; int district_id; + int expr_id_lb; + int expr_id_ub; enum MAAT_TABLE_TYPE region_type; }; -struct _Maat_group_rule_t +struct _Maat_group_inner_t { int group_id; int region_boundary; @@ -180,10 +181,11 @@ struct _Maat_group_rule_t void* compile_shortcut; pthread_mutex_t mutex; }; -struct _Maat_compile_rule_t +struct _Maat_compile_inner_t { struct db_compile_rule_t *db_c_rule; dynamic_array_t *groups; + int group_boundary; int group_cnt; int compile_id;//equal to db_c_rule->m_rule.config_id pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read @@ -374,6 +376,7 @@ struct _Maat_feather_t int cmd_num; struct _Maat_cmd_t* cmd_qhead, *cmd_qtail; pthread_mutex_t redis_write_lock; //protect redis_write_ctx + long long base_rgn_seq,base_grp_seq; //for stat>>>> screen_stat_handle_t stat_handle; int total_stat_id; @@ -431,7 +434,13 @@ void maat_stat_init(struct _Maat_feather_t* feather); void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num); void maat_stat_output(struct _Maat_feather_t* feather); - +void redis_monitor_traverse(unsigned int version,redisContext *c, + void (*start)(unsigned int ,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para + void (*update)(const char* ,const char*,void* ),//table name ,line ,u_para + void (*finish)(void*),//u_para + void* u_para, + const unsigned char* dec_key, + _Maat_feather_t* feather); #endif