diff --git a/inc/Maat_command.h b/inc/Maat_command.h index 1d824cb..b749175 100644 --- a/inc/Maat_command.h +++ b/inc/Maat_command.h @@ -1,5 +1,5 @@ -#ifndef H_MAAT_RULE_H_INCLUDE -#define H_MAAT_RULE_H_INCLUDE +#ifndef H_MAAT_COMMAND_H_INCLUDE +#define H_MAAT_COMMAND_H_INCLUDE #ifndef __cplusplus #error("This file should be compiled with C++ compiler") #endif @@ -59,10 +59,10 @@ struct Maat_rgn_str_t struct Maat_rgn_addr_t { enum MAAT_ADDR_TYPE addr_type; - char* src_ip; - char* mask_src_ip; - char* dst_ip; - char* mask_dst_ip; + const char* src_ip; + const char* mask_src_ip; + const char* dst_ip; + const char* mask_dst_ip; unsigned short src_port; unsigned short mask_src_port; unsigned short dst_port; @@ -78,7 +78,7 @@ struct Maat_rgn_intv_t struct Maat_rgn_digest_t { unsigned long long orgin_len; - char* digest_string; + const char* digest_string; short confidence_degree; }; struct Maat_rgn_sim_t @@ -88,7 +88,7 @@ struct Maat_rgn_sim_t }; struct Maat_region_t { - char* table_name; + const char* table_name; int region_id; //Any, maat will assigned one. enum MAAT_REGION_TYPE region_type; union @@ -113,9 +113,8 @@ struct Maat_command_t int group_num; // for MAAT_OP_DEL, Any. struct Maat_group_t* groups;// for MAAT_OP_DEL, SET to NULL. }; -struct Maat_command_t* Maat_create_comand(const struct Maat_rule_t* rule, const char*table_name, int group_num); -int Maat_set_command(struct Maat_command_t* cmd,int which_group,const struct Maat_region_t* region); - +struct Maat_command_t* Maat_create_comand(const struct Maat_rule_t* rule, int group_num); +void Maat_cmd_add_region(struct Maat_command_t* cmd,int which_group,const struct Maat_region_t* region); void Maat_free_command(struct Maat_command_t* cmd); int Maat_format_command(struct Maat_command_t* cmd, char* buffer, int size); diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index bca8d49..1970332 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -185,7 +185,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int int exprid2region_id(struct _Maat_group_inner_t* group_rule,int expr_id,int* district_id) { - int i=0,j=0,region_id=-1; + int i=0,region_id=-1; struct _Maat_region_inner_t* region_rule=NULL; assert(group_rule->group_id>=0); pthread_mutex_lock(&(group_rule->mutex)); @@ -442,11 +442,11 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* if(feather->p_table_info[i]->table_type==TABLE_TYPE_GROUP) { feather->GROUP_MODE_ON=1; - memcpy(feather->group_tn,feather->p_table_info[i].table_name[0],sizeof(feather->group_tn)); + strncpy(feather->group_tn,feather->p_table_info[i]->table_name[0],sizeof(feather->group_tn)); } if(feather->p_table_info[i]->table_type==TABLE_TYPE_COMPILE) { - memcpy(feather->compile_tn,feather->p_table_info[i].table_name[0],sizeof(feather->compile_tn)); + strncpy(feather->compile_tn,feather->p_table_info[i]->table_name[0],sizeof(feather->compile_tn)); } for(j=0;jp_table_info[i]->conj_cnt;j++) { @@ -526,8 +526,9 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo memcpy(_feather->inc_dir,(const char*)value,size); break; case MAAT_OPT_JSON_FILE_PATH: - ret=json2iris((const char*)value, _feather->full_dir + ret=json2iris((const char*)value ,_feather->compile_tn,_feather->group_tn + ,_feather->full_dir ,sizeof(_feather->full_dir),_feather->logger); if(ret<0) { @@ -568,21 +569,21 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo (const char*)value); break; case MAAT_OPT_DECRYPT_KEY: - if(size>sizeof(_feather->decrypt_key)) + if((size_t)size>sizeof(_feather->decrypt_key)) { return -1; } memcpy(_feather->decrypt_key,value,size); break; case MAAT_OPT_REDIS_IP: - if(size>sizeof(_feather->redis_ip)) + if((size_t)size>sizeof(_feather->redis_ip)) { return -1; } memcpy(_feather->redis_ip,value,size); break; case MAAT_OPT_REDIS_PORT: - if(size!=sizeof(unsigned short)) + if((size_t)size!=sizeof(unsigned short)) { return -1; } diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 10e81a9..0426759 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -1,6 +1,10 @@ #include "Maat_command.h" +#include "Maat_rule.h" +#include "Maat_rule_internal.h" #include "config_monitor.h" +#include "map_str2int.h" #include "hiredis.h" +#include #include #include #include @@ -20,16 +24,17 @@ struct serial_rule_t //rm= Redis Maat char* table_line; }; -struct _Maat_cmd_t +struct _Maat_cmd_inner_t { struct Maat_command_t cmd; enum MAAT_OPERATION op; int ref_cnt; - struct _Maat_cmd_t* next; + int region_size[MAX_EXPR_ITEM_NUM]; + struct _Maat_cmd_inner_t* next; }; enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p) { - enum MAAT_TABLE_TYPE ret=0; + enum MAAT_TABLE_TYPE ret=TABLE_TYPE_IP; switch(p->region_type) { case REGION_IP: @@ -59,6 +64,59 @@ enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p) } return ret; } +void invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) +{ + unsigned int offset=0; + unsigned int i=0,j=0; + switch(type) + { + case TABLE_TYPE_EXPR: + offset=7; + break; + case TABLE_TYPE_IP: + offset=14; + break; + case TABLE_TYPE_COMPILE: + offset=8; + break; + case TABLE_TYPE_PLUGIN: + if(valid_column_seq<0) + { + return; + } + offset=(unsigned int)valid_column_seq; + break; + case TABLE_TYPE_INTERVAL: + offset=5; + break; + case TABLE_TYPE_DIGEST: + offset=6; + break; + case TABLE_TYPE_EXPR_PLUS: + offset=8; + break; + case TABLE_TYPE_GROUP: + offset=3; + break; + default: + assert(0); + } + for(i=0;iip_rule.region_id + ,p->region_id ,group_id ,p->ip_rule.addr_type ,p->ip_rule.src_ip @@ -127,21 +185,21 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int assert(rettable_line!=NULL) { free(rule->table_line); rule->table_line=NULL; } - free(rule); + memset(rule,0,sizeof(struct serial_rule_t)); return; } void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,const char* table_name,const char* line) { rule->op=op; rule->rule_id=rule_id; - assert(srtlen(table_name)table_name)); + assert(strlen(table_name)table_name)); memcpy(rule->table_name,table_name,strlen(table_name)); if(line!=NULL) { @@ -151,22 +209,19 @@ void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_ } int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** list,void* logger, unsigned int* new_version,int *update_type) { - redisReply* ctrl_reply=NULL,*data_reply=NULL; + redisReply* reply=NULL,*tmp_reply=NULL; char err_buff[256]; - char op_str[4],table_name[256]; - int rule_id; - long long version_in_redis=0; - int i=0,ret=0,retry=0; + char op_str[4]; + long long version_in_redis=0,nearest_rule_version=0; + int ret=0,retry=0; + unsigned int i=0; struct serial_rule_t *s_rule=NULL; - if(version==0) - { - goto FULL_UPDATE; - } + while(retry<1) { - ctrl_reply=(redisReply*)redisCommand(c, "GET MAAT_VERSION"); - if(ctrl_reply!=NULL) + reply=(redisReply*)redisCommand(c, "GET MAAT_VERSION"); + if(reply!=NULL) { break; } @@ -187,56 +242,60 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** return 0; } } - - assert(ctrl_reply->type==REDIS_REPLY_INTEGER); - assert(ctrl_reply->integer>=version); - if(ctrl_reply->integer==version) - { - freeReplyObject(ctrl_reply); + assert(reply->type==REDIS_REPLY_STRING); + version_in_redis=atoll(reply->str); + assert(version_in_redis>=version); + freeReplyObject(reply); + if(version_in_redis==version) + { return 0; } - version_in_redis=ctrl_reply->integer; *new_version=version_in_redis; - freeReplyObject(ctrl_reply); + + if(version==0) + { + goto FULL_UPDATE; + } + + //Returns all the elements in the sorted set at key with a score that version < score <= version_in_redis. //The elements are considered to be ordered from low to high scores(version). - ctrl_reply=(redisReply*)redisCommand(c, "ZRANGE MAAT_UPDATE_STATUS (%d %d",version,version_in_redis); - if(ctrl_reply==NULL) + reply=(redisReply*)redisCommand(c, "ZRANGE MAAT_UPDATE_STATUS (%d %d",version,version_in_redis); + if(reply==NULL) { __redis_strerror_r(errno,err_buff,sizeof(err_buff)); MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, "GET MAAT_UPDATE_STATUS failed %s.",err_buff); return 0; } - assert(ctrl_reply->type==REDIS_REPLY_ARRAY); - data_reply=(redisReply*)redisCommand(c, "ZRANK MAAT_UPDATE_STATUS %s",ctrl_reply->element[0]->str); - if(data_reply->integer!=version+1) + assert(reply->type==REDIS_REPLY_ARRAY); + tmp_reply=(redisReply*)redisCommand(c, "ZRANK MAAT_UPDATE_STATUS %s",reply->element[0]->str); + nearest_rule_version=tmp_reply->integer; + freeReplyObject(tmp_reply); + tmp_reply=NULL; + if(nearest_rule_version!=version+1) { MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, - "Noncontinuous VERSION Redis: %lld MAAT: %d.",data_reply->integer,version); - freeReplyObject(data_reply); - data_reply=NULL; + "Noncontinuous VERSION Redis: %lld MAAT: %d.",tmp_reply->integer,version); + goto FULL_UPDATE; } else { MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, "Inc Update form version %d to %lld.",version,version_in_redis); - } - freeReplyObject(data_reply); - data_reply=NULL; - s_rule=(struct serial_rule_t*)calloc(ctrl_reply->elements,sizeof(struct serial_rule_t)); - for(i=0;ielements;i++) + s_rule=(struct serial_rule_t*)calloc(reply->elements,sizeof(struct serial_rule_t)); + for(i=0;ielements;i++) { - assert(ctrl_reply->element[i]->type==REDIS_REPLY_STRING); - ret=sscanf(ctrl_reply->element[i]->str,"%s,%s,%d",op_str,s_rule.table_name,&(s_rule[i].rule_id)); + assert(reply->element[i]->type==REDIS_REPLY_STRING); + ret=sscanf(reply->element[i]->str,"%[^:]:%[^,],%d",op_str,s_rule[i].table_name,&(s_rule[i].rule_id)); assert(ret==3); - if(strncmp(op_str,"ADD")==0) + if(strncmp(op_str,"ADD",strlen("ADD"))==0) { s_rule[i].op=MAAT_OP_ADD; } - else if(strncmp(op_str,"DEL")==0) + else if(strncmp(op_str,"DEL",strlen("DEL"))==0) { s_rule[i].op=MAAT_OP_DEL; } @@ -247,305 +306,42 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** } *list=s_rule; *update_type=CM_UPDATE_TYPE_INC; - freeReplyObject(ctrl_reply); + freeReplyObject(reply); return i; FULL_UPDATE: MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, "Initiate full udpate from version %d to %lld.",version,version_in_redis); - data_reply=(redisReply*)redisCommand(c, "KEYS EFFECTIVE_RULE:*"); - assert(data_reply->type==REDIS_REPLY_ARRAY); - s_rule=(struct serial_rule_t*)calloc(ctrl_reply->elements,sizeof(struct serial_rule_t)); - for(i=0;ielements;i++) + reply=(redisReply*)redisCommand(c, "KEYS EFFECTIVE_RULE:*"); + assert(reply->type==REDIS_REPLY_ARRAY); + s_rule=(struct serial_rule_t*)calloc(reply->elements,sizeof(struct serial_rule_t)); + for(i=0;ielements;i++) { - assert(ctrl_reply->element[i]->type==REDIS_REPLY_STRING); - ret=sscanf(ctrl_reply->element[i]->str,"EFFECTIVE_RULE:%s,%d",s_rule[i].table_name,&(s_rule[i].rule_id)); - s_rule[i].op=1; + assert(reply->element[i]->type==REDIS_REPLY_STRING); + ret=sscanf(reply->element[i]->str,"%*[^:]:%[^,],%d",s_rule[i].table_name,&(s_rule[i].rule_id)); + s_rule[i].op=MAAT_OP_ADD; assert(ret==2); } *list=s_rule; *update_type=CM_UPDATE_TYPE_FULL; - freeReplyObject(ctrl_reply); - ctrl_reply=NULL; - freeReplyObject(data_reply); - data_reply=NULL; + + freeReplyObject(reply); + reply=NULL; return i; } -void redis_monitor_traverse(unsigned int version,redisContext *c, - void (*start)(unsigned int ,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para - void (*update)(const char* ,const char*,void* ),//table name ,line ,u_para - void (*finish)(void*),//u_para - void* u_para, - const unsigned char* dec_key, - _Maat_feather_t* feather) +int _wrap_redisGetReply(redisContext *c, redisReply **reply) { - redisReply* ctrl_reply=NULL,*data_reply=NULL; - int rule_num=0,i=0,table_id=0; - int ret=0; - struct serial_rule_t* rule_list=NULL; - int update_type=0; - unsigned int new_version=0; - enum MAAT_TABLE_TYPE table_type=0; - void* logger=feather->logger; - rule_num=get_rm_key_list(version, c, &rule_list, logger,&new_version, &update_type); - if(rule_num==0) - { - return; - } - for(i=0;ielements==rule_num); - start(new_version,update_type); - for(i=0;ielements;i++) - { - if(rule_list[i].op==MAAT_OP_DEL) - { - ret=feather->map_tablename2id(rule_list[i].table_name,&table_id); - if(ret<0) - { - continue; - } - table_type=feather->p_table_info[table_id].table_type; - invalidate_line(data_reply->element[i].str,table_type); - } - update(rule_list[i].table_name,data_reply->element[i].str,u_para); - } - finish(u_para); - freeReplyObject(data_reply); - free(rule_list); - rule_list=NULL; - return; + return redisGetReply(c, (void **)reply); } -struct Maat_command_t* Maat_create_comand(const struct Maat_rule_t* rule, int group_num) +redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...) { - struct _Maat_cmd_t* _cmd=(struct _Maat_cmd_t*)calloc(sizeof(struct _Maat_cmd_t),1); - memcpy(&(_cmd->cmd.compile),rule,sizeof(_cmd->cmd.compile)); - _cmd->ref_cnt=1; - _cmd->cmd.group_num=group_num; - _cmd->cmd.groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),group_num); - - return (struct Maat_command_t*)_cmd; + va_list ap; + void *reply = NULL; + va_start(ap,format); + reply = redisvCommand(c,format,ap); + va_end(ap); + return (redisReply *)reply; } -void Maat_cmd_set_group(struct Maat_command_t* cmd,int which_group,int region_num,const char* group_name) -{ - assert(which_groupgroup_num); - if(group_name!=NULL) - { - if(cmd->groups[which_group].group_name!=NULL) - { - free(cmd->groups[which_group].group_name); - } - cmd->groups[which_group].group_name=_maat_strdup(group_name); - } - cmd->groups->regions=(struct Maat_region_t*)calloc(sizeof(struct Maat_region_t),region_num); - cmd->groups->region_num=region_num; - return; -} -void Maat_copy_region(struct Maat_region_t* dst,const struct Maat_region_t* src) -{ - memcpy(dst,src,sizeof(struct Maat_region_t)); - if(src->table_name!=NULL) - { - dst->table_name=_maat_strdup(src->table_name) - } - switch(dst->region_type) - { - case REGION_IP: - dst->ip_rule.src_ip=_maat_strdup(src->ip_rule.src_ip); - dst->ip_rule.mask_src_ip=_maat_strdup(src->ip_rule.mask_src_ip); - dst->ip_rule.dst_ip=_maat_strdup(src->ip_rule.dst_ip); - dst->ip_rule.mask_src_ip=_maat_strdup(src->ip_rule.mask_src_ip); - break; - case REGION_EXPR: - dst->expr_rule.keywords=_maat_strdup(src->expr_rule.keywords); - dst->expr_rule.district=_maat_strdup(src->expr_rule.district); - break; - case REGION_INTERVAL: - break; - case REGION_DIGEST: - dst->digest_rule.digest_string=_maat_strdup(src->digest_rule.digest_string); - break; - case REGION_SIMILARITY: - assert(0); - break; - default: - assert(0); - } - return; -} -void Maat_empty_region(struct Maat_region_t* p) -{ - free(p->table_name); - p->table_name=NULL; - switch(p->region_type) - { - case REGION_IP: - free(p->ip_rule.src_ip); - free(p->ip_rule.mask_src_ip); - free(p->ip_rule.dst_ip); - free(p->ip_rule.mask_src_ip); - break; - case REGION_EXPR: - free(p->expr_rule.keywords); - free(p->expr_rule.district); - break; - case REGION_INTERVAL: - break; - case REGION_DIGEST: - free(p->digest_rule.digest_string); - break; - case REGION_SIMILARITY: - assert(0); - break; - default: - assert(0); - } - memset(p,0,sizeof(struct Maat_region_t)); - return; - -} -void Maat_cmd_set_region(struct Maat_command_t* cmd,int which_group,int which_region,const struct Maat_region_t* region) -{ - struct Maat_region_t* p=NULL; - assert(which_groupgroup_num); - assert(which_regiongroups[which_group].region_num); - assert(region->table_name!=NULL); - p=&(cmd->groups[which_group].regions[which_region]); - Maat_copy_region(p, region); - return; -} - -void Maat_free_command(struct Maat_command_t* cmd) -{ - struct _Maat_cmd_t* _cmd=(struct _Maat_cmd_t*)cmd; - int i=0,j=0; - _cmd->ref_cnt--; - if(_cmd->ref_cnt>0) - { - return; - } - for(i=0;i<_cmd->cmd.group_num;i++) - { - for(j=0;jcmd.groups); - _cmd->cmd.groups=NULL; - _cmd->next=NULL; - free(_cmd); - return; -} -int Maat_format_command(struct Maat_command_t* rule, char* buffer, int size) -{ - //TODO - return 0; -} -int Maat_command(Maat_feather_t feather,struct Maat_command_t* raw_rule,enum MAAT_OPERATION op) -{ - int ret=0; - ret=Maat_append_command(feather,raw_rule,op); - if(ret<0) - { - return ret; - } - ret=Maat_commit_command(feather); - return ret; -} - -int Maat_append_command(Maat_feather_t feather,struct Maat_command_t* cmd,enum MAAT_OPERATION op) -{ - _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct _Maat_cmd_t* _cmd=(struct _Maat_cmd_t*)cmd; - int i=0, ret=0; - _cmd->ref_cnt++; - _cmd->op=op; - assert(op==MAAT_OP_DEL||op==MAAT_OP_ADD); - if(op==MAAT_OP_DEL) - { - ret=reconstruct_cmd(_feather, _cmd) - } - else - { - ret=fix_table_name(_feather, cmd); - } - if(ret<0) - { - return -1; - } - if(_feather->cmd_q_cnt==0) - { - assert(_feather->cmd_qtail==_feather->cmd_qhead==NULL); - _feather->cmd_qhead=_feather->cmd_qtail=_cmd; - } - else - { - _feather->cmd_qtail->next=_cmd; - _feather->cmd_qtail=_cmd; - } - _feather->cmd_q_cnt++; - return; -} -//TODO: support plugin rule command. -void invalidate_line(char* line, enum MAAT_TABLE_TYPE type) -{ - int offset=0; - int i=0,j=0; - switch(type) - { - case TABLE_TYPE_EXPR: - offset=7; - break; - case TABLE_TYPE_IP: - offset=14; - break; - case TABLE_TYPE_COMPILE: - offset=8; - break; - case TABLE_TYPE_PLUGIN: - assert(0); - break; - case TABLE_TYPE_INTERVAL: - offset=5; - break; - case TABLE_TYPE_DIGEST: - offset=6; - break; - case TABLE_TYPE_EXPR_PLUS: - offset=8; - break; - case TABLE_TYPE_GROUP: - offset=3; - break; - default: - assert(0); - } - for(i=0;iop==MAAT_OP_ADD) { *new_region_cnt+=cmd->groups[i].region_num; - *new_group_cnt++; + (*new_group_cnt)++; } } return serial_num; } -int reconstruct_cmd(struct _Maat_feather_t feather, struct _Maat_cmd_t* _cmd) +int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _cmd) { - int ret=0; int i=0,j=0,grp_idx=0; - long long group_ref_cnt=0; struct Maat_command_t* cmd=&(_cmd->cmd); struct Maat_group_t* group_cmd=NULL; struct Maat_region_t* region_cmd=NULL; @@ -616,10 +410,10 @@ int reconstruct_cmd(struct _Maat_feather_t feather, struct _Maat_cmd_t* _cmd) continue; } region_cmd=&(group_cmd->regions[group_cmd->region_num]); - region_cmd->table_name=_maat_strdup(feather->p_table_info[region_inner->table_id].table_name[0]); + region_cmd->table_name=_maat_strdup(feather->p_table_info[region_inner->table_id]->table_name[0]); region_cmd->region_id=region_inner->table_id; - //NOTICE: a table_type stored in region_type. - region_cmd->region_type=region_inner->table_type; + //NOTICE: region_type only avilable when OP_ADD, + region_cmd->region_type=REGION_EXPR; group_cmd->region_num++; } grp_idx++; @@ -627,54 +421,18 @@ int reconstruct_cmd(struct _Maat_feather_t feather, struct _Maat_cmd_t* _cmd) return 0; } -int build_serial_rule_from_redis(_Maat_feather_t *feather,redisContext *ctx,struct Maat_command_t* cmd,struct serial_rule_t* list, int size) -{ - struct Maat_group_t* p_group=NULL; - struct Maat_region_t* p_region=NULL; - enum MAAT_TABLE_TYPE table_type=0; - redisReply* data_reply=NULL; - int rule_num=0; - int i=0,j=0; - - set_serial_rule(list+rule_num,MAAT_OP_DEL,cmd->compile.config_id,feather->compile_tn,NULL); - rule_num++; - for(i=0;igroup_num;i++) - { - p_group=&(cmd->groups[i]); - - set_serial_rule(list+rule_num,MAAT_OP_DEL,p_group->group_id,feather->group_tn,NULL); - rule_num++; - freeReplyObject(data_reply); - for(j=0;jregion_num;j++) - { - p_region=&(p_group->regions[j]); - data_reply=redisCommand(ctx,"GET %s:%s,%d",rm_key_prefix[MAAT_OP_ADD] - ,p_region->table_name - ,p_region->region_id); - table_type=type_region2table(p_region); - invalidate_line(data_reply->str, (enum MAAT_TABLE_TYPE)(p_region->region_type)); - set_serial_rule(list+rule_num,MAAT_OP_DEL,p_region->region_id,p_region->table_name,data_reply->str); - rule_num++; - freeReplyObject(data_reply); - } - } - assert(rule_numcmd); enum MAAT_OPERATION op=_cmd->op; - redisContext* ctx=feather->redis_write_ctx; - - redisReply* data_reply=NULL; + int rule_num=0,i=0,j=0; p_m_rule=&(cmd->compile); char line[1024]; - if(op===MAAT_OP_ADD) + if(op==MAAT_OP_ADD) { snprintf(line,sizeof(line),"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t1\t%d",p_m_rule->config_id ,p_m_rule->service_id @@ -694,7 +452,7 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_t* _cmd,struct s for(i=0;cmd->group_num;i++) { p_group=&(cmd->groups[i]); - if(op===MAAT_OP_ADD) + if(op==MAAT_OP_ADD) { p_group->group_id=feather->base_grp_seq; feather->base_grp_seq++; @@ -713,18 +471,19 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_t* _cmd,struct s } for(j=0;jregion_num;j++) { - if(op===MAAT_OP_ADD) + p_region=&(p_group->regions[j]); + if(op==MAAT_OP_ADD) { - p_group->regions[j].region_id=feather->base_rgn_seq; + p_region->region_id=feather->base_rgn_seq; feather->base_rgn_seq++; - serialize_region(p_group->regions+j, p_group->group_id, line, sizeof(line)); + serialize_region(p_region, p_group->group_id, line, sizeof(line)); set_serial_rule(list+rule_num,MAAT_OP_ADD - ,p_group->regions[j].region_id,p_group->regions[j].table_name,line); + ,p_region->region_id,p_region->table_name,line); } else { set_serial_rule(list+rule_num,MAAT_OP_DEL - ,p_group->regions[j].region_id,p_group->regions[j].table_name,NULL); + ,p_region->region_id,p_region->table_name,NULL); } rule_num++; @@ -735,10 +494,10 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_t* _cmd,struct s } int mr_transaction_success(redisReply* data_reply) { - int i=0; + unsigned int i=0; for(i=0;ielements;i++) { - if(data_reply->element[i].type==REDIS_REPLY_NIL) + if(data_reply->element[i]->type==REDIS_REPLY_NIL) { return 0; } @@ -748,14 +507,18 @@ int mr_transaction_success(redisReply* data_reply) int fix_table_name(_Maat_feather_t* feather,struct Maat_command_t* cmd) { int i=0,j=0,ret=0; - char *table_name=NULL; + const char *table_name=NULL; int table_id=0; + struct Maat_group_t* p_group=NULL; + struct Maat_region_t* p_region=NULL; enum MAAT_TABLE_TYPE table_type; for(i=0;igroup_num;i++) { - for(j=0;jgroups[i].region_num;j++) + p_group=&(cmd->groups[i]); + for(j=0;jregion_num;j++) { - table_name=cmd->groups[i].regions[j].table_name; + p_region=&(p_group->regions[j]); + table_name=p_region->table_name; ret=map_str2int(feather->map_tablename2id, table_name, &table_id); if(ret<0) { @@ -765,34 +528,274 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_command_t* cmd) return -1; } - table_type=type_region2table(&(cmd->groups[i].regions[j])); - if(table_type!=feather->p_table_info[i].table_type) + table_type=type_region2table(p_region); + if(table_type!=feather->p_table_info[i]->table_type) { MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module ,"Table %s not support region type %d of Maat_command_t[%d]->group[%d]->region[%d]." ,table_name - ,cmd->groups[i].regions[j].region_type + ,p_region->region_type ,cmd->compile.config_id,i,j); return -1; } - free(cmd->groups[i].regions[j].table_name); - cmd->groups[i].regions[j].table_name=_maat_strdup(feather->p_table_info[i].table_name[0]); + free((char*)p_region->table_name); + p_region->table_name=_maat_strdup(feather->p_table_info[i]->table_name[0]); } } return 0; } +void redis_monitor_traverse(unsigned int version,redisContext *c, + void (*start)(unsigned int ,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para + void (*update)(const char* ,const char*,void* ),//table name ,line ,u_para + void (*finish)(void*),//u_para + void* u_para, + const unsigned char* dec_key, + _Maat_feather_t* feather) +{ + redisReply* data_reply=NULL; + unsigned int rule_num=0,i=0; + int table_id=0; + int ret=0; + char redis_cmd[256]; + struct serial_rule_t* rule_list=NULL; + int update_type=0; + unsigned int new_version=0; + enum MAAT_TABLE_TYPE table_type; + void* logger=feather->logger; + rule_num=get_rm_key_list(version, c, &rule_list, logger,&new_version, &update_type); + if(rule_num==0) + { + return; + } + for(i=0;imap_tablename2id,rule_list[i].table_name,&table_id); + if(ret<0)//Unrecognized table. + { + continue; + } + table_type=feather->p_table_info[table_id]->table_type; + invalidate_line(data_reply->str,table_type,feather->p_table_info[table_id]->valid_flag_column); + } + update(rule_list[i].table_name,data_reply->str,u_para); + freeReplyObject(data_reply); + } + finish(u_para); + // no need to calll empty_serial_rules + free(rule_list); + rule_list=NULL; + return; +} + +void Maat_copy_region(struct Maat_region_t* dst,const struct Maat_region_t* src) +{ + memcpy(dst,src,sizeof(struct Maat_region_t)); + if(src->table_name!=NULL) + { + dst->table_name=_maat_strdup(src->table_name); + } + switch(dst->region_type) + { + case REGION_IP: + dst->ip_rule.src_ip=_maat_strdup(src->ip_rule.src_ip); + dst->ip_rule.mask_src_ip=_maat_strdup(src->ip_rule.mask_src_ip); + dst->ip_rule.dst_ip=_maat_strdup(src->ip_rule.dst_ip); + dst->ip_rule.mask_src_ip=_maat_strdup(src->ip_rule.mask_src_ip); + break; + case REGION_EXPR: + dst->expr_rule.keywords=_maat_strdup(src->expr_rule.keywords); + dst->expr_rule.district=_maat_strdup(src->expr_rule.district); + break; + case REGION_INTERVAL: + break; + case REGION_DIGEST: + dst->digest_rule.digest_string=_maat_strdup(src->digest_rule.digest_string); + break; + case REGION_SIMILARITY: + assert(0); + break; + default: + assert(0); + } + return; +} +void Maat_empty_region(struct Maat_region_t* p) +{ + free((char*)p->table_name); + p->table_name=NULL; + switch(p->region_type) + { + case REGION_IP: + free((char*)p->ip_rule.src_ip); + free((char*)p->ip_rule.mask_src_ip); + free((char*)p->ip_rule.dst_ip); + free((char*)p->ip_rule.mask_src_ip); + break; + case REGION_EXPR: + free((char*)p->expr_rule.keywords); + free((char*)p->expr_rule.district); + break; + case REGION_INTERVAL: + break; + case REGION_DIGEST: + free((char*)p->digest_rule.digest_string); + break; + case REGION_SIMILARITY: + assert(0); + break; + default: + assert(0); + } + memset(p,0,sizeof(struct Maat_region_t)); + return; + +} +struct Maat_command_t* Maat_create_comand(const struct Maat_rule_t* rule, int group_num) +{ + int i=0; + struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t*)calloc(sizeof(struct _Maat_cmd_inner_t),1); + memcpy(&(_cmd->cmd.compile),rule,sizeof(_cmd->cmd.compile)); + _cmd->ref_cnt=1; + _cmd->cmd.group_num=group_num; + _cmd->cmd.groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),group_num); + for(i=0;icmd.groups[i].regions=(struct Maat_region_t*)calloc(sizeof(struct Maat_region_t),1); + _cmd->region_size[i]=1; + } + return (struct Maat_command_t*)_cmd; +} +void Maat_cmd_set_group(struct Maat_command_t* cmd,int which_group,const char* group_name) +{ + assert(which_groupgroup_num); + + if(cmd->groups[which_group].group_name!=NULL) + { + free(cmd->groups[which_group].group_name); + } + cmd->groups[which_group].group_name=_maat_strdup(group_name); + + return; +} +void Maat_cmd_add_region(struct Maat_command_t* cmd,int which_group,const struct Maat_region_t* region) +{ + struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t*)cmd; + struct Maat_region_t* dst=NULL; + struct Maat_group_t* p_group=NULL; + p_group=&(cmd->groups[which_group]); + assert(which_groupgroup_num); + assert(region->table_name!=NULL); + if(p_group->region_num==_cmd->region_size[which_group]) + { + _cmd->region_size[which_group]*=2; + p_group->regions=(struct Maat_region_t*)realloc(p_group->regions,sizeof(struct Maat_region_t)*_cmd->region_size[which_group]); + } + dst=&(p_group->regions[p_group->region_num]); + p_group->region_num++; + Maat_copy_region(dst, region); + return; +} + +void Maat_free_command(struct Maat_command_t* cmd) +{ + struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t*)cmd; + int i=0,j=0; + _cmd->ref_cnt--; + if(_cmd->ref_cnt>0) + { + return; + } + for(i=0;igroup_num;i++) + { + for(j=0;jgroups[i].region_num;j++) + { + Maat_empty_region(&(cmd->groups[i].regions[j])); + } + free(cmd->groups[i].group_name); + free(cmd->groups[i].regions); + cmd->groups[i].regions=NULL; + } + free(cmd->groups); + cmd->groups=NULL; + _cmd->next=NULL; + free(_cmd); + return; +} +int Maat_format_command(struct Maat_command_t* rule, char* buffer, int size) +{ + //TODO + return 0; +} +int Maat_command(Maat_feather_t feather,struct Maat_command_t* raw_rule,enum MAAT_OPERATION op) +{ + int ret=0; + ret=Maat_append_command(feather,raw_rule,op); + if(ret<0) + { + return ret; + } + ret=Maat_commit_command(feather); + return ret; +} + +int Maat_append_command(Maat_feather_t feather,struct Maat_command_t* cmd,enum MAAT_OPERATION op) +{ + _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t*)cmd; + int ret=0; + _cmd->op=op; + assert(op==MAAT_OP_DEL||op==MAAT_OP_ADD); + if(op==MAAT_OP_DEL) + { + ret=reconstruct_cmd(_feather, _cmd); + } + else + { + ret=fix_table_name(_feather, cmd); + } + if(ret<0) + { + return -1; + } + _cmd->ref_cnt++; + if(_feather->cmd_q_cnt==0) + { + assert(_feather->cmd_qtail==NULL); + assert(_feather->cmd_qtail==_feather->cmd_qhead); + _feather->cmd_qhead=_feather->cmd_qtail=_cmd; + } + else + { + _feather->cmd_qtail->next=_cmd; + _feather->cmd_qtail=_cmd; + } + _feather->cmd_q_cnt++; + return 0; +} + int Maat_commit_command(Maat_feather_t feather) { _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - redisReply* reply=NULL; - struct serial_rule_t* update_status=NULL; - int status_cnt=0; - int ret=0,i=0,redis_ret=REDIS_ERR,retry=0; - long long maat_redis_version=0,new_region_num=0,new_group_num=0; + int ret=0,i=0,retry=0; + long long maat_redis_version=0; + int new_region_num=0,new_group_num=0; int serial_rule_num=0,serial_rule_idx=0; - struct _Maat_cmd_t* p=NULL,*n=NULL; + struct _Maat_cmd_inner_t* p=NULL,*n=NULL; redisContext* ctx=NULL; redisReply* data_reply=NULL; @@ -813,38 +816,40 @@ int Maat_commit_command(Maat_feather_t feather) } ctx=_feather->redis_write_ctx; - for(i=0,p=feather->cmd_qhead;icmd_num;i++) + for(i=0,p=_feather->cmd_qhead;i<_feather->cmd_q_cnt;i++) { p=p->next; - serial_rule_num+=calculate_serial_rule_num(&(p->cmd), &new_region_num, &new_group_num); + serial_rule_num+=calculate_serial_rule_num(p, &new_region_num, &new_group_num); } - data_reply=redisCommand(ctx,"INCRBY SEQUENCE_REGION %d",new_region_num); + data_reply=_wrap_redisCommand(ctx,"INCRBY SEQUENCE_REGION %d",new_region_num); + assert(data_reply->type==REDIS_REPLY_INTEGER); _feather->base_rgn_seq=data_reply->integer-new_region_num; freeReplyObject(data_reply); - data_reply=redisCommand(ctx,"INCRBY SEQUENCE_GROUP %d",new_group_num); + data_reply=_wrap_redisCommand(ctx,"INCRBY SEQUENCE_GROUP %d",new_group_num); + assert(data_reply->type==REDIS_REPLY_INTEGER); _feather->base_rgn_seq=data_reply->integer-new_group_num; freeReplyObject(data_reply); s_rule=(struct serial_rule_t*)calloc(sizeof(struct serial_rule_t),serial_rule_num); - for(i=0,p=feather->cmd_qhead;icmd_num;i++) + for(i=0,p=_feather->cmd_qhead;i<_feather->cmd_q_cnt;i++) { p=p->next; - serial_rule_idx+=build_serial_rule(feather,p,s_rule, serial_rule_num-serial_rule_idx); + serial_rule_idx+=build_serial_rule(_feather,p,s_rule, serial_rule_num-serial_rule_idx); } assert(serial_rule_idx==serial_rule_num); while(1) { - data_reply=redisCommand(ctx, "WATCH MAAT_VERSION"); + data_reply=_wrap_redisCommand(ctx, "WATCH MAAT_VERSION"); freeReplyObject(data_reply); - data_reply=redisCommand(ctx, "GET MAAT_VERSION"); + data_reply=_wrap_redisCommand(ctx, "GET MAAT_VERSION"); freeReplyObject(data_reply); - maat_redis_version=data_reply->integer; + maat_redis_version=atoll(data_reply->str); maat_redis_version++; - data_reply=redisCommand(ctx,"MULTI"); + data_reply=_wrap_redisCommand(ctx,"MULTI"); freeReplyObject(data_reply); for(i=0;icmd_commit_num+=_feather->cmd_q_cnt; + _feather->cmd_acc_num+=_feather->cmd_q_cnt; error_out: p=_feather->cmd_qhead; for(i=0;i<_feather->cmd_q_cnt;i++) @@ -912,8 +917,7 @@ error_out: for(i=0;iinput_bytes); aligment_int64_array_free(p->stream_num); aligment_int64_array_free(p->hit_cnt); + if(p->cb_info!=NULL) + { + dynamic_array_destroy(p->cb_info->cache_lines, free); + p->cb_info->cache_lines=NULL; + free(p->cb_info); + p->cb_info=NULL; + } free(p); return; } +int read_expr_table_info(const char* line,int line_num,struct _Maat_table_info_t* p,MESA_htable_handle string2int_map) +{ + int j=0,ret[4]={0}; + char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0}; + char *token=NULL,*sub_token=NULL,*saveptr; + + sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(p->table_id) + ,p->table_name[0] + ,table_type + ,src_charset + ,dst_charset + ,merge + ,&(p->cross_cache_size) + ,quick_str_scan); + memset(ret,0,sizeof(ret)); + ret[0]=map_str2int(string2int_map,strlwr(table_type),(int*)&(p->table_type)); + ret[1]=map_str2int(string2int_map,strlwr(src_charset),(int*)&(p->src_charset)); + ret[2]=map_str2int(string2int_map,strlwr(merge),&(p->do_charset_merge)); + if(strlen(quick_str_scan)>0) + { + ret[3]=map_str2int(string2int_map,strlwr(quick_str_scan),&(p->quick_expr_switch)); + } + memset(quick_str_scan,0,sizeof(quick_str_scan)); + + for(j=0;j<4;j++) + { + if(ret[j]<0) + { + return -1; + } + } + j=0; + for (token = dst_charset; ; token= NULL) + { + sub_token= strtok_r(token,"/", &saveptr); + if (sub_token == NULL) + break; + ret[3]=map_str2int(string2int_map,strlwr(sub_token),(int*)&(p->dst_charset[j])); + if(ret[3]>0) + { + if(p->dst_charset[j]==p->src_charset) + { + p->src_charset_in_dst=TRUE; + } + j++; + } + else + { + return -1; + } + + } + return 0; +} int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) { FILE*fp=NULL; char line[MAX_TABLE_LINE_SIZE]; - int i=0,j=0,ret[4]={0},table_cnt=0; - char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0}; + int i=0,ret=0,table_cnt=0; + char table_type_str[16],not_care[256]; MESA_htable_handle string2int_map=map_create(); - char *token=NULL,*sub_token=NULL,*saveptr; struct _Maat_table_info_t*p=NULL; struct _Maat_table_info_t*conj_table=NULL; @@ -511,14 +570,6 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* } } -/* - map_register(string2int_map,"gbk", CHARSET_GBK); - map_register(string2int_map,"big5", CHARSET_BIG5); - map_register(string2int_map,"unicode", CHARSET_UNICODE); - map_register(string2int_map,"utf8", CHARSET_UTF8); - map_register(string2int_map,"unicode_hex", CHARSET_UNICODE_ASCII_ESC); - map_register(string2int_map,"unicode_hex", CHARSET_UNICODE_ASCII_ESC); -*/ map_register(string2int_map,"yes", 1); map_register(string2int_map,"no", 0); @@ -540,57 +591,43 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* } p=create_table_info(max_thread_num); - sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(p->table_id) + sscanf(line,"%hu\t%s\t%s\t%s",&(p->table_id) ,p->table_name[0] - ,table_type - ,src_charset - ,dst_charset - ,merge - ,&(p->cross_cache_size) - ,quick_str_scan); - ret[0]=map_str2int(string2int_map,strlwr(table_type),(int*)&(p->table_type)); - ret[1]=map_str2int(string2int_map,strlwr(src_charset),(int*)&(p->src_charset)); - ret[2]=map_str2int(string2int_map,strlwr(merge),&(p->do_charset_merge)); - if(strlen(quick_str_scan)>0) + ,table_type_str + ,not_care); + ret=map_str2int(string2int_map,strlwr(table_type_str),(int*)&(p->table_type)); + if(ret<0) { - ret[3]=map_str2int(string2int_map,strlwr(quick_str_scan),&(p->quick_expr_switch)); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:invalid table type.",table_info_path,i); + goto error_jump; } - memset(quick_str_scan,0,sizeof(quick_str_scan)); - - for(j=0;j<4;j++) + switch(p->table_type) { - if(ret[j]<0) - { - fprintf(stderr,"Maat read table info %s line %d error:unknown column.\n",table_info_path,i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:unknown column.",table_info_path,i); - goto error_jump; - } - } - j=0; - for (token = dst_charset; ; token= NULL) - { - sub_token= strtok_r(token,"/", &saveptr); - if (sub_token == NULL) - break; - ret[3]=map_str2int(string2int_map,strlwr(sub_token),(int*)&(p->dst_charset[j])); - if(ret[3]>0) - { - if(p->dst_charset[j]==p->src_charset) + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + ret=read_expr_table_info(line, i, p, string2int_map); + if(ret<0) { - p->src_charset_in_dst=TRUE; + fprintf(stderr,"Maat read table info %s line %d error:unknown column.\n",table_info_path,i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:unknown column.",table_info_path,i); + goto error_jump; } - j++; - } - else - { - fprintf(stderr,"Maat read table info %s line %d error:unknown dest charset %s.\n",table_info_path,i,sub_token); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error: unknown dest charset %s.",table_info_path,i,sub_token); - goto error_jump; - } - + break; + case TABLE_TYPE_PLUGIN: + p->cb_info=(struct _plugin_table_info*)calloc(sizeof(struct _plugin_table_info),1); + p->cb_info->cache_lines=dynamic_array_create(1024,1024); + ret=sscanf(not_care,"%d",&(p->valid_flag_column)); + if(ret==0||ret==EOF) + { + p->valid_flag_column=-1; + } + break; + default: + break; } + if(p->table_id>=num) { fprintf(stderr,"Maat read table info %s:%d error: table id %uh > %d.\n",table_info_path,i,p->table_id,num); @@ -618,11 +655,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* //use goto to free the conjunctioned table_info goto error_jump; } - if(p->table_type==TABLE_TYPE_PLUGIN) - { - p->cb_info=(struct _plugin_table_info*)calloc(sizeof(struct _plugin_table_info),1); - p->cb_info->cache_lines=dynamic_array_create(1024,1024); - } + p_table_info[p->table_id]=p; table_cnt++; continue; @@ -1357,7 +1390,7 @@ void cancel_last_region_from_group(struct _Maat_group_inner_t* group,int region_ { struct _Maat_region_inner_t* region_rule=NULL; pthread_mutex_lock(&(group->mutex)); - region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group->regions,group->region_boundary); + region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group->regions,group->region_boundary-1); assert(region_rule->expr_id_ub==expr_id&®ion_rule->region_id==region_id); if(region_rule->expr_id_cnt==1) { @@ -1369,7 +1402,7 @@ void cancel_last_region_from_group(struct _Maat_group_inner_t* group,int region_ else { region_rule->expr_id_ub--; - region_rule->region_cnt--; + region_rule->expr_id_cnt--; } pthread_mutex_unlock(&(group->mutex)); return; @@ -1771,6 +1804,8 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule scanner->dedup_expr_num++; cancel_last_region_from_group(group_rule,db_rule->region_id,op_expr->p_expr->expr_id); destroy_op_expr(op_expr); + //redeem expr_id + scanner->exprid_generator--; op_expr=NULL; } else @@ -3075,9 +3110,9 @@ void *thread_rule_monitor(void *arg) ,maat_start_cb ,maat_update_cb ,maat_finish_cb - , feather + ,feather ,feather->decrypt_key //Not used. - ,feather->logger); + ,feather); } else { @@ -3094,7 +3129,7 @@ void *thread_rule_monitor(void *arg) if(feather->update_tmp_scanner!=NULL) { old_scanner=feather->scanner; - //__sync_lock_test_and_set not work in some OS. + //Some OS doesn't have __sync_lock_test_and_set. //feather->scanner=__sync_lock_test_and_set(&(feather->scanner),feather->update_tmp_scanner); feather->scanner=feather->update_tmp_scanner; if(old_scanner!=NULL) diff --git a/src/entry/Maat_rule_internal.h b/src/entry/Maat_rule_internal.h index 940243c..110f310 100644 --- a/src/entry/Maat_rule_internal.h +++ b/src/entry/Maat_rule_internal.h @@ -165,19 +165,22 @@ struct op_expr_t struct _Maat_region_inner_t { int region_id; - int expr_id_cnt; int district_id; + int table_id; + int expr_id_cnt; int expr_id_lb; int expr_id_ub; - enum MAAT_TABLE_TYPE region_type; + enum MAAT_TABLE_TYPE table_type; }; struct _Maat_group_inner_t { int group_id; + int table_id; int region_boundary; int region_cnt; int ref_cnt; - dynamic_array_t *region_rules; + char* group_name; + dynamic_array_t *regions; void* compile_shortcut; pthread_mutex_t mutex; }; @@ -185,9 +188,10 @@ struct _Maat_compile_inner_t { struct db_compile_rule_t *db_c_rule; dynamic_array_t *groups; + int compile_id;//equal to db_c_rule->m_rule.config_id + int table_id; int group_boundary; int group_cnt; - int compile_id;//equal to db_c_rule->m_rule.config_id pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read }; struct _compile_result_t @@ -252,6 +256,7 @@ struct _Maat_table_info_t int ipv6_rule_cnt; }; struct _plugin_table_info *cb_info; + int valid_flag_column; //for stat>>>>>>>> unsigned long long udpate_err_cnt; unsigned long long iconv_err_cnt; @@ -369,14 +374,14 @@ struct _Maat_feather_t pthread_mutex_t plugin_table_reg_mutex; unsigned char decrypt_key[MAX_TABLE_NAME_LEN]; - unsigned char redis_ip[MAX_TABLE_NAME_LEN]; + char redis_ip[MAX_TABLE_NAME_LEN]; int redis_port; struct timeval connect_timeout; redisContext *redis_read_ctx; redisContext *redis_write_ctx; // not thread safe. int on_redis_writing; int cmd_q_cnt; - struct _Maat_cmd_t* cmd_qhead, *cmd_qtail; + struct _Maat_cmd_inner_t* cmd_qhead, *cmd_qtail; pthread_mutex_t redis_write_lock; //protect redis_write_ctx long long base_rgn_seq,base_grp_seq; //for stat>>>> @@ -408,8 +413,8 @@ struct _maat_garbage_t union { struct _Maat_scanner_t* scanner; - struct _Maat_group_rule_t* group_rule; - struct _Maat_compile_rule_t* compile_rule; + struct _Maat_group_inner_t* group_rule; + struct _Maat_compile_inner_t* compile_rule; void* bool_matcher; void * raw; MESA_htable_handle str2int_map; @@ -417,7 +422,7 @@ struct _maat_garbage_t }; void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); -void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bool_expr_t* a_set); +void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set); int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); void maat_start_cb(unsigned int new_version,int update_type,void*u_para); void maat_update_cb(const char* table_name,const char* line,void *u_para); @@ -433,10 +438,13 @@ inline void ipv6_ntoh(unsigned int *v6_addr) } return; } +void * HASH_fetch_by_id(MESA_htable_handle hash,int id); +int HASH_add_by_id(MESA_htable_handle hash,int id,void*data); +int HASH_delete_by_id(MESA_htable_handle hash,int id); void maat_stat_init(struct _Maat_feather_t* feather); void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num); void maat_stat_output(struct _Maat_feather_t* feather); - +char* _maat_strdup(const char* s); void redis_monitor_traverse(unsigned int version,redisContext *c, void (*start)(unsigned int ,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para void (*update)(const char* ,const char*,void* ),//table name ,line ,u_para diff --git a/src/entry/Makefile b/src/entry/Makefile index b052f5e..4aca2c6 100644 --- a/src/entry/Makefile +++ b/src/entry/Makefile @@ -4,11 +4,11 @@ CC = gcc CCC = g++ CFLAGS = -Wall -g -fPIC -GCOV_FLAGS = -fprofile-arcs -ftest-coverage +#GCOV_FLAGS = -fprofile-arcs -ftest-coverage CFLAGS += $(OPTFLAGS) #CFLAGS += $(GCOV_FLAGS) LDDICTATOR = -Wl,-wrap,malloc -Wl,-wrap,calloc -Wl,-wrap,free -Wl,-wrap,realloc -LDFLAGS = -lMESA_handle_logger -lMESA_htable -lpthread -lrt -lm -lrulescan -lpcre -lMESA_field_stat2 -lcrypto +LDFLAGS = -lMESA_handle_logger -lMESA_htable -lpthread -lrt -lm -lrulescan -lpcre -lMESA_field_stat2 -lcrypto -lhiredis_vip #LDFLAGS += $(LDDICTATOR) LDFLAGS += $(GCOV_FLAGS) MAILLIB = ../lib @@ -18,8 +18,8 @@ H_DIR =-I$(G_H_DIR) -I../../inc LIBMAAT = libmaatframe.a LIBMAAT_SO = libmaatframe.so -OBJS=config_monitor.o Maat_rule.o Maat_api.o Maat_stat.o UniversalBoolMatch.o dynamic_array.o cJSON.o\ - json2iris.o map_str2int.o interval_index.o great_index_engine.o mesa_fuzzy.o rbtree.o +OBJS=config_monitor.o Maat_rule.o Maat_api.o Maat_command.o Maat_stat.o UniversalBoolMatch.o dynamic_array.o\ + cJSON.o json2iris.o map_str2int.o interval_index.o great_index_engine.o mesa_fuzzy.o rbtree.o .c.o: $(CC) -c $(CFLAGS) -I. $(H_DIR) $< diff --git a/src/entry/config_monitor.cpp b/src/entry/config_monitor.cpp index 3566a5f..ba230f2 100644 --- a/src/entry/config_monitor.cpp +++ b/src/entry/config_monitor.cpp @@ -52,7 +52,7 @@ int decrypt_open(FILE* in,const unsigned char* key, const char* algorithm,unsign cipher=EVP_get_cipherbyname(algorithm); if(cipher==NULL) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"Not cipher:%s not supported."); + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"Not cipher:%s not supported.",algorithm); return 0; } dgst=EVP_get_digestbyname("md5"); diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 02145a1..a288fde 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -55,8 +55,6 @@ struct traslate_command_t }; int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,const char* group_tn,struct iris_description_t *iris_cfg,void * logger) { - cJSON* j=NULL; - char* table_name=NULL; memset(iris_cfg,0,sizeof(struct iris_description_t)); snprintf(iris_cfg->tmp_iris_dir,sizeof(iris_cfg->tmp_iris_dir),"%s_iris_tmp",json_file); snprintf(iris_cfg->tmp_iris_index_dir,sizeof(iris_cfg->tmp_iris_index_dir),"%s_iris_tmp/index",json_file); diff --git a/src/inc_internal/sds.h b/src/inc_internal/sds.h new file mode 100644 index 0000000..19a2abd --- /dev/null +++ b/src/inc_internal/sds.h @@ -0,0 +1,105 @@ +/* SDS (Simple Dynamic Strings), A C dynamic strings library. + * + * Copyright (c) 2006-2014, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SDS_H +#define __SDS_H + +#define SDS_MAX_PREALLOC (1024*1024) + +#include +#include +#ifdef _MSC_VER +#include "win32.h" +#endif + +typedef char *sds; + +struct sdshdr { + int len; + int free; + char buf[]; +}; + +static inline size_t sdslen(const sds s) { + struct sdshdr *sh = (struct sdshdr *)(s-sizeof *sh); + return sh->len; +} + +static inline size_t sdsavail(const sds s) { + struct sdshdr *sh = (struct sdshdr *)(s-sizeof *sh); + return sh->free; +} + +sds sdsnewlen(const void *init, size_t initlen); +sds sdsnew(const char *init); +sds sdsempty(void); +size_t sdslen(const sds s); +sds sdsdup(const sds s); +void sdsfree(sds s); +size_t sdsavail(const sds s); +sds sdsgrowzero(sds s, size_t len); +sds sdscatlen(sds s, const void *t, size_t len); +sds sdscat(sds s, const char *t); +sds sdscatsds(sds s, const sds t); +sds sdscpylen(sds s, const char *t, size_t len); +sds sdscpy(sds s, const char *t); + +sds sdscatvprintf(sds s, const char *fmt, va_list ap); +#ifdef __GNUC__ +sds sdscatprintf(sds s, const char *fmt, ...) + __attribute__((format(printf, 2, 3))); +#else +sds sdscatprintf(sds s, const char *fmt, ...); +#endif + +sds sdscatfmt(sds s, char const *fmt, ...); +void sdstrim(sds s, const char *cset); +void sdsrange(sds s, int start, int end); +void sdsupdatelen(sds s); +void sdsclear(sds s); +int sdscmp(const sds s1, const sds s2); +sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); +void sdsfreesplitres(sds *tokens, int count); +void sdstolower(sds s); +void sdstoupper(sds s); +sds sdsfromlonglong(long long value); +sds sdscatrepr(sds s, const char *p, size_t len); +sds *sdssplitargs(const char *line, int *argc); +sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen); +sds sdsjoin(char **argv, int argc, char *sep, size_t seplen); +sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen); + +/* Low level functions exposed to the user API */ +sds sdsMakeRoomFor(sds s, size_t addlen); +void sdsIncrLen(sds s, int incr); +sds sdsRemoveFreeSpace(sds s); +size_t sdsAllocSize(sds s); + +#endif