diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 1970332..0d2f3d8 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -601,6 +601,10 @@ int Maat_initiate_feather(Maat_feather_t feather) if(strlen(_feather->redis_ip)>0&&_feather->redis_port!=0) { _feather->REDIS_MODE_ON=1; + MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , + "Maat initiate from Redis %s:%hu" + ,_feather->redis_ip + ,_feather->redis_port); _feather->redis_read_ctx=redisConnectWithTimeout(_feather->redis_ip,_feather->redis_port,_feather->connect_timeout); if(_feather->redis_read_ctx==NULL) { @@ -639,8 +643,19 @@ int Maat_initiate_feather(Maat_feather_t feather) } if(_feather->update_tmp_scanner==NULL) { - MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module , + if(_feather->REDIS_MODE_ON==1) + { + MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module , + "At initiation: no avilable rule in redis in %s:%hu" + ,_feather->redis_ip + ,_feather->redis_port); + + } + else + { + MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module , "At initiation: no valid index file in %s",_feather->full_dir); + } } _feather->scanner=_feather->update_tmp_scanner; _feather->update_tmp_scanner=NULL; diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 0ba2653..3089476 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -13,7 +13,7 @@ const char* maat_redis_monitor="MAAT_REDIS_MONITOR"; const char* maat_redis_command="MAAT_REDIS_COMMAND"; const char* rm_key_prefix[2]={"OBSOLETE_RULE","EFFECTIVE_RULE"}; - +const char* rm_status_key="MAAT_UPDATE_STATUS"; const int MAAT_REDIS_SYNC_TIME=30*60; struct serial_rule_t //rm= Redis Maat @@ -32,6 +32,39 @@ struct _Maat_cmd_inner_t int region_size[MAX_EXPR_ITEM_NUM]; struct _Maat_cmd_inner_t* next; }; +int _wrap_redisGetReply(redisContext *c, redisReply **reply) +{ + return redisGetReply(c, (void **)reply); +} +redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...) +{ + va_list ap; + void *reply = NULL; + va_start(ap,format); + reply = redisvCommand(c,format,ap); + va_end(ap); + return (redisReply *)reply; +} +long long read_redis_integer(const redisReply* reply) +{ + switch(reply->type) + { + case REDIS_REPLY_INTEGER: + return reply->integer; + break; + case REDIS_REPLY_ARRAY: + assert(reply->element[0]->type==REDIS_REPLY_INTEGER); + return reply->element[0]->integer; + break; + case REDIS_REPLY_STRING: + return atoll(reply->str); + break; + default: + assert(0); + break; + } + return 0; +} enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p) { enum MAAT_TABLE_TYPE ret=TABLE_TYPE_IP; @@ -107,11 +140,12 @@ void invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) { j++; } - if(j==offset) + if(j==offset-1) { break; } } + i++; assert(itype==REDIS_REPLY_STRING); - version_in_redis=atoll(reply->str); + version_in_redis=read_redis_integer(reply); assert(version_in_redis>=version); freeReplyObject(reply); if(version_in_redis==version) @@ -260,17 +293,17 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** //Returns all the elements in the sorted set at key with a score that version < score <= version_in_redis. //The elements are considered to be ordered from low to high scores(version). - reply=(redisReply*)redisCommand(c, "ZRANGE MAAT_UPDATE_STATUS (%d %d",version,version_in_redis); + reply=(redisReply*)redisCommand(c, "ZRANGEBYSCORE %s (%d %d",rm_status_key,version,version_in_redis); if(reply==NULL) { __redis_strerror_r(errno,err_buff,sizeof(err_buff)); MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "GET MAAT_UPDATE_STATUS failed %s.",err_buff); + "GET %s failed %s.",rm_status_key,err_buff); return 0; } assert(reply->type==REDIS_REPLY_ARRAY); - tmp_reply=(redisReply*)redisCommand(c, "ZRANK MAAT_UPDATE_STATUS %s",reply->element[0]->str); - nearest_rule_version=tmp_reply->integer; + tmp_reply=(redisReply*)redisCommand(c, "ZSCORE %s %s",rm_status_key,reply->element[0]->str); + nearest_rule_version=read_redis_integer(tmp_reply); freeReplyObject(tmp_reply); tmp_reply=NULL; if(nearest_rule_version!=version+1) @@ -289,7 +322,7 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** for(i=0;ielements;i++) { assert(reply->element[i]->type==REDIS_REPLY_STRING); - ret=sscanf(reply->element[i]->str,"%[^:]:%[^,],%d",op_str,s_rule[i].table_name,&(s_rule[i].rule_id)); + ret=sscanf(reply->element[i]->str,"%[^,],%[^,],%d",op_str,s_rule[i].table_name,&(s_rule[i].rule_id)); assert(ret==3); if(strncmp(op_str,"ADD",strlen("ADD"))==0) { @@ -328,19 +361,7 @@ FULL_UPDATE: reply=NULL; return i; } -int _wrap_redisGetReply(redisContext *c, redisReply **reply) -{ - return redisGetReply(c, (void **)reply); -} -redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...) -{ - va_list ap; - void *reply = NULL; - va_start(ap,format); - reply = redisvCommand(c,format,ap); - va_end(ap); - return (redisReply *)reply; -} + int calculate_serial_rule_num(struct _Maat_cmd_inner_t* _cmd,int * new_region_cnt, int* new_group_cnt) { int serial_num=0; @@ -359,6 +380,9 @@ int calculate_serial_rule_num(struct _Maat_cmd_inner_t* _cmd,int * new_region_cn *new_region_cnt+=cmd->groups[i].region_num; (*new_group_cnt)++; } + //for MAAT_OP_DEL, if the group's refcnt>0, group->region_num=0. + //so it's OK to add. + serial_num+=cmd->groups[i].region_num; } return serial_num; } @@ -449,7 +473,7 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st set_serial_rule(list+rule_num,MAAT_OP_DEL,cmd->compile.config_id,feather->compile_tn,NULL); } rule_num++; - for(i=0;cmd->group_num;i++) + for(i=0;igroup_num;i++) { p_group=&(cmd->groups[i]); if(op==MAAT_OP_ADD) @@ -489,20 +513,19 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st rule_num++; } } - assert(rule_numelements;i++) + if(data_reply->type==REDIS_REPLY_NIL) { - if(data_reply->element[i]->type==REDIS_REPLY_NIL) - { - return 0; - } + return 0; + } + else + { + return 1; } - return 1; } int fix_table_name(_Maat_feather_t* feather,struct Maat_command_t* cmd) { @@ -512,6 +535,18 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_command_t* cmd) struct Maat_group_t* p_group=NULL; struct Maat_region_t* p_region=NULL; enum MAAT_TABLE_TYPE table_type; + + struct _Maat_compile_inner_t *compile_rule=NULL; + if(feather->scanner!=NULL) + { + compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id); + if(compile_rule!=NULL) + { + MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module + ,"Maat rule %d already exisits.",cmd->compile.config_id); + return -1; + } + } for(i=0;igroup_num;i++) { p_group=&(cmd->groups[i]); @@ -529,7 +564,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_command_t* cmd) return -1; } table_type=type_region2table(p_region); - if(table_type!=feather->p_table_info[i]->table_type) + if(table_type!=feather->p_table_info[table_id]->table_type) { MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module ,"Table %s not support region type %d of Maat_command_t[%d]->group[%d]->region[%d]." @@ -539,7 +574,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_command_t* cmd) return -1; } free((char*)p_region->table_name); - p_region->table_name=_maat_strdup(feather->p_table_info[i]->table_name[0]); + p_region->table_name=_maat_strdup(feather->p_table_info[table_id]->table_name[0]); } } return 0; @@ -671,7 +706,14 @@ struct Maat_command_t* Maat_create_comand(const struct Maat_rule_t* rule, int gr memcpy(&(_cmd->cmd.compile),rule,sizeof(_cmd->cmd.compile)); _cmd->ref_cnt=1; _cmd->cmd.group_num=group_num; - _cmd->cmd.groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),group_num); + if(group_num>0) + { + _cmd->cmd.groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),group_num); + } + else + { + _cmd->cmd.groups=NULL; + } for(i=0;icmd.groups[i].regions=(struct Maat_region_t*)calloc(sizeof(struct Maat_region_t),1); @@ -795,13 +837,21 @@ int Maat_commit_command(Maat_feather_t feather) long long maat_redis_version=0; int new_region_num=0,new_group_num=0; int serial_rule_num=0,serial_rule_idx=0; + int redis_transaction_failed=1; struct _Maat_cmd_inner_t* p=NULL,*n=NULL; redisContext* ctx=NULL; redisReply* data_reply=NULL; struct serial_rule_t* s_rule=NULL; - + if(_feather->REDIS_MODE_ON==0) + { + return -1; + } + if(_feather->cmd_q_cnt==0) + { + return 0; + } if(_feather->redis_write_ctx==NULL) { _feather->redis_write_ctx=redisConnectWithTimeout(_feather->redis_ip, _feather->redis_port,_feather->connect_timeout); @@ -815,11 +865,10 @@ int Maat_commit_command(Maat_feather_t feather) } } ctx=_feather->redis_write_ctx; - for(i=0,p=_feather->cmd_qhead;i<_feather->cmd_q_cnt;i++) { - p=p->next; serial_rule_num+=calculate_serial_rule_num(p, &new_region_num, &new_group_num); + p=p->next; } data_reply=_wrap_redisCommand(ctx,"INCRBY SEQUENCE_REGION %d",new_region_num); @@ -836,34 +885,36 @@ int Maat_commit_command(Maat_feather_t feather) for(i=0,p=_feather->cmd_qhead;i<_feather->cmd_q_cnt;i++) { - p=p->next; serial_rule_idx+=build_serial_rule(_feather,p,s_rule, serial_rule_num-serial_rule_idx); + p=p->next; } assert(serial_rule_idx==serial_rule_num); - - while(1) + redis_transaction_failed=1; + while(redis_transaction_failed) { data_reply=_wrap_redisCommand(ctx, "WATCH MAAT_VERSION"); freeReplyObject(data_reply); data_reply=_wrap_redisCommand(ctx, "GET MAAT_VERSION"); - freeReplyObject(data_reply); - maat_redis_version=atoll(data_reply->str); + maat_redis_version=read_redis_integer(data_reply); maat_redis_version++; + freeReplyObject(data_reply); data_reply=_wrap_redisCommand(ctx,"MULTI"); freeReplyObject(data_reply); + append_cmd_cnt=0; for(i=0;icmd_acc_num+=_feather->cmd_q_cnt; diff --git a/src/entry/config_monitor.cpp b/src/entry/config_monitor.cpp index ba230f2..87fa793 100644 --- a/src/entry/config_monitor.cpp +++ b/src/entry/config_monitor.cpp @@ -100,7 +100,7 @@ int decrypt_open(FILE* in,const unsigned char* key, const char* algorithm,unsign } if (!EVP_CipherFinal_ex(ctx, *pp_out+buff_offset, &out_blk_len)) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"EVP_CipherFinal_ex failed."); + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"EVP_CipherFinal_ex failed. Maybe password is wrong?"); EVP_CIPHER_CTX_free(ctx); goto error_out; } diff --git a/test/maat_redis_reset.sh b/test/maat_redis_reset.sh deleted file mode 100644 index 2fdc7e7..0000000 --- a/test/maat_redis_reset.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -host="127.0.0.1" -port="6379" -echo "Reset Redis For Maat Starting" -redis-cli -h $host -p $port GET MAAT_VERSION -redis-cli -h $host -p $port FLUSHALL -redis-cli -h $host -p $port SET MAAT_VERSION 0 -redis-cli -h $host -p $port SET SEQUENCE_REGION 0 -redis-cli -h $host -p $port SET SEQUENCE_GROUP 0 - diff --git a/test/maat_test.cpp b/test/maat_test.cpp index 180f8a7..a6713ef 100644 --- a/test/maat_test.cpp +++ b/test/maat_test.cpp @@ -471,6 +471,7 @@ int test_command(Maat_feather_t feather) struct Maat_rule_t result; struct Maat_region_t region; int group_num=1,ret=0; + memset(&rule,0,sizeof(rule)); rule.config_id=201; strcpy(rule.service_defined,"maat_command"); //MUST acqire by function, because Maat_command_t has some hidden members. @@ -479,17 +480,28 @@ int test_command(Maat_feather_t feather) region.region_type=REGION_EXPR; region.table_name=table_name; region.expr_rule.district=NULL; - region.expr_rule.keywords="Hiredis&C\bClient"; + region.expr_rule.keywords="Hiredis&C\\bClient"; region.expr_rule.expr_type=EXPR_TYPE_AND; region.expr_rule.match_method=MATCH_METHOD_SUB; region.expr_rule.hex_bin=UNCASE_PLAIN; Maat_cmd_add_region(cmd, 0, ®ion); //use pipeline model. - Maat_append_command(feather, cmd, MAAT_OP_ADD); + ret=Maat_append_command(feather, cmd, MAAT_OP_ADD); + if(ret<0) + { + printf("Add Maat command %d failed.\n",rule.config_id); + Maat_free_command(cmd); + return 0; + } //cmd has been saved in feather, so free before commit is allowed. Maat_free_command(cmd); - Maat_commit_command(feather); - sleep(3);//waiting for commands go into effect + ret=Maat_commit_command(feather); + if(ret<0) + { + printf("Commit Maat command %d failed.\n",rule.config_id); + return 0; + } + sleep(1);//waiting for commands go into effect table_id=Maat_table_register(feather,table_name); ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), &result,NULL, 1, @@ -497,11 +509,36 @@ int test_command(Maat_feather_t feather) if(ret>0&&result.config_id==rule.config_id) { - printf("Test Maat command success %s\n",print_maat_result(&result,ret)); + printf("Test Maat add command Success %s\n",print_maat_result(&result,ret)); } else { - printf("Test Maat command failed.\n"); + printf("Test Maat add command failed.\n"); + } + Maat_clean_status(&mid); + + memset(&rule,0,sizeof(rule)); + rule.config_id=201; + cmd=Maat_create_comand(&rule, 0); + ret=Maat_command(feather, cmd, MAAT_OP_DEL); + if(ret<0) + { + printf("Delete Maat command %d failed.\n",rule.config_id); + Maat_free_command(cmd); + return 0; + } + Maat_free_command(cmd); + sleep(1);//waiting for commands go into effect + ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), + &result,NULL, 1, + &mid, 0); + if(ret>0) + { + printf("Test Maat delete command failed\n"); + } + else + { + printf("Test Maat delete command Success.\n"); } Maat_clean_status(&mid); return 0; @@ -518,11 +555,12 @@ int main(int argc,char* argv[]) const char* log_file="./test.log"; const char* stat_file="./scan_staus.log"; const char* decrypt_key="mesa2017wy"; + int scan_interval_ms=10; const char* redis_ip="127.0.0.1"; unsigned short redis_port=6379; int scan_detail=0; scan_status_t mid=NULL; - int wait_second=4; + int wait_second=400; void *logger=MESA_create_runtime_log_handle(log_file,0); feather=Maat_feather(g_iThreadNum, table_info_path, logger); @@ -544,6 +582,7 @@ int main(int argc,char* argv[]) { Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, json_path, strlen(json_path)+1); } + Maat_set_feather_opt(feather, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms)); Maat_set_feather_opt(feather, MAAT_OPT_STAT_FILE_PATH, stat_file, strlen(stat_file)+1); Maat_set_feather_opt(feather, MAAT_OPT_STAT_ON, NULL, 0); Maat_set_feather_opt(feather, MAAT_OPT_PERF_ON, NULL, 0); diff --git a/test/reset_redis4maat.sh b/test/reset_redis4maat.sh new file mode 100644 index 0000000..3c2032b --- /dev/null +++ b/test/reset_redis4maat.sh @@ -0,0 +1,10 @@ +#!/bin/sh +host="127.0.0.1" +port="6379" +echo "Reseting Redis For Maat..." +redis-cli -h $host -p $port GET MAAT_VERSION +redis-cli -h $host -p $port FLUSHALL +redis-cli -h $host -p $port SET MAAT_VERSION 1 +redis-cli -h $host -p $port SET SEQUENCE_REGION 1 +redis-cli -h $host -p $port SET SEQUENCE_GROUP 1 + diff --git a/test/table_info.conf b/test/table_info.conf index 4e51399..1f1554a 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -6,13 +6,13 @@ #dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/' #do_merege yes or no #id name type src_charset dst_charset do_merge -0 COMPILE compile GBK GBK no 0 -1 GROUP group GBK GBK no 0 +0 COMPILE compile -- +1 GROUP group -- 2 HTTP_URL expr UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff 2 HTTP_HOST expr UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff 3 KEYWORDS_TABLE expr UTF8 GBK/BIG5/UNICODE/UTF8/unicode_ascii_esc/unicode_ascii_aligned/unicode_ncr_dec/unicode_ncr_hex yes 0 -4 IP_CONFIG ip GBK GBK no 0 -5 CONTENT_SIZE intval GBK GBK no 0 -6 QD_ENTRY_INFO plugin GBK GBK no 0 -7 FILE_DIGEST digest GBK GBK no 0 +4 IP_CONFIG ip -- +5 CONTENT_SIZE intval -- +6 QD_ENTRY_INFO plugin 4 +7 FILE_DIGEST digest -- 8 HTTP_REGION expr_plus GBK GBK no 0