gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加Redis重启、不可用场景下的错误处理，已在线部署一个局点。

Browse Source
This commit is contained in:
zhengchao
2017-08-12 11:13:47 +08:00
parent 448a712a20
commit 94f69b0f6d
4 changed files with 202 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/entry/Maat_api.cpp
View File

@@ -619,17 +619,19 @@ int Maat_initiate_feather(Maat_feather_t feather)
{ {
_feather->REDIS_MODE_ON=1; _feather->REDIS_MODE_ON=1;
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"Maat initiate from Redis %s:%hu" "Maat initiate from Redis %s:%hu db%d."
,_feather->redis_ip ,_feather->redis_ip
,_feather->redis_port); ,_feather->redis_port
,_feather->redis_index);
_feather->redis_read_ctx=redisConnectWithTimeout(_feather->redis_ip,_feather->redis_port,_feather->connect_timeout); _feather->redis_read_ctx=redisConnectWithTimeout(_feather->redis_ip,_feather->redis_port,_feather->connect_timeout);
if(_feather->redis_read_ctx==NULL) if(_feather->redis_read_ctx==NULL)
{ {
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module
,"Redis connect %s:%d failed." ,"Redis connect %s:%d failed."
,_feather->redis_ip,_feather->redis_port); ,_feather->redis_ip,_feather->redis_port);
return -1;
} }
else
{
redisEnableKeepAlive(_feather->redis_read_ctx); redisEnableKeepAlive(_feather->redis_read_ctx);
reply=_wrap_redisCommand(_feather->redis_read_ctx, "select %d",_feather->redis_index); reply=_wrap_redisCommand(_feather->redis_read_ctx, "select %d",_feather->redis_index);
freeReplyObject(reply); freeReplyObject(reply);
@@ -642,6 +644,7 @@ int Maat_initiate_feather(Maat_feather_t feather)
,_feather->decrypt_key //Not used. ,_feather->decrypt_key //Not used.
,_feather); ,_feather);
} }
}
else else
{ {
if(strlen(_feather->full_dir)==0) if(strlen(_feather->full_dir)==0)

70
src/entry/Maat_command.cpp
View File

@@ -14,7 +14,7 @@ const char* maat_redis_command="MAAT_REDIS_COMMAND";
const char* rm_key_prefix[2]={"OBSOLETE_RULE","EFFECTIVE_RULE"}; const char* rm_key_prefix[2]={"OBSOLETE_RULE","EFFECTIVE_RULE"};
const char* rm_status_sset="MAAT_UPDATE_STATUS"; const char* rm_status_sset="MAAT_UPDATE_STATUS";
const char* rm_expire_sset="MAAT_RULE_TIMER"; const char* rm_expire_sset="MAAT_EXPIRE_TIMER";
const char* rm_label_sset="MAAT_LABEL_INDEX"; const char* rm_label_sset="MAAT_LABEL_INDEX";
const char* rm_version_sset="MAAT_VERSION_TIMER"; const char* rm_version_sset="MAAT_VERSION_TIMER";
const static int MAAT_REDIS_SYNC_TIME=30*60; const static int MAAT_REDIS_SYNC_TIME=30*60;
@@ -327,34 +327,36 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
char err_buff[256]; char err_buff[256];
char op_str[4]; char op_str[4];
long long version_in_redis=0,nearest_rule_version=0; long long version_in_redis=0,nearest_rule_version=0;
int ret=0,retry=0; int ret=0;
unsigned int i=0,full_idx =0,append_cmd_cnt=0; unsigned int i=0,full_idx =0,append_cmd_cnt=0;
struct serial_rule_t *s_rule=NULL; struct serial_rule_t *s_rule=NULL;
while(retry<1)
{
reply=(redisReply*)redisCommand(c, "GET MAAT_VERSION"); reply=(redisReply*)redisCommand(c, "GET MAAT_VERSION");
if(reply!=NULL) if(reply!=NULL)
{ {
break;
} if(reply->type==REDIS_REPLY_NIL||reply->type==REDIS_REPLY_ERROR)
if(c->err==REDIS_ERR_EOF)
{ {
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,"GET MAAT_VERSION failed, maybe Redis is busy.");
return 0;
}
}
else
{
memset(err_buff,0,sizeof(err_buff));
__redis_strerror_r(errno,err_buff,sizeof(err_buff)); __redis_strerror_r(errno,err_buff,sizeof(err_buff));
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
"GET MAAT_VERSION failed %s. Reconnecting...",err_buff); "GET MAAT_VERSION failed %s. Reconnecting...",err_buff);
ret=redisReconnect(c); ret=redisReconnect(c);
retry++; if(ret==REDIS_OK)
continue; {
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,"Reconnect success.");
} }
else else
{ {
__redis_strerror_r(errno,err_buff,sizeof(err_buff)); MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,"Reconnect failed.");
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
"GET MAAT_VERSION failed %s.",err_buff);
return 0;
} }
return 0;
} }
version_in_redis=read_redis_integer(reply); version_in_redis=read_redis_integer(reply);
assert(version_in_redis>=version); assert(version_in_redis>=version);
@@ -382,6 +384,12 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
return 0; return 0;
} }
assert(reply->type==REDIS_REPLY_ARRAY); assert(reply->type==REDIS_REPLY_ARRAY);
if(reply->elements==0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,"Got nothing after ZRANGEBYSCORE %s (%d %d",rm_status_sset,version,version_in_redis);
freeReplyObject(reply);
return 0;
}
tmp_reply=_wrap_redisCommand(c, "ZSCORE %s %s",rm_status_sset,reply->element[0]->str); tmp_reply=_wrap_redisCommand(c, "ZSCORE %s %s",rm_status_sset,reply->element[0]->str);
nearest_rule_version=read_redis_integer(tmp_reply); nearest_rule_version=read_redis_integer(tmp_reply);
freeReplyObject(tmp_reply); freeReplyObject(tmp_reply);
@@ -394,7 +402,7 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
goto FULL_UPDATE; goto FULL_UPDATE;
} }
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
"Inc Update form version %d to %lld.",version,version_in_redis); "Inc Update form version %d to %lld (%lld entries).",version,version_in_redis,reply->elements);
s_rule=(struct serial_rule_t*)calloc(reply->elements,sizeof(struct serial_rule_t)); s_rule=(struct serial_rule_t*)calloc(reply->elements,sizeof(struct serial_rule_t));
for(i=0;i<reply->elements;i++) for(i=0;i<reply->elements;i++)
@@ -455,7 +463,7 @@ FULL_UPDATE:
return full_idx ; return full_idx ;
} }
void get_rm_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger) int get_rm_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger)
{ {
int i=0,ret=0,failed_cnt=0,idx=0; int i=0,ret=0,failed_cnt=0,idx=0;
int *retry_ids=(int*)malloc(sizeof(int)*rule_num); int *retry_ids=(int*)malloc(sizeof(int)*rule_num);
@@ -478,10 +486,21 @@ void get_rm_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,v
} }
else else
{ {
assert(reply->type==REDIS_REPLY_NIL); if(reply->type==REDIS_REPLY_NIL)
{
retry_ids[failed_cnt]=i; retry_ids[failed_cnt]=i;
failed_cnt++; failed_cnt++;
} }
else
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor
,"Redis GET %s:%s,%d failed",rm_key_prefix[rule_list[i].op]
,rule_list[i].table_name
,rule_list[i].rule_id);
free(retry_ids);
return -1;
}
}
freeReplyObject(reply); freeReplyObject(reply);
} }
for(i=0;i<failed_cnt;i++) for(i=0;i<failed_cnt;i++)
@@ -496,7 +515,7 @@ void get_rm_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,v
freeReplyObject(reply); freeReplyObject(reply);
} }
free(retry_ids); free(retry_ids);
return; return 0;
} }
int calculate_serial_rule_num(struct _Maat_cmd_inner_t* _cmd,int * new_region_cnt, int* new_group_cnt) int calculate_serial_rule_num(struct _Maat_cmd_inner_t* _cmd,int * new_region_cnt, int* new_group_cnt)
{ {
@@ -833,12 +852,12 @@ void check_maat_expiration(redisContext *ctx, void *logger)
if(is_success==1) if(is_success==1)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor
,"Succesfully expried %d rules in Redis.", s_rule_num); ,"Succesfully expired %d rules in Redis.", s_rule_num);
} }
else else
{ {
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor
,"Failed to expried %d rules in Redis.", s_rule_num); ,"Failed to expired %d rules in Redis, try later.", s_rule_num);
} }
free(s_rule); free(s_rule);
@@ -885,7 +904,7 @@ void cleanup_update_status(redisContext *ctx, void *logger)
freeReplyObject(reply); freeReplyObject(reply);
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor
,"Clean up updaste status from version %lld to %lld (%lld versions, %lld entries)." ,"Clean up update status from version %lld to %lld (%lld versions, %lld entries)."
,version_lower_bound ,version_lower_bound
,version_upper_bound ,version_upper_bound
,version_num ,version_num
@@ -919,8 +938,12 @@ void redis_monitor_traverse(unsigned int version,redisContext *c,
{ {
return; return;
} }
get_rm_value(c,rule_list,rule_num, logger); ret=get_rm_value(c,rule_list,rule_num, logger);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Get Redis value failed, abandon update.");
goto clean_up;
}
start(new_version,update_type,u_para); start(new_version,update_type,u_para);
for(i=0;i<rule_num;i++) for(i=0;i<rule_num;i++)
{ {
@@ -937,6 +960,7 @@ void redis_monitor_traverse(unsigned int version,redisContext *c,
update(rule_list[i].table_name,rule_list[i].table_line,u_para); update(rule_list[i].table_name,rule_list[i].table_line,u_para);
} }
finish(u_para); finish(u_para);
clean_up:
for(i=0;i<rule_num;i++) for(i=0;i<rule_num;i++)
{ {
empty_serial_rules(rule_list+i); empty_serial_rules(rule_list+i);

4
src/entry/Maat_rule.cpp
View File

@@ -28,7 +28,7 @@
#include "stream_fuzzy_hash.h" #include "stream_fuzzy_hash.h"
#include "gram_index_engine.h" #include "gram_index_engine.h"
int MAAT_FRAME_VERSION_2_0_20170810=1; int MAAT_FRAME_VERSION_2_0_20170811=1;
const char *maat_module="MAAT Frame"; const char *maat_module="MAAT Frame";
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
@@ -3052,7 +3052,7 @@ void maat_finish_cb(void* u_para)
{ {
feather->scanner->cfg_num=total; feather->scanner->cfg_num=total;
feather->scanner->version=feather->maat_version; feather->scanner->version=feather->maat_version;
if(time(NULL)-feather->scanner->last_update_time>60) if(time(NULL)-feather->scanner->last_update_time>feather->effect_interval_ms/1000)
{ {
do_scanner_update(feather->scanner do_scanner_update(feather->scanner
,feather->garbage_q ,feather->garbage_q

2
test/reset_redis4maat.sh
View File

@@ -4,7 +4,7 @@ port="6379"
echo "Reseting Redis For Maat..." echo "Reseting Redis For Maat..."
redis-cli -h $host -p $port GET MAAT_VERSION redis-cli -h $host -p $port GET MAAT_VERSION
redis-cli -h $host -p $port FLUSHALL redis-cli -h $host -p $port FLUSHALL
redis-cli -h $host -p $port SET MAAT_VERSION "1" redis-cli -h $host -p $port SET MAAT_VERSION "0"
redis-cli -h $host -p $port SET SEQUENCE_REGION "1" redis-cli -h $host -p $port SET SEQUENCE_REGION "1"
redis-cli -h $host -p $port SET SEQUENCE_GROUP "1" redis-cli -h $host -p $port SET SEQUENCE_GROUP "1"