修复_exec_serial_rule_begin中maat_redis_version多加1的bug。该bug影响主版本。

支持运行中暂停后台配置更新,通过MAAT_OPT_DISABLE_UPDATE选项设置。
This commit is contained in:
zhengchao
2018-06-17 20:03:17 +08:00
parent 8ab53fae3b
commit 9f649a0eb1
6 changed files with 199 additions and 128 deletions

View File

@@ -153,7 +153,9 @@ enum MAAT_INIT_OPT
MAAT_OPT_CMD_AUTO_NUMBERING, //VALUE is a interger *, 1 or 0, SIZE=sizeof(int). DEFAULT: 1. MAAT_OPT_CMD_AUTO_NUMBERING, //VALUE is a interger *, 1 or 0, SIZE=sizeof(int). DEFAULT: 1.
MAAT_OPT_DEFERRED_LOAD, //VALUE is NULL,SIZE is 0. Default: Deffered initialization OFF. MAAT_OPT_DEFERRED_LOAD, //VALUE is NULL,SIZE is 0. Default: Deffered initialization OFF.
MAAT_OPT_CUMULATIVE_UPDATE_OFF, //VALUE is NULL,SIZE is 0. Default: CUMMULATIVE UPDATE ON. MAAT_OPT_CUMULATIVE_UPDATE_OFF, //VALUE is NULL,SIZE is 0. Default: CUMMULATIVE UPDATE ON.
MAAT_OPT_LOAD_VERSION_FROM //VALUE is a long long, SIZE=sizeof(long long). Default: Load the Latest. Only valid in redis mode, and maybe failed for too old. MAAT_OPT_LOAD_VERSION_FROM, //VALUE is a long long, SIZE=sizeof(long long). Default: Load the Latest. Only valid in redis mode, and maybe failed for too old.
//This option also disables background update.
MAAT_OPT_DISABLE_UPDATE //VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled. DEFAULT: Backgroud update is enabled. Runtime setting is allowed.
}; };
//return -1 if failed, return 0 on success; //return -1 if failed, return 0 on success;
int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size); int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size);

View File

@@ -484,7 +484,8 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
feather->AUTO_NUMBERING_ON=1; feather->AUTO_NUMBERING_ON=1;
feather->connect_timeout.tv_sec=0; feather->connect_timeout.tv_sec=0;
feather->connect_timeout.tv_usec=100*1000; // 100 ms feather->connect_timeout.tv_usec=100*1000; // 100 ms
pthread_mutex_init(&(feather->plugin_table_reg_mutex),NULL); feather->backgroud_update_enabled=1;
pthread_mutex_init(&(feather->backgroud_update_mutex),NULL);
pthread_mutex_init(&(feather->redis_write_lock),NULL); pthread_mutex_init(&(feather->redis_write_lock),NULL);
snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path); snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path);
return feather; return feather;
@@ -493,7 +494,41 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
{ {
_Maat_feather_t* _feather=(_Maat_feather_t*)feather; _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int intval=0,ret=-1; int intval=0,ret=-1;
if(_feather->still_working==1)// not allowed set after Maat_initiate_feather; switch(type)
{
case MAAT_OPT_DISABLE_UPDATE:
intval=*((int*)value);
if(_feather->backgroud_update_enabled!=intval)
{
if(intval==0)
{
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"Background update is disabled, current version %lld."
,_feather->maat_version);
}
else
{
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"Background update is enabled, current version %lld."
,_feather->maat_version);
}
_feather->backgroud_update_enabled=intval;
}
else
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"Duplicated operation, background update is ALREADY %s, current version %lld."
,_feather->backgroud_update_enabled?"enabled":"disabled"
,_feather->maat_version);
}
return 0;
default:
break;
}
if(_feather->still_working==1)//The following options are not allowed to set after initiation;
{ {
return -2; return -2;
} }
@@ -501,7 +536,7 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
{ {
case MAAT_OPT_EFFECT_INVERVAL_MS: case MAAT_OPT_EFFECT_INVERVAL_MS:
intval=*(const int*)value; intval=*(const int*)value;
if(size!=sizeof(int)||intval<=0) if(size!=sizeof(int)||intval<0)
{ {
return -1; return -1;
} }
@@ -623,7 +658,12 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
_feather->cumulative_update_off=1; _feather->cumulative_update_off=1;
break; break;
case MAAT_OPT_LOAD_VERSION_FROM: case MAAT_OPT_LOAD_VERSION_FROM:
_feather->load_from_specific_version=*((long long*)value); _feather->load_version_from=*((long long*)value);
_feather->backgroud_update_enabled=1;
pthread_mutex_lock((&_feather->backgroud_update_mutex));
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"Maat load version from %lld, stops backgroud update."
,_feather->load_version_from);
break; break;
default: default:
return -1; return -1;
@@ -853,11 +893,12 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
{ {
return -1; return -1;
} }
pthread_mutex_lock(&(_feather->plugin_table_reg_mutex)); //plugin table register blocks background update.
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
idx=p_table->cb_info->cb_plug_cnt; idx=p_table->cb_info->cb_plug_cnt;
if(idx==MAX_PLUGING_NUM) if(idx==MAX_PLUGING_NUM)
{ {
pthread_mutex_unlock(&(_feather->plugin_table_reg_mutex)); pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return -1; return -1;
} }
p_table->cb_info->cb_plug_cnt++; p_table->cb_info->cb_plug_cnt++;
@@ -885,7 +926,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
finish(u_para); finish(u_para);
} }
} }
pthread_mutex_unlock(&(_feather->plugin_table_reg_mutex)); pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return 1; return 1;
} }
int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id

View File

@@ -552,7 +552,7 @@ int get_rm_key_list(redisContext *c, long long instance_version, long long desir
goto FULL_UPDATE; goto FULL_UPDATE;
} }
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
"Inc Update form instance_version %lld to %lld (%lld entries).",instance_version,target_version,rule_num); "Inc Update form instance_version %lld to %lld (%d entries).",instance_version,target_version,rule_num);
*list=s_rule_array; *list=s_rule_array;
*update_type=CM_UPDATE_TYPE_INC; *update_type=CM_UPDATE_TYPE_INC;
*new_version=target_version; *new_version=target_version;
@@ -935,7 +935,6 @@ long long _exec_serial_rule_begin(redisContext* ctx)
redisReply* data_reply=NULL; redisReply* data_reply=NULL;
data_reply=_wrap_redisCommand(ctx, "INCRBY MAAT_PRE_VER 1"); data_reply=_wrap_redisCommand(ctx, "INCRBY MAAT_PRE_VER 1");
maat_redis_version=read_redis_integer(data_reply); maat_redis_version=read_redis_integer(data_reply);
maat_redis_version++;
freeReplyObject(data_reply); freeReplyObject(data_reply);
data_reply=_wrap_redisCommand(ctx,"MULTI"); data_reply=_wrap_redisCommand(ctx,"MULTI");
return maat_redis_version; return maat_redis_version;
@@ -1285,8 +1284,8 @@ void redis_monitor_traverse(long long version,redisContext *c,
return; return;
} }
rule_num=get_rm_key_list(c, version, feather->load_from_specific_version, &new_version, &rule_list, &update_type, logger, feather->cumulative_update_off); rule_num=get_rm_key_list(c, version, feather->load_version_from, &new_version, &rule_list, &update_type, logger, feather->cumulative_update_off);
feather->load_from_specific_version=0;//only valid for one time. feather->load_version_from=0;//only valid for one time.
if(rule_num<0||(rule_num==0&&update_type==CM_UPDATE_TYPE_INC))//error or nothing changed if(rule_num<0||(rule_num==0&&update_type==CM_UPDATE_TYPE_INC))//error or nothing changed
{ {
return; return;
@@ -1317,6 +1316,8 @@ void redis_monitor_traverse(long long version,redisContext *c,
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor,"%d of %d rules are empty.",empty_value_num,rule_num); MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor,"%d of %d rules are empty.",empty_value_num,rule_num);
} }
} }
start(new_version,update_type,u_para);
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Start %s update: %lld->%lld (%d entries)\n",
update_type==CM_UPDATE_TYPE_INC?"INC":"FULL",version,new_version,rule_num); update_type==CM_UPDATE_TYPE_INC?"INC":"FULL",version,new_version,rule_num);
for(i=0;i<rule_num;i++) for(i=0;i<rule_num;i++)
{ {
@@ -1340,8 +1341,10 @@ void redis_monitor_traverse(long long version,redisContext *c,
continue; continue;
} }
} }
update(rule_list[i].table_name,rule_list[i].table_line,u_para);
//printf("%s %s,%d\n", rule_list[i].op==MAAT_OP_DEL?"DEL":"ADD", rule_list[i].table_name, rule_list[i].rule_id); //printf("%s %s,%d\n", rule_list[i].op==MAAT_OP_DEL?"DEL":"ADD", rule_list[i].table_name, rule_list[i].rule_id);
} }
finish(u_para);
//printf("Finish %s update: %lld->%lld\n",update_type==CM_UPDATE_TYPE_INC?"INC":"FULL",version,new_version); //printf("Finish %s update: %lld->%lld\n",update_type==CM_UPDATE_TYPE_INC?"INC":"FULL",version,new_version);
clean_up: clean_up:
for(i=0;i<rule_num;i++) for(i=0;i<rule_num;i++)

View File

@@ -30,7 +30,7 @@
#include "stream_fuzzy_hash.h" #include "stream_fuzzy_hash.h"
#include "gram_index_engine.h" #include "gram_index_engine.h"
int MAAT_FRAME_VERSION_2_2_20180607=1; int MAAT_FRAME_VERSION_2_2_20180617=1;
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};
@@ -802,6 +802,12 @@ void* create_bool_matcher(MESA_htable_handle compile_hash,int thread_num,void* l
MESA_htable_iterate(compile_hash, walk_compile_hash, update_q); MESA_htable_iterate(compile_hash, walk_compile_hash, update_q);
const long q_cnt=MESA_lqueue_get_count(update_q); const long q_cnt=MESA_lqueue_get_count(update_q);
if(q_cnt==0)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
"No compile rule to build a bool matcher.");
return NULL;
}
set_array=(universal_bool_expr_t*)malloc(sizeof(universal_bool_expr_t)*q_cnt); set_array=(universal_bool_expr_t*)malloc(sizeof(universal_bool_expr_t)*q_cnt);
for(i=0;i<q_cnt;i++) for(i=0;i<q_cnt;i++)
{ {
@@ -2087,7 +2093,7 @@ int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_g
if(ret<0) if(ret<0)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module,
"update error,add %s group %d to compile %d error,compile rule is full or duplicate group." "update error,add group: %s %d to compile rule %d error, compile rule is full or duplicate group."
,table->table_name[table->updating_name] ,table->table_name[table->updating_name]
,db_group_rule->group_id ,db_group_rule->group_id
,db_group_rule->compile_id); ,db_group_rule->compile_id);
@@ -2163,9 +2169,12 @@ int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t*
return -1; return -1;
} }
pthread_rwlock_wrlock(&(compile_rule->rwlock)); pthread_rwlock_wrlock(&(compile_rule->rwlock));
if(compile_rule->db_c_rule!=NULL)
{
free(compile_rule->db_c_rule->service_defined); free(compile_rule->db_c_rule->service_defined);
free(compile_rule->db_c_rule); free(compile_rule->db_c_rule);
compile_rule->db_c_rule=NULL; compile_rule->db_c_rule=NULL;
}
pthread_rwlock_unlock(&(compile_rule->rwlock)); pthread_rwlock_unlock(&(compile_rule->rwlock));
if(compile_rule->group_cnt==0) if(compile_rule->group_cnt==0)
@@ -2198,7 +2207,7 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s
if(ret<0) if(ret<0)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"duplicate config of group table %s group_id %d compile_id %d.",table->table_name[table->conj_cnt] "duplicate config of group table %s group_id %d compile_id %d.",table->table_name[0]
,db_group_rule.group_id ,db_group_rule.group_id
,db_group_rule.compile_id); ,db_group_rule.compile_id);
@@ -3084,7 +3093,7 @@ void maat_finish_cb(void* u_para)
feather->scanner->version=feather->maat_version; feather->scanner->version=feather->maat_version;
expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_total_q_size; feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_total_q_size;
if(time(NULL)-feather->scanner->last_update_time>feather->effect_interval_ms/1000) if(time(NULL)-feather->scanner->last_update_time>=feather->effect_interval_ms/1000)
{ {
do_scanner_update(feather->scanner do_scanner_update(feather->scanner
,feather->garbage_q ,feather->garbage_q
@@ -3204,8 +3213,8 @@ void *thread_rule_monitor(void *arg)
{ {
usleep(feather->scan_interval_ms*1000); usleep(feather->scan_interval_ms*1000);
scan_dir_cnt++; scan_dir_cnt++;
//plugin table register is not allowed during update; if(0==pthread_mutex_trylock(&(feather->backgroud_update_mutex)))
pthread_mutex_lock(&(feather->plugin_table_reg_mutex)); {
if(feather->REDIS_MODE_ON==1) if(feather->REDIS_MODE_ON==1)
{ {
redis_monitor_traverse(feather->maat_version redis_monitor_traverse(feather->maat_version
@@ -3228,7 +3237,7 @@ void *thread_rule_monitor(void *arg)
feather->decrypt_key, feather->decrypt_key,
feather->logger); feather->logger);
} }
pthread_mutex_unlock(&(feather->plugin_table_reg_mutex));
if(feather->update_tmp_scanner!=NULL) if(feather->update_tmp_scanner!=NULL)
{ {
old_scanner=feather->scanner; old_scanner=feather->scanner;
@@ -3261,7 +3270,7 @@ void *thread_rule_monitor(void *arg)
{ {
expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_total_q_size; feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_total_q_size;
if(feather->postpone_q_size>0&&time(NULL)-feather->scanner->last_update_time>feather->effect_interval_ms/1000) if(feather->postpone_q_size>0&&time(NULL)-feather->scanner->last_update_time>=feather->effect_interval_ms/1000)
{ {
do_scanner_update(feather->scanner do_scanner_update(feather->scanner
,feather->garbage_q ,feather->garbage_q
@@ -3273,8 +3282,10 @@ void *thread_rule_monitor(void *arg)
feather->scanner->version,feather->scanner->cfg_num); feather->scanner->version,feather->scanner->cfg_num);
} }
} }
pthread_mutex_unlock(&(feather->backgroud_update_mutex));
}
garbage_bury(feather->garbage_q,feather->effect_interval_ms/1000+10,feather->logger); garbage_bury(feather->garbage_q,feather->effect_interval_ms/1000+10,feather->logger);
if(feather->stat_on==1&&scan_dir_cnt%2==0)//output every 2 seconds if(feather->stat_on==1&&time(NULL)%2==0)//output every 2 seconds
{ {
maat_stat_output(feather); maat_stat_output(feather);
} }

View File

@@ -378,7 +378,7 @@ struct _Maat_feather_t
char table_info_fn[MAX_TABLE_NAME_LEN]; char table_info_fn[MAX_TABLE_NAME_LEN];
char compile_tn[MAX_TABLE_NAME_LEN]; char compile_tn[MAX_TABLE_NAME_LEN];
char group_tn[MAX_TABLE_NAME_LEN]; char group_tn[MAX_TABLE_NAME_LEN];
pthread_mutex_t plugin_table_reg_mutex; pthread_mutex_t backgroud_update_mutex;
unsigned char decrypt_key[MAX_TABLE_NAME_LEN]; unsigned char decrypt_key[MAX_TABLE_NAME_LEN];
char redis_ip[MAX_TABLE_NAME_LEN]; char redis_ip[MAX_TABLE_NAME_LEN];
@@ -393,13 +393,14 @@ struct _Maat_feather_t
struct _Maat_cmd_inner_t* cmd_qhead, *cmd_qtail; struct _Maat_cmd_inner_t* cmd_qhead, *cmd_qtail;
pthread_mutex_t redis_write_lock; //protect redis_write_ctx pthread_mutex_t redis_write_lock; //protect redis_write_ctx
long long base_rgn_seq,base_grp_seq,server_time; long long base_rgn_seq,base_grp_seq,server_time;
long long load_from_specific_version; long long load_version_from;
//internal states //internal states
long long new_version; long long new_version;
int active_plugin_table_num; int active_plugin_table_num;
int is_last_plugin_table_updating; int is_last_plugin_table_updating;
//for stat>>>> //for stat>>>>
int backgroud_update_enabled;
screen_stat_handle_t stat_handle; screen_stat_handle_t stat_handle;
int total_stat_id; int total_stat_id;
int fs_status_id[MAX_MAAT_STAT_NUM]; int fs_status_id[MAX_MAAT_STAT_NUM];

View File

@@ -22,6 +22,7 @@ unsigned short test_maat_redis_port=6379;
const char* json_path="./maat_json.json"; const char* json_path="./maat_json.json";
const char* ful_cfg_dir="./rule/full/index/"; const char* ful_cfg_dir="./rule/full/index/";
const char* inc_cfg_dir="./rule/inc/index/"; const char* inc_cfg_dir="./rule/inc/index/";
#define WAIT_FOR_EFFECTIVE_US 1000*1000
extern int my_scandir(const char *dir, struct dirent ***namelist, extern int my_scandir(const char *dir, struct dirent ***namelist,
int(*filter)(const struct dirent *), int(*filter)(const struct dirent *),
int(*compar)(const void *, const void *)); int(*compar)(const void *, const void *));
@@ -575,7 +576,6 @@ void test_set_cmd_line(Maat_feather_t feather)
struct Maat_line_t line_rule[TEST_CMD_LINE_NUM]; struct Maat_line_t line_rule[TEST_CMD_LINE_NUM];
char table_line[TEST_CMD_LINE_NUM][128]; char table_line[TEST_CMD_LINE_NUM][128];
int ret=0,i=0; int ret=0,i=0;
long long version=0;
memset(&line_rule,0,sizeof(line_rule)); memset(&line_rule,0,sizeof(line_rule));
for(i=0;i<TEST_CMD_LINE_NUM;i++) for(i=0;i<TEST_CMD_LINE_NUM;i++)
{ {
@@ -587,21 +587,19 @@ void test_set_cmd_line(Maat_feather_t feather)
line_rule[i].expire_after=0; line_rule[i].expire_after=0;
p_line[i]=line_rule+i; p_line[i]=line_rule+i;
} }
ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD); ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
assert(ret==0); assert(ret==0);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version)); usleep(WAIT_FOR_EFFECTIVE_US);
assert(ret==0);
printf("Maat Version %lld after add lines.\n",version);
sleep(1);
for(i=0;i<TEST_CMD_LINE_NUM;i++) for(i=0;i<TEST_CMD_LINE_NUM;i++)
{ {
line_rule[i].table_line=NULL; line_rule[i].table_line=NULL;
} }
ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL); ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
assert(ret==0); assert(ret==0);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version));
assert(ret==0);
printf("Maat Version %lld after delete lines.\n",version);
return; return;
} }
@@ -665,9 +663,7 @@ int test_add_ip_command(Maat_feather_t feather,const char* region_table)
struct Maat_cmd_t* cmd=NULL; struct Maat_cmd_t* cmd=NULL;
struct Maat_rule_t rule; struct Maat_rule_t rule;
int config_id=0,timeout=2; int config_id=0,timeout=2;
long long version_before=0,version_after=0;
struct Maat_region_t region; struct Maat_region_t region;
int group_num=1,ret=0; int group_num=1,ret=0;
@@ -699,17 +695,21 @@ int test_add_ip_command(Maat_feather_t feather,const char* region_table)
region.ip_rule.protocol=0;//means any protocol should hit. region.ip_rule.protocol=0;//means any protocol should hit.
Maat_add_region2cmd(cmd, 0, &region); Maat_add_region2cmd(cmd, 0, &region);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
ret=Maat_cmd(feather, cmd, MAAT_OP_ADD); ret=Maat_cmd(feather, cmd, MAAT_OP_ADD);
if(ret<0) if(ret<0)
{ {
printf("Add Maat command %d failed.\n",rule.config_id); printf("Add command %d failed.\n",rule.config_id);
Maat_free_cmd(cmd); Maat_free_cmd(cmd);
return 0; return 0;
} }
Maat_free_cmd(cmd); Maat_free_cmd(cmd);
//TEST if the command go into effective. //TEST if the command go into effective.
sleep(1); //waiting for commands go into effect
usleep(WAIT_FOR_EFFECTIVE_US); //waiting for commands go into effect
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_after, sizeof(version_after));
struct ipaddr ipv4_addr; struct ipaddr ipv4_addr;
struct stream_tuple4_v4 v4_addr; struct stream_tuple4_v4 v4_addr;
ipv4_addr.addrtype=ADDR_TYPE_IPV4; ipv4_addr.addrtype=ADDR_TYPE_IPV4;
@@ -737,27 +737,27 @@ int test_add_ip_command(Maat_feather_t feather,const char* region_table)
} }
else else
{ {
printf("Test Maat add IP rule Failed.\n"); printf("Test add IP rule Failed, version %lld->%lld.\n",version_before,version_after);
} }
} }
Maat_clean_status(&mid); Maat_clean_status(&mid);
sleep(timeout+1);//wait for commands expired. usleep(timeout*1000*1000+WAIT_FOR_EFFECTIVE_US);//wait for commands expired.
ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,&result,1, &mid,0); ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,&result,1, &mid,0);
if(ret==0) if(ret==0)
{ {
printf("Test Maat expired IP rule Success.\n"); printf("Test expire IP rule Success.\n");
} }
else else
{ {
printf("Test Maat expired IP rule Failed.\n"); printf("Test expire IP rule Failed.\n");
} }
Maat_clean_status(&mid); Maat_clean_status(&mid);
return 0; return 0;
} }
int test_del_command(Maat_feather_t feather,int config_id) int del_command(Maat_feather_t feather,int config_id)
{ {
struct Maat_cmd_t* cmd=NULL; struct Maat_cmd_t* cmd=NULL;
struct Maat_rule_t rule; struct Maat_rule_t rule;
@@ -773,7 +773,7 @@ int test_del_command(Maat_feather_t feather,int config_id)
Maat_free_cmd(cmd); Maat_free_cmd(cmd);
return 0; return 0;
} }
void test_command(Maat_feather_t feather) void test_expr_command(Maat_feather_t feather)
{ {
const char* scan_data="Hiredis is a minimalistic C client library for the Redis database.\r\n"; const char* scan_data="Hiredis is a minimalistic C client library for the Redis database.\r\n";
const char* table_name="HTTP_URL"; const char* table_name="HTTP_URL";
@@ -789,12 +789,18 @@ void test_command(Maat_feather_t feather)
struct Maat_rule_t result; struct Maat_rule_t result;
int timeout=0;//seconds int timeout=0;//seconds
int label_id=5210; int label_id=5210;
long long version_before=0,version_after=0;
Maat_str_escape(escape_buff1, sizeof(escape_buff1),keywords1); Maat_str_escape(escape_buff1, sizeof(escape_buff1),keywords1);
Maat_str_escape(escape_buff2, sizeof(escape_buff2),keywords2); Maat_str_escape(escape_buff2, sizeof(escape_buff2),keywords2);
snprintf(keywords,sizeof(keywords),"%s&%s",escape_buff1,escape_buff2); snprintf(keywords,sizeof(keywords),"%s&%s",escape_buff1,escape_buff2);
config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1); config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
test_add_expr_command(feather,table_name,config_id, 0, label_id, keywords); test_add_expr_command(feather,table_name,config_id, 0, label_id, keywords);
sleep(1);//waiting for commands go into effect usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_after, sizeof(version_after));
table_id=Maat_table_register(feather,table_name); table_id=Maat_table_register(feather,table_name);
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
&result,NULL, 1, &result,NULL, 1,
@@ -802,50 +808,51 @@ void test_command(Maat_feather_t feather)
if(ret>0&&result.config_id==config_id) if(ret>0&&result.config_id==config_id)
{ {
printf("Test Maat add EXPR rule success %s\n",print_maat_result(&result,ret)); printf("Test add EXPR rule success %s\n",print_maat_result(&result,ret));
} }
else else
{ {
printf("Test Maat add EXPR rule failed.\n"); printf("Test add EXPR rule failed, version %lld->%lld.\n",version_before,version_after);
} }
Maat_clean_status(&mid); Maat_clean_status(&mid);
output_id_cnt=Maat_cmd_select(feather,label_id, output_ids, 4); output_id_cnt=Maat_cmd_select(feather,label_id, output_ids, 4);
if(output_id_cnt==1&&output_ids[0]==config_id) if(output_id_cnt==1&&output_ids[0]==config_id)
{ {
printf("Test Maat select command success.\n"); printf("Test select command success.\n");
} }
else else
{ {
printf("Test Maat select command label %d failed.\n",label_id); printf("Test select command label %d failed.\n",label_id);
} }
test_del_command(feather, config_id); del_command(feather, config_id);
sleep(1);//waiting for commands go into effect usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
&result,NULL, 1, &result,NULL, 1,
&mid, 0); &mid, 0);
if(ret>0) if(ret>0)
{ {
printf("Test Maat delete EXPR command failed\n"); printf("Test delete EXPR command failed.\n");
} }
else else
{ {
printf("Test Maat delete EXPR command success.\n"); printf("Test delete EXPR command success.\n");
} }
Maat_clean_status(&mid); Maat_clean_status(&mid);
timeout=1; timeout=1;
config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
test_add_expr_command(feather,table_name,config_id, timeout, label_id, keywords); test_add_expr_command(feather,table_name,config_id, timeout, label_id, keywords);
sleep(timeout+1); usleep(timeout*1000*1000+WAIT_FOR_EFFECTIVE_US);
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
&result,NULL, 1, &result,NULL, 1,
&mid, 0); &mid, 0);
if(ret>0&&result.config_id==config_id)//should not hit if(ret>0&&result.config_id==config_id)//should not hit
{ {
printf("Test Maat command expire EXPR failed."); printf("Test command expire EXPR failed.\n");
} }
else else
{ {
printf("Test Maat command expire success.\n"); printf("Test command expire success.\n");
} }
} }
#define FILE_CHUNK_SIZE 4096 #define FILE_CHUNK_SIZE 4096
@@ -960,13 +967,13 @@ int main(int argc,char* argv[])
const char* stat_file="./scan_staus.log"; const char* stat_file="./scan_staus.log";
const char* decrypt_key="mesa2017wy"; const char* decrypt_key="mesa2017wy";
const char* test_digest_file="./testdata/digest_test.data"; const char* test_digest_file="./testdata/digest_test.data";
int scan_interval_ms=10; int scan_interval_ms=1;
int effective_interval_ms=10; int effective_interval_ms=0;
int scan_detail=0,deferred_load_on=0; int scan_detail=0,deferred_load_on=0;
int using_redis=0; int using_redis=0;
scan_status_t mid=NULL; scan_status_t mid=NULL;
int wait_second=400; int rest_second=400;
if(argc<2||argv[1][0]!='-') if(argc<2||argv[1][0]!='-')
{ {
@@ -986,7 +993,7 @@ int main(int argc,char* argv[])
case 'u'://update case 'u'://update
Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, ful_cfg_dir, strlen(ful_cfg_dir)+1); Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, ful_cfg_dir, strlen(ful_cfg_dir)+1);
Maat_set_feather_opt(feather, MAAT_OPT_INC_CFG_DIR, inc_cfg_dir, strlen(inc_cfg_dir)+1); Maat_set_feather_opt(feather, MAAT_OPT_INC_CFG_DIR, inc_cfg_dir, strlen(inc_cfg_dir)+1);
wait_second=14; rest_second=14;
break; break;
case 'r'://redis case 'r'://redis
Maat_set_feather_opt(feather, MAAT_OPT_REDIS_IP, test_maat_redis_ip, strlen(test_maat_redis_ip)+1); Maat_set_feather_opt(feather, MAAT_OPT_REDIS_IP, test_maat_redis_ip, strlen(test_maat_redis_ip)+1);
@@ -1010,7 +1017,7 @@ int main(int argc,char* argv[])
Maat_set_feather_opt(feather, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms)); Maat_set_feather_opt(feather, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms));
//Set a short intevral for testing. //Set a short intevral for testing.
Maat_set_feather_opt(feather, MAAT_OPT_EFFECT_INVERVAL_MS,&effective_interval_ms, sizeof(scan_interval_ms)); Maat_set_feather_opt(feather, MAAT_OPT_EFFECT_INVERVAL_MS,&effective_interval_ms, sizeof(effective_interval_ms));
Maat_set_feather_opt(feather, MAAT_OPT_STAT_FILE_PATH, stat_file, strlen(stat_file)+1); Maat_set_feather_opt(feather, MAAT_OPT_STAT_FILE_PATH, stat_file, strlen(stat_file)+1);
Maat_set_feather_opt(feather, MAAT_OPT_STAT_ON, NULL, 0); Maat_set_feather_opt(feather, MAAT_OPT_STAT_ON, NULL, 0);
@@ -1026,7 +1033,7 @@ int main(int argc,char* argv[])
if(deferred_load_on==1) if(deferred_load_on==1)
{ {
printf("Deferred Load ON, Waiting...\n"); printf("Deferred Load ON, Waiting...\n");
sleep(1); usleep(WAIT_FOR_EFFECTIVE_US);
} }
test_plugin_table(feather, "QD_ENTRY_INFO", test_plugin_table(feather, "QD_ENTRY_INFO",
Maat_read_entry_start_cb, Maat_read_entry_start_cb,
@@ -1073,20 +1080,26 @@ int main(int argc,char* argv[])
Maat_clean_status(&mid); Maat_clean_status(&mid);
test_offset_str_scan(feather,"IMAGE_FP"); test_offset_str_scan(feather,"IMAGE_FP");
int value=0;
if(1==using_redis) if(1==using_redis)
{ {
test_command(feather);
test_set_cmd_line(feather);
test_add_ip_command(feather,"IP_CONFIG");
if(0==Maat_cmd_flushDB(feather)) if(0==Maat_cmd_flushDB(feather))
{ {
printf("Test empty Redis Success.\n"); printf("Flush Redis Database.\n");
} }
test_expr_command(feather);
test_set_cmd_line(feather); test_set_cmd_line(feather);
test_add_ip_command(feather,"IP_CONFIG");
Maat_set_feather_opt(feather, MAAT_OPT_DISABLE_UPDATE, &value, sizeof(value));
printf("Disable background udpate.\n");
test_set_cmd_line(feather);
value=1;
Maat_set_feather_opt(feather, MAAT_OPT_DISABLE_UPDATE, &value, sizeof(value));
printf("Enable background udpate.\n");
} }
test_sfh_digest(test_digest_file); test_sfh_digest(test_digest_file);
sleep(wait_second); sleep(rest_second);
Maat_burn_feather(feather); Maat_burn_feather(feather);