diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 81d5419..ffda39b 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -151,16 +151,17 @@ enum MAAT_INIT_OPT MAAT_OPT_REDIS_PORT, //VALUE is a unsigned short or a signed int, host order, SIZE= sizeof(unsigned short) or sizeof(int). No DEFAULT. MAAT_OPT_REDIS_INDEX, //VALUE is interger *, 0~15, SIZE=sizeof(int). DEFAULT: 0. MAAT_OPT_CMD_AUTO_NUMBERING, //VALUE is interger *, 1 or 0, SIZE=sizeof(int). DEFAULT: 1. - MAAT_OPT_DEFERRED_LOAD //VALUE is NULL,SIZE is 0. Default: Deffered initialization OFF. + MAAT_OPT_DEFERRED_LOAD, //VALUE is NULL,SIZE is 0. Default: Deffered initialization OFF. + MAAT_OPT_CUMULATIVE_UPDATE_OFF //VALUE is NULL,SIZE is 0. Default: CUMMULATIVE UPDATE OFF. }; //return -1 if failed, return 0 on success; int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size); enum MAAT_STATE_OPT { - MAAT_STATE_VERSION=1, //Get current maat version. VALUE is interger, SIZE=sizeof(int). + MAAT_STATE_VERSION=1, //Get current maat version. VALUE is long long, SIZE=sizeof(long long). MAAT_STATE_LAST_UPDATING_TABLE //Query at Maat_finish_callback_t to determine whether this table is the last one to update. VALUE is interger, SIZE=sizeof(int), 1:yes, 0: no }; -int Maat_read_feather_state(Maat_feather_t feather,enum MAAT_STATE_OPT type,const void* value,int size); +int Maat_read_state(Maat_feather_t feather, enum MAAT_STATE_OPT type, void* value, int size); void Maat_burn_feather(Maat_feather_t feather); diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index af1a3c9..5d17446 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -617,6 +617,8 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo case MAAT_OPT_DEFERRED_LOAD: _feather->DEFERRED_LOAD_ON=1; break; + case MAAT_OPT_CUMULATIVE_UPDATE_OFF: + _feather->cumulative_update_off=1; default: return -1; } @@ -721,6 +723,13 @@ int Maat_initiate_feather(Maat_feather_t feather) "At initiation: STAT tirigger OFF, TURN OFF PERF trigger."); _feather->perf_on=0; } + if(_feather->cumulative_update_off==1) + { + MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , + "Update with cumulative version OFF."); + } + + maat_stat_init(_feather); pthread_t cfg_mon_t; @@ -1832,3 +1841,34 @@ void Maat_clean_status(scan_status_t* mid) *mid=NULL; return; } +int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* value,int size) +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + int * int_val=(int *)value; + long long* longlong_val=(long long*)value; + switch(type) + { + case MAAT_STATE_VERSION: + if(size!=sizeof(long long)) + { + return -1; + } + if(_feather->new_version!=-1) + { + *longlong_val=_feather->new_version; + } + else + { + *longlong_val=_feather->maat_version; + } + break; + case MAAT_STATE_LAST_UPDATING_TABLE: + *int_val=_feather->is_last_plugin_table_updating; + break; + default: + return -1; + break; + } + return 0; +} + diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index a9e0dc3..add90d9 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -355,7 +355,7 @@ int _wrap_redisReconnect(redisContext* c, void*logger) return -1; } } -int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** list,void* logger, unsigned int* new_version,int *update_type) +int get_rm_key_list(long long version,redisContext *c,struct serial_rule_t** list,void* logger, long long* new_version,int *update_type, int cumulative_off) { redisReply* reply=NULL,*sub_reply=NULL,*tmp_reply=NULL; char err_buff[256]; @@ -392,7 +392,6 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** { return 0; } - *new_version=version_in_redis; if(version==0) { @@ -401,14 +400,18 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** if(version_in_redis Redis: %lld.",version,version_in_redis); + "VERSION roll back MAAT: %lld -> Redis: %lld.",version,version_in_redis); goto FULL_UPDATE; } - + if(version_in_redis>version&&cumulative_off==1) + { + version_in_redis=version+1; + } //Returns all the elements in the sorted set at key with a score that version < score <= version_in_redis. //The elements are considered to be ordered from low to high scores(version). - reply=(redisReply*)redisCommand(c, "ZRANGEBYSCORE %s (%d %d",rm_status_sset,version,version_in_redis); + reply=(redisReply*)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld",rm_status_sset,version,version_in_redis); + if(reply==NULL) { __redis_strerror_r(errno,err_buff,sizeof(err_buff)); @@ -420,7 +423,7 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** if(reply->elements==0) { //a duplicate rule_id would induce this error. - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,"Got nothing after ZRANGEBYSCORE %s (%d %d",rm_status_sset,version,version_in_redis); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,"Got nothing after ZRANGEBYSCORE %s (%lld %lld",rm_status_sset,version,version_in_redis); freeReplyObject(reply); return -1; } @@ -428,7 +431,7 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** if(tmp_reply->type!=REDIS_REPLY_STRING) { MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, - "ZSCORE %s %s failed Version: %d->%d",rm_status_sset,reply->element[0]->str,version, version_in_redis); + "ZSCORE %s %s failed Version: %lld->%lld",rm_status_sset,reply->element[0]->str,version, version_in_redis); free(tmp_reply); free(reply); goto FULL_UPDATE; @@ -440,12 +443,12 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** if(nearest_rule_version!=version+1) { MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, - "Noncontinuous VERSION Redis: %lld MAAT: %d.",nearest_rule_version,version); + "Noncontinuous VERSION Redis: %lld MAAT: %lld.",nearest_rule_version,version); goto FULL_UPDATE; } MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, - "Inc Update form version %d to %lld (%lld entries).",version,version_in_redis,reply->elements); + "Inc Update form version %lld to %lld (%lld entries).",version,version_in_redis,reply->elements); s_rule=(struct serial_rule_t*)calloc(reply->elements,sizeof(struct serial_rule_t)); for(i=0;ielements;i++) @@ -469,6 +472,7 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** *list=s_rule; *update_type=CM_UPDATE_TYPE_INC; freeReplyObject(reply); + *new_version=version_in_redis; return i; FULL_UPDATE: MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, @@ -1048,8 +1052,8 @@ void cleanup_update_status(redisContext *ctx, void *logger) ,entry_num); } -void redis_monitor_traverse(unsigned int version,redisContext *c, - void (*start)(unsigned int ,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para +void redis_monitor_traverse(long long version,redisContext *c, + void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para void (*finish)(void*),//u_para void* u_para, @@ -1060,7 +1064,7 @@ void redis_monitor_traverse(unsigned int version,redisContext *c, int ret=0; struct serial_rule_t* rule_list=NULL; int update_type=CM_UPDATE_TYPE_INC; - unsigned int new_version=0; + long long new_version=0; enum MAAT_TABLE_TYPE table_type; void* logger=feather->logger; if(feather->redis_write_ctx!=NULL)//authorized to write @@ -1069,7 +1073,7 @@ void redis_monitor_traverse(unsigned int version,redisContext *c, check_maat_expiration(feather->redis_read_ctx, logger); cleanup_update_status(feather->redis_read_ctx, logger); } - rule_num=get_rm_key_list(version, c, &rule_list, logger,&new_version, &update_type); + rule_num=get_rm_key_list(version, c, &rule_list, logger,&new_version, &update_type,feather->cumulative_update_off); if(rule_num<0||(rule_num==0&&update_type==CM_UPDATE_TYPE_INC))//error or nothing changed { return; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index e1f6a56..f5e845d 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -30,7 +30,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_1_20171205=1; +int MAAT_FRAME_VERSION_2_1_20171206_dev=1; const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; @@ -2974,13 +2974,14 @@ void clear_plugin_table_info(struct _plugin_table_info *cb_info) cb_info->acc_line_num=0; return; } -void maat_start_cb(unsigned int new_version,int update_type,void*u_para) +void maat_start_cb(long long new_version,int update_type,void*u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; struct _Maat_table_info_t* p_table=NULL; struct _plugin_table_info* p_table_cb=NULL; int i=0,j=0; feather->new_version=new_version; + if(update_type==CM_UPDATE_TYPE_FULL) { feather->update_tmp_scanner=create_maat_scanner(new_version,feather); @@ -3011,15 +3012,18 @@ void maat_start_cb(unsigned int new_version,int update_type,void*u_para) feather->maat_version,new_version); feather->maat_version=new_version; } + feather->active_plugin_table_num=0; for(i=0;ip_table_info[i]; - if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN) + if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->cb_info->cb_plug_cnt==0) { continue; } - feather->active_plugin_table_num++; p_table_cb=p_table->cb_info; + + feather->active_plugin_table_num++; + for(j=0;jcb_plug_cnt;j++) { if(p_table_cb->cb_plug[j].start!=NULL) @@ -3045,16 +3049,18 @@ void maat_finish_cb(void* u_para) total+=p_table->cfg_num; } } + int call_plugin_table_cnt=0; for(i=0;ip_table_info[i]; - if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN) + if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->cb_info->cb_plug_cnt==0) { continue; } + call_plugin_table_cnt++; p_table_cb=p_table->cb_info; - if(i==feather->active_plugin_table_num) + if(call_plugin_table_cnt==feather->active_plugin_table_num) { feather->is_last_plugin_table_updating=1; } diff --git a/src/entry/Maat_rule_internal.h b/src/entry/Maat_rule_internal.h index dc78dfd..363ed1e 100644 --- a/src/entry/Maat_rule_internal.h +++ b/src/entry/Maat_rule_internal.h @@ -326,7 +326,7 @@ struct GIE_aux_t }; struct _Maat_scanner_t { - int version; + long long version; time_t last_update_time; long long *ref_cnt; //optimized for cache_alignment 64 rule_scanner_t region; @@ -361,13 +361,14 @@ struct _Maat_feather_t int still_working; int scan_interval_ms; int effect_interval_ms; + int cumulative_update_off; int stat_on; int perf_on; struct _Maat_table_info_t *p_table_info[MAX_TABLE_NUM]; MESA_htable_handle map_tablename2id; void* logger; - int maat_version; - int last_full_version; + long long maat_version; + long long last_full_version; int scan_thread_num; int rule_scan_type; char inc_dir[MAX_TABLE_NAME_LEN]; @@ -393,7 +394,7 @@ struct _Maat_feather_t pthread_mutex_t redis_write_lock; //protect redis_write_ctx long long base_rgn_seq,base_grp_seq,server_time; //internal states - int new_version; + long long new_version; int active_plugin_table_num; int is_last_plugin_table_updating; @@ -446,7 +447,7 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set); int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); -void maat_start_cb(unsigned int new_version,int update_type,void*u_para); +void maat_start_cb(long long new_version,int update_type,void*u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para); void maat_finish_cb(void* u_para); void *thread_rule_monitor(void *arg); @@ -470,16 +471,17 @@ void maat_stat_output(struct _Maat_feather_t* feather); char* _maat_strdup(const char* s); char* str_unescape(char* s); redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...); -int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** list,void* logger, unsigned int* new_version,int *update_type); +int get_rm_key_list(long long version,redisContext *c,struct serial_rule_t** list,void* logger, long long* new_version,int *update_type, int cumulative_off); int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger,int print_process); void empty_serial_rules(struct serial_rule_t* rule); -void redis_monitor_traverse(unsigned int version,redisContext *c, - void (*start)(unsigned int ,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para - int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para - void (*finish)(void*),//u_para - void* u_para, - const unsigned char* dec_key, - _Maat_feather_t* feather); +void redis_monitor_traverse(long long version,redisContext *c, + void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para + int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para + void (*finish)(void*),//u_para + void* u_para, + const unsigned char* dec_key, + _Maat_feather_t* feather); + const char* module_name_str(const char*name); #define maat_module (module_name_str("MAAT_Frame")) diff --git a/src/entry/config_monitor.cpp b/src/entry/config_monitor.cpp index d79c83e..24f0aec 100644 --- a/src/entry/config_monitor.cpp +++ b/src/entry/config_monitor.cpp @@ -210,14 +210,14 @@ int my_scandir(const char *dir, struct dirent ***namelist, return (strncmp(ent->d_name,"full_config_index",strlen("full_config_index")) == 0|| strncmp(ent->d_name,"inc_config_index",strlen("inc_config_index")) == 0); } -int get_new_idx_path(unsigned int current_version,const char*file_dir,void* logger,char*** idx_path,int*idx_num) +int get_new_idx_path(long long current_version,const char*file_dir,void* logger,char*** idx_path,int*idx_num) { struct dirent **namelist; int n=0,i=0,sscanf_ret; char update_str[32]={0}; - unsigned int latest_ful_version=0,latest_inc_version=0; - unsigned int config_seq=0; + long long latest_ful_version=0,latest_inc_version=0; + long long config_seq=0; int *inc_file_idx; int full_file_idx=0,inc_idx_num=0,path_len=0; @@ -247,7 +247,7 @@ int get_new_idx_path(unsigned int current_version,const char*file_dir,void* log ,namelist[i]->d_name); continue; } - sscanf_ret=sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%u",update_str,&config_seq); + sscanf_ret=sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld",update_str,&config_seq); if(sscanf_ret!=2) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor @@ -455,17 +455,18 @@ const char* path2filename(const char*path) } return path+i+1; } -void config_monitor_traverse(unsigned int version,const char*idx_dir, - void (*start)(unsigned int ,int ,void*), - int (*update)(const char* ,const char*,void* ), - void (*finish)(void*), +void config_monitor_traverse(long long version,const char*idx_dir, + void (*start)(long long, int, void*),//vesion,CM_UPDATE_TYPE_*,u_para + int (*update)(const char*, const char*, void*),//table name ,line ,u_para + void (*finish)(void*),//u_para void* u_para, const unsigned char* dec_key, void* logger) + { int update_type=CM_UPDATE_TYPE_NONE; - unsigned int new_version=0; + long long new_version=0; char**idx_path_array=NULL; const char* table_filename=NULL; char str_not_care[256]={0}; @@ -488,7 +489,7 @@ void config_monitor_traverse(unsigned int version,const char*idx_dir, break; } table_filename=path2filename(idx_path_array[i]); - sscanf(table_filename,"%[a-zA-Z]_config_index.%u",str_not_care,&new_version); + sscanf(table_filename,"%[a-zA-Z]_config_index.%lld",str_not_care,&new_version); start(new_version,update_type,u_para); for(j=0;j