gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAAT版本号从32位升级到64位，同时支持多个内部状态暴露，支持设置逐版本号加载。

Browse Source
This commit is contained in:
zhengchao
2017-12-06 18:12:32 +08:00
parent 6e7e8214cc
commit 9aa6917b31
9 changed files with 129 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
inc/Maat_rule.h
View File

@@ -151,16 +151,17 @@ enum MAAT_INIT_OPT
MAAT_OPT_REDIS_PORT, //VALUE is a unsigned short or a signed int, host order, SIZE= sizeof(unsigned short) or sizeof(int). No DEFAULT.
MAAT_OPT_REDIS_INDEX, //VALUE is interger *, 0~15, SIZE=sizeof(int). DEFAULT: 0.
MAAT_OPT_CMD_AUTO_NUMBERING, //VALUE is interger *, 1 or 0, SIZE=sizeof(int). DEFAULT: 1.
MAAT_OPT_DEFERRED_LOAD //VALUE is NULL,SIZE is 0. Default: Deffered initialization OFF.
MAAT_OPT_DEFERRED_LOAD, //VALUE is NULL,SIZE is 0. Default: Deffered initialization OFF.
MAAT_OPT_CUMULATIVE_UPDATE_OFF //VALUE is NULL,SIZE is 0. Default: CUMMULATIVE UPDATE OFF.
};
//return -1 if failed, return 0 on success;
int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size);
enum MAAT_STATE_OPT
{
MAAT_STATE_VERSION=1, //Get current maat version. VALUE is interger, SIZE=sizeof(int).
MAAT_STATE_VERSION=1, //Get current maat version. VALUE is long long, SIZE=sizeof(long long).
MAAT_STATE_LAST_UPDATING_TABLE //Query at Maat_finish_callback_t to determine whether this table is the last one to update. VALUE is interger, SIZE=sizeof(int), 1:yes, 0: no
};
int Maat_read_feather_state(Maat_feather_t feather,enum MAAT_STATE_OPT type,const void* value,int size);
int Maat_read_state(Maat_feather_t feather, enum MAAT_STATE_OPT type, void* value, int size);
void Maat_burn_feather(Maat_feather_t feather);

40
src/entry/Maat_api.cpp
View File

@@ -617,6 +617,8 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
case MAAT_OPT_DEFERRED_LOAD:
_feather->DEFERRED_LOAD_ON=1;
break;
case MAAT_OPT_CUMULATIVE_UPDATE_OFF:
_feather->cumulative_update_off=1;
default:
return -1;
}
@@ -721,6 +723,13 @@ int Maat_initiate_feather(Maat_feather_t feather)
"At initiation: STAT tirigger OFF, TURN OFF PERF trigger.");
_feather->perf_on=0;
}
if(_feather->cumulative_update_off==1)
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"Update with cumulative version OFF.");
}
maat_stat_init(_feather);
pthread_t cfg_mon_t;
@@ -1832,3 +1841,34 @@ void Maat_clean_status(scan_status_t* mid)
*mid=NULL;
return;
}
int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* value,int size)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int * int_val=(int *)value;
long long* longlong_val=(long long*)value;
switch(type)
{
case MAAT_STATE_VERSION:
if(size!=sizeof(long long))
{
return -1;
}
if(_feather->new_version!=-1)
{
*longlong_val=_feather->new_version;
}
else
{
*longlong_val=_feather->maat_version;
}
break;
case MAAT_STATE_LAST_UPDATING_TABLE:
*int_val=_feather->is_last_plugin_table_updating;
break;
default:
return -1;
break;
}
return 0;
}

30
src/entry/Maat_command.cpp
View File

@@ -355,7 +355,7 @@ int _wrap_redisReconnect(redisContext* c, void*logger)
return -1;
}
}
int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** list,void* logger, unsigned int* new_version,int *update_type)
int get_rm_key_list(long long version,redisContext *c,struct serial_rule_t** list,void* logger, long long* new_version,int *update_type, int cumulative_off)
{
redisReply* reply=NULL,*sub_reply=NULL,*tmp_reply=NULL;
char err_buff[256];
@@ -392,7 +392,6 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
{
return 0;
}
*new_version=version_in_redis;
if(version==0)
{
@@ -401,14 +400,18 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
if(version_in_redis<version)
{
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
"VERSION roll back MAAT: %d -> Redis: %lld.",version,version_in_redis);
"VERSION roll back MAAT: %lld -> Redis: %lld.",version,version_in_redis);
goto FULL_UPDATE;
}
if(version_in_redis>version&&cumulative_off==1)
{
version_in_redis=version+1;
}
//Returns all the elements in the sorted set at key with a score that version < score <= version_in_redis.
//The elements are considered to be ordered from low to high scores(version).
reply=(redisReply*)redisCommand(c, "ZRANGEBYSCORE %s (%d %d",rm_status_sset,version,version_in_redis);
reply=(redisReply*)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld",rm_status_sset,version,version_in_redis);
if(reply==NULL)
{
__redis_strerror_r(errno,err_buff,sizeof(err_buff));
@@ -420,7 +423,7 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
if(reply->elements==0)
{
//a duplicate rule_id would induce this error.
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,"Got nothing after ZRANGEBYSCORE %s (%d %d",rm_status_sset,version,version_in_redis);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,"Got nothing after ZRANGEBYSCORE %s (%lld %lld",rm_status_sset,version,version_in_redis);
freeReplyObject(reply);
return -1;
}
@@ -428,7 +431,7 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
if(tmp_reply->type!=REDIS_REPLY_STRING)
{
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
"ZSCORE %s %s failed Version: %d->%d",rm_status_sset,reply->element[0]->str,version, version_in_redis);
"ZSCORE %s %s failed Version: %lld->%lld",rm_status_sset,reply->element[0]->str,version, version_in_redis);
free(tmp_reply);
free(reply);
goto FULL_UPDATE;
@@ -440,12 +443,12 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
if(nearest_rule_version!=version+1)
{
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
"Noncontinuous VERSION Redis: %lld MAAT: %d.",nearest_rule_version,version);
"Noncontinuous VERSION Redis: %lld MAAT: %lld.",nearest_rule_version,version);
goto FULL_UPDATE;
}
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
"Inc Update form version %d to %lld (%lld entries).",version,version_in_redis,reply->elements);
"Inc Update form version %lld to %lld (%lld entries).",version,version_in_redis,reply->elements);
s_rule=(struct serial_rule_t*)calloc(reply->elements,sizeof(struct serial_rule_t));
for(i=0;i<reply->elements;i++)
@@ -469,6 +472,7 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
*list=s_rule;
*update_type=CM_UPDATE_TYPE_INC;
freeReplyObject(reply);
*new_version=version_in_redis;
return i;
FULL_UPDATE:
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
@@ -1048,8 +1052,8 @@ void cleanup_update_status(redisContext *ctx, void *logger)
,entry_num);
}
void redis_monitor_traverse(unsigned int version,redisContext *c,
void (*start)(unsigned int ,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para
void redis_monitor_traverse(long long version,redisContext *c,
void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para
int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para
void (*finish)(void*),//u_para
void* u_para,
@@ -1060,7 +1064,7 @@ void redis_monitor_traverse(unsigned int version,redisContext *c,
int ret=0;
struct serial_rule_t* rule_list=NULL;
int update_type=CM_UPDATE_TYPE_INC;
unsigned int new_version=0;
long long new_version=0;
enum MAAT_TABLE_TYPE table_type;
void* logger=feather->logger;
if(feather->redis_write_ctx!=NULL)//authorized to write
@@ -1069,7 +1073,7 @@ void redis_monitor_traverse(unsigned int version,redisContext *c,
check_maat_expiration(feather->redis_read_ctx, logger);
cleanup_update_status(feather->redis_read_ctx, logger);
}
rule_num=get_rm_key_list(version, c, &rule_list, logger,&new_version, &update_type);
rule_num=get_rm_key_list(version, c, &rule_list, logger,&new_version, &update_type,feather->cumulative_update_off);
if(rule_num<0||(rule_num==0&&update_type==CM_UPDATE_TYPE_INC))//error or nothing changed
{
return;

18
src/entry/Maat_rule.cpp
View File

@@ -30,7 +30,7 @@
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
int MAAT_FRAME_VERSION_2_1_20171205=1;
int MAAT_FRAME_VERSION_2_1_20171206_dev=1;
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};
@@ -2974,13 +2974,14 @@ void clear_plugin_table_info(struct _plugin_table_info *cb_info)
cb_info->acc_line_num=0;
return;
}
void maat_start_cb(unsigned int new_version,int update_type,void*u_para)
void maat_start_cb(long long new_version,int update_type,void*u_para)
{
struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para;
struct _Maat_table_info_t* p_table=NULL;
struct _plugin_table_info* p_table_cb=NULL;
int i=0,j=0;
feather->new_version=new_version;
if(update_type==CM_UPDATE_TYPE_FULL)
{
feather->update_tmp_scanner=create_maat_scanner(new_version,feather);
@@ -3011,15 +3012,18 @@ void maat_start_cb(unsigned int new_version,int update_type,void*u_para)
feather->maat_version,new_version);
feather->maat_version=new_version;
}
feather->active_plugin_table_num=0;
for(i=0;i<MAX_TABLE_NUM;i++)
{
p_table=feather->p_table_info[i];
if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN)
if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->cb_info->cb_plug_cnt==0)
{
continue;
}
feather->active_plugin_table_num++;
p_table_cb=p_table->cb_info;
feather->active_plugin_table_num++;
for(j=0;j<p_table_cb->cb_plug_cnt;j++)
{
if(p_table_cb->cb_plug[j].start!=NULL)
@@ -3045,16 +3049,18 @@ void maat_finish_cb(void* u_para)
total+=p_table->cfg_num;
}
}
int call_plugin_table_cnt=0;
for(i=0;i<MAX_TABLE_NUM;i++)
{
p_table=feather->p_table_info[i];
if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN)
if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->cb_info->cb_plug_cnt==0)
{
continue;
}
call_plugin_table_cnt++;
p_table_cb=p_table->cb_info;
if(i==feather->active_plugin_table_num)
if(call_plugin_table_cnt==feather->active_plugin_table_num)
{
feather->is_last_plugin_table_updating=1;
}

18
src/entry/Maat_rule_internal.h
View File

@@ -326,7 +326,7 @@ struct GIE_aux_t
};
struct _Maat_scanner_t
{
int version;
long long version;
time_t last_update_time;
long long *ref_cnt; //optimized for cache_alignment 64
rule_scanner_t region;
@@ -361,13 +361,14 @@ struct _Maat_feather_t
int still_working;
int scan_interval_ms;
int effect_interval_ms;
int cumulative_update_off;
int stat_on;
int perf_on;
struct _Maat_table_info_t *p_table_info[MAX_TABLE_NUM];
MESA_htable_handle map_tablename2id;
void* logger;
int maat_version;
int last_full_version;
long long maat_version;
long long last_full_version;
int scan_thread_num;
int rule_scan_type;
char inc_dir[MAX_TABLE_NAME_LEN];
@@ -393,7 +394,7 @@ struct _Maat_feather_t
pthread_mutex_t redis_write_lock; //protect redis_write_ctx
long long base_rgn_seq,base_grp_seq,server_time;
//internal states
int new_version;
long long new_version;
int active_plugin_table_num;
int is_last_plugin_table_updating;
@@ -446,7 +447,7 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag
void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set);
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
void maat_start_cb(unsigned int new_version,int update_type,void*u_para);
void maat_start_cb(long long new_version,int update_type,void*u_para);
int maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para);
void *thread_rule_monitor(void *arg);
@@ -470,16 +471,17 @@ void maat_stat_output(struct _Maat_feather_t* feather);
char* _maat_strdup(const char* s);
char* str_unescape(char* s);
redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...);
int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t** list,void* logger, unsigned int* new_version,int *update_type);
int get_rm_key_list(long long version,redisContext *c,struct serial_rule_t** list,void* logger, long long* new_version,int *update_type, int cumulative_off);
int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger,int print_process);
void empty_serial_rules(struct serial_rule_t* rule);
void redis_monitor_traverse(unsigned int version,redisContext *c,
void (*start)(unsigned int ,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para
void redis_monitor_traverse(long long version,redisContext *c,
void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para
int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para
void (*finish)(void*),//u_para
void* u_para,
const unsigned char* dec_key,
_Maat_feather_t* feather);
const char* module_name_str(const char*name);
#define maat_module (module_name_str("MAAT_Frame"))

21
src/entry/config_monitor.cpp
View File

@@ -210,14 +210,14 @@ int my_scandir(const char *dir, struct dirent ***namelist,
return (strncmp(ent->d_name,"full_config_index",strlen("full_config_index")) == 0||
strncmp(ent->d_name,"inc_config_index",strlen("inc_config_index")) == 0);
}
int get_new_idx_path(unsigned int current_version,const char*file_dir,void* logger,char*** idx_path,int*idx_num)
int get_new_idx_path(long long current_version,const char*file_dir,void* logger,char*** idx_path,int*idx_num)
{
struct dirent **namelist;
int n=0,i=0,sscanf_ret;
char update_str[32]={0};
unsigned int latest_ful_version=0,latest_inc_version=0;
unsigned int config_seq=0;
long long latest_ful_version=0,latest_inc_version=0;
long long config_seq=0;
int *inc_file_idx;
int full_file_idx=0,inc_idx_num=0,path_len=0;
@@ -247,7 +247,7 @@ int get_new_idx_path(unsigned int current_version,const char*file_dir,void* log
,namelist[i]->d_name);
continue;
}
sscanf_ret=sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%u",update_str,&config_seq);
sscanf_ret=sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld",update_str,&config_seq);
if(sscanf_ret!=2)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor
@@ -455,17 +455,18 @@ const char* path2filename(const char*path)
}
return path+i+1;
}
void config_monitor_traverse(unsigned int version,const char*idx_dir,
void (*start)(unsigned int ,int ,void*),
int (*update)(const char* ,const char*,void* ),
void (*finish)(void*),
void config_monitor_traverse(long long version,const char*idx_dir,
void (*start)(long long, int, void*),//vesion,CM_UPDATE_TYPE_*,u_para
int (*update)(const char*, const char*, void*),//table name ,line ,u_para
void (*finish)(void*),//u_para
void* u_para,
const unsigned char* dec_key,
void* logger)
{
int update_type=CM_UPDATE_TYPE_NONE;
unsigned int new_version=0;
long long new_version=0;
char**idx_path_array=NULL;
const char* table_filename=NULL;
char str_not_care[256]={0};
@@ -488,7 +489,7 @@ void config_monitor_traverse(unsigned int version,const char*idx_dir,
break;
}
table_filename=path2filename(idx_path_array[i]);
sscanf(table_filename,"%[a-zA-Z]_config_index.%u",str_not_care,&new_version);
sscanf(table_filename,"%[a-zA-Z]_config_index.%lld",str_not_care,&new_version);
start(new_version,update_type,u_para);
for(j=0;j<table_num;j++)
{

6
src/entry/config_monitor.h
View File

@@ -4,9 +4,9 @@
#define CM_UPDATE_TYPE_FULL 1
#define CM_UPDATE_TYPE_INC 2
void config_monitor_traverse(unsigned int version,const char*idx_dir,
void (*start)(unsigned int ,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para
int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para
void config_monitor_traverse(long long version,const char*idx_dir,
void (*start)(long long, int, void*),//vesion,CM_UPDATE_TYPE_*,u_para
int (*update)(const char*, const char*, void*),//table name ,line ,u_para
void (*finish)(void*),//u_para
void* u_para,
const unsigned char* dec_key,

23
test/maat_test.cpp
View File

@@ -53,6 +53,15 @@ void Maat_read_entry_cb(int table_id,const char* table_line,void* u_para)
}
void Maat_read_entry_finish_cb(void* u_para)
{
Maat_feather_t feather=u_para;
long long version=0;
int ret=0,is_last_updating_table=0;
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version));
assert(ret==0);
ret=Maat_read_state(feather,MAAT_STATE_LAST_UPDATING_TABLE, &is_last_updating_table, sizeof(is_last_updating_table));
assert(ret==0);
//printf("Maat Version %lld at plugin finish callback, is_last_update=%d.\n",version,is_last_updating_table);
return;
}
void print_maat_ret(int ret)
@@ -308,7 +317,7 @@ int test_plugin_table(Maat_feather_t feather,const char* table_name,void* logger
Maat_read_entry_start_cb,
Maat_read_entry_cb,
Maat_read_entry_finish_cb,
logger);
feather);
if(ret<0)
{
printf("Maat callback register table %s error.\n",table_name);
@@ -559,6 +568,7 @@ void test_set_cmd_line(Maat_feather_t feather)
struct Maat_line_t line_rule[TEST_CMD_LINE_NUM];
char table_line[TEST_CMD_LINE_NUM][128];
int ret=0,i=0;
long long version=0;
memset(&line_rule,0,sizeof(line_rule));
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
@@ -572,9 +582,20 @@ void test_set_cmd_line(Maat_feather_t feather)
}
ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
assert(ret==0);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version));
assert(ret==0);
printf("Maat Version %lld after add lines.\n",version);
sleep(1);
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
line_rule[i].table_line=NULL;
}
ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
assert(ret==0);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version));
assert(ret==0);
printf("Maat Version %lld after delete lines.\n",version);
return;
}
int test_add_expr_command(Maat_feather_t feather,const char* region_table,int config_id, int timeout,int label_id, const char* keywords)

10
tools/maat_redis_tool.cpp
View File

@@ -37,7 +37,7 @@ void read_rule_from_redis(const char*redis_ip, int redis_port, int redis_db,cons
int rule_num=0,line_count=0;
int i=0,ret=0;
int update_type=CM_UPDATE_TYPE_INC;
unsigned int version=0;
long long version=0;
const char* cur_table=NULL;
char table_path[256],index_path[256];
@@ -56,7 +56,7 @@ void read_rule_from_redis(const char*redis_ip, int redis_port, int redis_db,cons
}
printf("Reading key list from %s:%d db%d.\n",redis_ip,redis_port,redis_db);
rule_num=get_rm_key_list(0, ctx, &rule_list, logger,&version, &update_type);
rule_num=get_rm_key_list(0, ctx, &rule_list, logger,&version, &update_type,0);
if(rule_num==0)
{
printf("No Effective Rules.\n");
@@ -69,7 +69,7 @@ void read_rule_from_redis(const char*redis_ip, int redis_port, int redis_db,cons
}
assert(update_type==CM_UPDATE_TYPE_FULL);
printf("MAAT Version: %d, key number: %d\n", version, rule_num);
printf("MAAT Version: %lld, key number: %d\n", version, rule_num);
printf("Reading value: ");
ret=get_maat_redis_value(ctx,rule_list,rule_num,logger,1);
if(ret<0)
@@ -85,7 +85,7 @@ void read_rule_from_redis(const char*redis_ip, int redis_port, int redis_db,cons
printf("mkdir %s error\n",output_path);
}
snprintf(index_path,sizeof(index_path),"%s/full_config_index.%010d",output_path,version);
snprintf(index_path,sizeof(index_path),"%s/full_config_index.%020lld",output_path,version);
index_fp=fopen(index_path,"w");
if(index_fp==NULL)
{
@@ -105,7 +105,7 @@ void read_rule_from_redis(const char*redis_ip, int redis_port, int redis_db,cons
set_file_rulenum(table_path,line_count, logger);
line_count=0;
}
snprintf(table_path,sizeof(table_path),"%s/%s.%010d",output_path,rule_list[i].table_name,version);
snprintf(table_path,sizeof(table_path),"%s/%s.%020lld",output_path,rule_list[i].table_name,version);
set_file_rulenum(table_path, 0, logger);
table_fp=fopen(table_path,"a");
if(table_fp==NULL)