gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'bugfix-calculate-cmd-cnt-on-renew' into 'master'

Browse Source 
Bugfix calculate cmd cnt on renew

See merge request MESA_framework/maat!19
This commit is contained in:
郑超
2018-12-07 15:57:19 +08:00
parent 323f15ebb1 1607e48d77
commit 1a44e3aa53
19 changed files with 1506 additions and 867 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
inc/Maat_rule.h
View File

@@ -6,7 +6,7 @@
* to reside in the heart) of the departed would reach the paradise of afterlife
* successfully.
* Author: zhengchao@iie.ac.cn, MESA
* Version 2018-11-06 Maat Rule Extra Data.
* Version 2018-12-07 Plugin Extra Data.
* NOTE: MUST compile with G++
* All right reserved by Institute of Infomation Engineering,Chinese Academic of Science 2014~2018
*********************************************************
@@ -165,8 +165,9 @@ enum MAAT_INIT_OPT
int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size);
enum MAAT_STATE_OPT
{
MAAT_STATE_VERSION=1, //Get current maat version. VALUE is long long, SIZE=sizeof(long long).
MAAT_STATE_LAST_UPDATING_TABLE //Query at Maat_finish_callback_t to determine whether this table is the last one to update. VALUE is interger, SIZE=sizeof(int), 1:yes, 0: no
MAAT_STATE_VERSION=1, //Get current maat version, if maat is in update progress, the updating version is returned. VALUE is long long, SIZE=sizeof(long long).
MAAT_STATE_LAST_UPDATING_TABLE, //Query at Maat_finish_callback_t to determine whether this table is the last one to update. VALUE is interger, SIZE=sizeof(int), 1:yes, 0: no
MAAT_STATE_IN_UPDATING
};
int Maat_read_state(Maat_feather_t feather, enum MAAT_STATE_OPT type, void* value, int size);
@@ -257,6 +258,21 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table
//returned data is duplicated by dup_func of Maat_rule_get_ex_new_index, caller is responsible to free the data.
MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx);
//Following functions are similar to Maat_rule_get_ex_data, except they are effective on plugin table.
typedef void* MAAT_PLUGIN_EX_DATA;
typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);
typedef void Maat_plugin_EX_free_func_t(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);
typedef void Maat_plugin_EX_dup_func_t(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp);
typedef int Maat_plugin_EX_key2index_func_t(const char* key);
int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp);
//Data is duplicated by dup_func of Maat_plugin_EX_register, caller is responsible to free the data.
MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key);
enum MAAT_RULE_OPT
{

22
inc/gram_index_engine.h
View File

@@ -4,12 +4,16 @@
#ifdef __cplusplus
extern "C" {
#endif
#define GIE_INSERT_OPT 0
#define GIE_DELETE_OPT 1
#define GIE_INPUT_FORMAT_SFH 1
#define GIE_INPUT_FORMAT_PLAIN 0
enum GIE_operation
{
GIE_INSERT_OPT,
GIE_DELETE_OPT
};
enum GIE_INPUT_FORMAT
{
GIE_INPUT_FORMAT_PLAIN,
GIE_INPUT_FORMAT_SFH
};
typedef struct
{
@@ -21,7 +25,7 @@ typedef struct
{
unsigned int id;
unsigned int sfh_length;//size of fuzzy_hash
short operation;//GIE_INSERT_OPT or GIE_DELETE_OPT.if operation is GIE_DELETE_OPT, only id is needed;
enum GIE_operation operation;//GIE_INSERT_OPT or GIE_DELETE_OPT.if operation is GIE_DELETE_OPT, only id is needed;
short cfds_lvl;
char * sfh;
void * tag;
@@ -41,9 +45,9 @@ typedef struct
unsigned int gram_value;
//unsigned int htable_num;
unsigned int position_accuracy;
short format; //if format==GIE_INPUT_FORMAT_SFH, means the input string is a GIE_INPUT_FORMAT_SFH string
enum GIE_INPUT_FORMAT format; //if format==GIE_INPUT_FORMAT_SFH, means the input string is a GIE_INPUT_FORMAT_SFH string
//else id format==PALIN, means the input string is common string
short ED_reexamine;//if ED_reexamine==1, calculate edit distance to verify the final result
int ED_reexamine;//if ED_reexamine==1, calculate edit distance to verify the final result
}GIE_create_para_t;

4
src/CMakeLists.txt
View File

@@ -1,8 +1,8 @@
cmake_minimum_required(VERSION 3.5)
set(MAAT_FRAME_MAJOR_VERSION 2)
set(MAAT_FRAME_MINOR_VERSION 4)
set(MAAT_FRAME_PATCH_VERSION 20181115)
set(MAAT_FRAME_MINOR_VERSION 5)
set(MAAT_FRAME_PATCH_VERSION 0)
set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION})
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")

448
src/entry/Maat_api.cpp
View File

@@ -16,9 +16,9 @@
#include "rulescan.h"
#include "json2iris.h"
struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type)
struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type)
{
struct _Maat_table_info_t *p_table=NULL;
struct Maat_table_desc *p_table=NULL;
if(table_id>MAX_TABLE_NUM)
{
return NULL;
@@ -113,7 +113,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
int ret=0,i=0;
int r_in_c_cnt=0;
int shortcut_avilable_cnt=0;
void* bool_matcher=feather->scanner->expr_compiler;
void* bool_matcher=feather->scanner->bool_macher_expr_compiler;
struct _Maat_group_inner_t* group_rule=NULL;
struct _Maat_compile_inner_t* array_mi_rule[MAX_SCANNER_HIT_NUM];
struct _Maat_compile_inner_t* _mi_rule=NULL;
@@ -443,7 +443,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
return NULL;
}
_Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1);
feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger);
feather->table_cnt=read_table_description(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger);
feather->map_tablename2id=map_create();
int i=0,j=0,ret=0;
for(i=0;i<MAX_TABLE_NUM;i++)
@@ -918,7 +918,9 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
int idx=0,i=0;
struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id];
struct Maat_table_desc *p_table=_feather->p_table_info[table_id];
struct plugin_table_desc *plugin_desc=&(p_table->plugin);
struct plugin_runtime* plugin_aux=NULL;
const char* lines=NULL;
if(p_table==NULL)
{
@@ -930,26 +932,32 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
}
//plugin table register blocks background update.
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
idx=p_table->cb_info->cb_plug_cnt;
idx=plugin_desc->cb_plug_cnt;
if(idx==MAX_PLUGIN_PER_TABLE)
{
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return -1;
}
p_table->cb_info->cb_plug_cnt++;
p_table->cb_info->cb_plug[idx].start=start;
p_table->cb_info->cb_plug[idx].update=update;
p_table->cb_info->cb_plug[idx].finish=finish;
p_table->cb_info->cb_plug[idx].u_para=u_para;
if(p_table->cb_info->cache_line_num>0)
plugin_desc->cb_plug_cnt++;
plugin_desc->cb_plug[idx].start=start;
plugin_desc->cb_plug[idx].update=update;
plugin_desc->cb_plug[idx].finish=finish;
plugin_desc->cb_plug[idx].u_para=u_para;
if(_feather->scanner==NULL)
{
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return 1;
}
plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin);
if(plugin_aux->cache_line_num>0)
{
if(start!=NULL)
{
start(MAAT_RULE_UPDATE_TYPE_FULL,u_para);
}
for(i=0;i<p_table->cb_info->cache_line_num;i++)
for(i=0;i<plugin_aux->cache_line_num;i++)
{
lines=(const char*)dynamic_array_read(p_table->cb_info->cache_lines,i);
lines=(const char*)dynamic_array_read(plugin_aux->cache_lines,i);
if(lines==NULL)
{
break;
@@ -996,31 +1004,31 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table
{
return -1;
}
struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id];
if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL)
struct Maat_table_desc *p_table=_feather->p_table_info[table_id];
if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL || dup_func==NULL)
{
return -1;
}
struct compile_table_desc* compile_desc=&(p_table->compile);
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
if(p_table->ex_data_num==MAX_COMPILE_EX_DATA_NUM)
if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM)
{
ret=-1;
goto failed;
}
idx=p_table->ex_data_num;
p_table->ex_desc[idx].idx=idx;
p_table->ex_desc[idx].table_id=table_id;
p_table->ex_desc[idx].argl=argl;
p_table->ex_desc[idx].argp=argp;
p_table->ex_desc[idx].new_func=new_func;
p_table->ex_desc[idx].free_func=free_func;
p_table->ex_desc[idx].dup_func=dup_func;
idx=compile_desc->ex_data_num;
compile_desc->ex_desc[idx].idx=idx;
compile_desc->ex_desc[idx].table_id=table_id;
compile_desc->ex_desc[idx].argl=argl;
compile_desc->ex_desc[idx].argp=argp;
compile_desc->ex_desc[idx].new_func=new_func;
compile_desc->ex_desc[idx].free_func=free_func;
compile_desc->ex_desc[idx].dup_func=dup_func;
p_table->ex_data_num++;
compile_desc->ex_data_num++;
if(_feather->scanner!=NULL)
{
MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, p_table->ex_desc+idx);
MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_desc->ex_desc+idx);
}
failed:
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
@@ -1031,7 +1039,8 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct _Maat_compile_inner_t *compile_inner=NULL;
const struct compile_ex_data_idx* ex_desc=NULL;
const struct compile_table_desc* compile_desc=NULL;
const struct compile_ex_data_idx* ex_desc=NULL;
MAAT_RULE_EX_DATA ad=NULL;
compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
@@ -1040,12 +1049,213 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa
return NULL;
}
pthread_rwlock_rdlock(&(compile_inner->rwlock));
assert(idx<compile_inner->ref_table->ex_data_num);
ex_desc=compile_inner->ref_table->ex_desc+idx;
compile_desc=&(compile_inner->ref_table->compile);
assert(idx<compile_desc->ex_data_num);
ex_desc=compile_desc->ex_desc+idx;
ex_desc->dup_func(ex_desc->idx, &ad, compile_inner->ads+idx, ex_desc->argl,ex_desc->argp);
pthread_rwlock_unlock(&(compile_inner->rwlock));
return ad;
}
struct wrap_plugin_EX_data
{
MAAT_RULE_EX_DATA exdata;
const struct Maat_table_desc* ref_plugin_table;
};
void wrap_plugin_EX_data_free(void *data)
{
struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data;
const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc);
ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp);
wrap_data->ref_plugin_table=NULL;
free(wrap_data);
return;
}
MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index)
{
MESA_htable_handle key2ex_hash=NULL;
unsigned int slot_size=1;
while(estimate_size!=0)
{
estimate_size=estimate_size>>1;
slot_size*=2;
}
if(slot_size==1)
{
slot_size=4096;
}
MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs));
hargs.thread_safe=0;
hargs.hash_slot_size = slot_size;
hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
hargs.expire_time = 0;
hargs.key_comp = NULL;
hargs.key2index = NULL; //Not supported yet.
hargs.recursive = 1;
hargs.data_free = wrap_plugin_EX_data_free;
hargs.data_expire_with_condition = NULL;
key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(key2ex_hash, 0);
return key2ex_hash;
}
int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line,
MESA_htable_handle key2ex_hash, void *logger)
{
size_t key_offset=0, key_len=0;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
int ret=0;
ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data del error: cannot find column %d of %s",
plugin_desc->key_column, line);
return -1;
}
ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data del error: no such key %.*s of %s",
key_len, line+key_offset, line);
return -1;
}
return 0;
}
int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line,
MESA_htable_handle key2ex_hash, void *logger)
{
char* key=NULL;
size_t key_offset=0, key_len=0;
MAAT_RULE_EX_DATA exdata=NULL;
struct wrap_plugin_EX_data* wrap_data=NULL;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
int ret=0;
ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data add error: cannot find column %d of %s",
plugin_desc->key_column, line);
return -1;
}
key=ALLOC(char, key_len+1);
memcpy(key, line+key_offset, key_len);
plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata,
plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
wrap_data=ALLOC(struct wrap_plugin_EX_data, 1);
wrap_data->exdata=exdata;
wrap_data->ref_plugin_table=plugin_table;
ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data);
free(key);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data add error: duplicated key %.*s of %s",
key_len, line+key_offset, line);
wrap_plugin_EX_data_free(wrap_data);
return -1;
}
return 0;
}
MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table,
struct dynamic_array_t* lines, size_t line_cnt, void* logger)
{
MESA_htable_handle key2ex_hash=NULL;
size_t i=0;
const char* line=NULL;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func);
for(i=0; i< line_cnt; i++)
{
line=(const char*)dynamic_array_read(lines, i);
plugin_EX_data_new(plugin_table, line, key2ex_hash, logger);
}
return key2ex_hash;
}
int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_table_desc *table_desc=_feather->p_table_info[table_id];
struct plugin_table_desc* plugin_desc=&(table_desc->plugin);
struct Maat_table_runtime* table_rt=NULL;
if(new_func==NULL || free_func==NULL || dup_func==NULL )
{
assert(0);
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
return -1;
}
if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata
|| plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0)
{
assert(0);
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module,
"%s failed: key or valid flag column are not specified", __FUNCTION__);
return -1;
}
plugin_desc->ex_desc.new_func=new_func;
plugin_desc->ex_desc.free_func=free_func;
plugin_desc->ex_desc.dup_func=dup_func;
plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used.
plugin_desc->ex_desc.argl=argl;
plugin_desc->ex_desc.argp=argp;
plugin_desc->have_exdata=1;
if(_feather->scanner==NULL)
{
return 0;
}
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
table_rt=_feather->scanner->table_rt[table_id];
assert(table_rt->plugin.key2ex_hash==NULL);
table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines,
table_rt->plugin.cache_line_num, _feather->logger);
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return 0;
}
MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_table_desc *table_desc=_feather->p_table_info[table_id];
struct Maat_table_runtime *table_rt= NULL;
struct plugin_table_desc* plugin_desc=&(table_desc->plugin);
struct wrap_plugin_EX_data* wrap_data=NULL;
MAAT_RULE_EX_DATA exdata=NULL;
if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata==0)
{
assert(0);
return NULL;
}
if(_feather->scanner==NULL)
{
return NULL;
}
table_rt= _feather->scanner->table_rt[table_id];
pthread_rwlock_rdlock(&table_rt->plugin.rwlock);
wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash,
(const unsigned char*)key, strlen(key));
if(wrap_data!=NULL)
{
plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata),
plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
}
pthread_rwlock_unlock(&table_rt->plugin.rwlock);
return exdata;
}
int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
,enum MAAT_CHARSET charset,const char* data,int data_len
,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num
@@ -1058,7 +1268,8 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];//dynamic array
struct _Maat_table_info_t *p_table=NULL;
struct Maat_table_desc *p_table=NULL;
struct expr_table_desc* expr_desc=NULL;
struct timespec start,end;
_Maat_scanner_t* my_scanner=NULL;
if(data==NULL||data_len<=0)
@@ -1076,17 +1287,13 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++;
return -1;
}
if(p_table->cfg_num==0)
{
return 0;
}
expr_desc=&(p_table->expr);
if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
{
_feather->scan_err_cnt++;
return -1;
}
if(p_table->do_charset_merge==1)
if(expr_desc->do_charset_merge==1)
{
sub_type=make_sub_type(table_id,CHARSET_NONE,0);
}
@@ -1104,9 +1311,8 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{
return 0;
}
struct _region_stat_t * region_stat=NULL;
region_stat=&(my_scanner->region_counter[p_table->table_id]);
if(region_stat->cfg_num==0)
struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id];
if(table_rt->origin_rule_num==0)
{
return 0;
}
@@ -1114,7 +1320,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
INC_SCANNER_REF(my_scanner, thread_num);
if(region_stat->expr_rule_cnt>0)
if(table_rt->expr.expr_rule_cnt>0)
{
scan_data.rule_type=RULETYPE_STR;
scan_data.sub_type=sub_type;
@@ -1124,7 +1330,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
hit_region_cnt+=region_ret;
}
}
if(region_stat->regex_rule_cnt>0)
if(table_rt->expr.regex_rule_cnt>0)
{
scan_data.rule_type=RULETYPE_REG;
scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0);
@@ -1141,7 +1347,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
}
if(hit_region_cnt>0)
{
alignment_int64_array_add(p_table->hit_cnt, thread_num,1);
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
_mid=grab_mid(mid,_feather,thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
@@ -1162,17 +1368,16 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
hit_detail,detail_num);
}
}
DEC_SCANNER_REF(my_scanner, thread_num);
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,data_len,&start, &end,thread_num);
maat_stat_table(table_rt, data_len, &start, &end, thread_num);
}
else
{
maat_stat_table(p_table,data_len,NULL, NULL,thread_num);
maat_stat_table(table_rt, data_len, NULL, NULL, thread_num);
}
DEC_SCANNER_REF(my_scanner, thread_num);
if(compile_ret==0&&hit_region_cnt>0)
{
return -2;
@@ -1206,7 +1411,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
intval_scan_data.rule_type=RULETYPE_INT;
intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0);
intval_scan_data.int_data=intval;
_Maat_table_info_t* p_table=NULL;
Maat_table_desc* p_table=NULL;
struct timespec start,end;
if(_feather->perf_on==1)
{
@@ -1218,18 +1423,13 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++;
return -1;
}
if(p_table->cfg_num==0)
{
return 0;
}
my_scanner=_feather->scanner;
if(my_scanner==NULL)
{
return 0;
}
struct _region_stat_t * region_stat=NULL;
region_stat=&(my_scanner->region_counter[p_table->table_id]);
if(region_stat->cfg_num==0)
struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id];
if(table_rt->origin_rule_num==0)
{
return 0;
}
@@ -1247,7 +1447,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
}
else if(region_ret>0)
{
alignment_int64_array_add(p_table->hit_cnt, thread_num,1);
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
@@ -1262,16 +1462,17 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
}
}
DEC_SCANNER_REF(my_scanner,thread_num);
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,0,&start, &end,thread_num);
maat_stat_table(table_rt, 0, &start, &end, thread_num);
}
else
{
maat_stat_table(p_table,0,NULL, NULL,thread_num);
maat_stat_table(table_rt, 0, NULL, NULL, thread_num);
}
DEC_SCANNER_REF(my_scanner,thread_num);
if(compile_ret==0&&region_ret>0)
{
return -2;
@@ -1290,7 +1491,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
scan_data_t ip_scan_data;
scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];
_Maat_table_info_t* p_table=NULL;
Maat_table_desc* p_table=NULL;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* my_scanner=NULL;
@@ -1305,26 +1506,22 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++;
return -1;
}
if(p_table->cfg_num==0)
{
return 0;
}
my_scanner=_feather->scanner;
if(my_scanner==NULL)
{
return 0;
}
struct _region_stat_t * region_stat=NULL;
region_stat=&(my_scanner->region_counter[p_table->table_id]);
if(region_stat->cfg_num==0)
struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id];
if(table_rt->origin_rule_num==0)
{
return 0;
}
if(region_stat->ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4)
if(table_rt->ip.ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4)
{
return 0;
}
if(region_stat->ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6)
if(table_rt->ip.ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6)
{
return 0;
}
@@ -1369,7 +1566,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
}
else if(region_ret>0)
{
alignment_int64_array_add(p_table->hit_cnt, thread_num,1);
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
@@ -1388,11 +1585,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,0,&start, &end,thread_num);
maat_stat_table(table_rt, 0, &start, &end, thread_num);
}
else
{
maat_stat_table(p_table,0,NULL, NULL,thread_num);
maat_stat_table(table_rt, 0, NULL, NULL, thread_num);
}
if(compile_ret==0&&region_ret>0)
{
@@ -1417,7 +1614,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* scanner=NULL;
struct _Maat_table_info_t *p_table=NULL;
struct Maat_table_desc *p_table=NULL;
assert(thread_num<_feather->scan_thread_num);
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR);
if(p_table==NULL)
@@ -1425,12 +1622,14 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
_feather->scan_err_cnt++;
return NULL;
}
if(p_table->quick_expr_switch==1)
struct expr_table_desc* expr_desc=&(p_table->expr);
if(expr_desc->quick_expr_switch==1)
{
_feather->scan_err_cnt++;
return NULL;
}
struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1);
struct _stream_para_t* sp=ALLOC(struct _stream_para_t ,1);
scanner=_feather->scanner;
sp->feather=_feather;
sp->version=_feather->maat_version;
@@ -1440,29 +1639,32 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
{
return sp;
}
struct _region_stat_t * region_stat=NULL;
region_stat=&(scanner->region_counter[p_table->table_id]);
struct Maat_table_runtime* table_rt=scanner->table_rt[table_id];
if(table_rt->origin_rule_num==0)
{
return 0;
}
INC_SCANNER_REF(scanner, thread_num);
sp->table_id=table_id;
sp->thread_num=thread_num;
sp->max_cross_size=p_table->cross_cache_size;
sp->max_cross_size=expr_desc->cross_cache_size;
sp->caching_size=0;
sp->scan_buff=NULL;
sp->last_cache=NULL;
if(p_table->do_charset_merge==1)
if(expr_desc->do_charset_merge==1)
{
sp->do_merge=1;
}
if(region_stat->expr_rule_cnt>0)
if(table_rt->expr.expr_rule_cnt>0)
{
sp->do_expr=1;
}
if(region_stat->regex_rule_cnt>0)
if(table_rt->expr.regex_rule_cnt>0)
{
sp->do_regex=1;
}
alignment_int64_array_add(p_table->stream_num,thread_num,1);
alignment_int64_array_add(table_rt->stream_num,thread_num,1);
sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num);
return sp;
}
@@ -1481,7 +1683,8 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
scan_result_t *region_result;
_compile_result_t compile_result[rule_num];//dynamic array
scan_data_t region_scan_data;
_Maat_table_info_t* p_table=NULL;
Maat_table_desc* p_table=NULL;
struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id];
struct timespec start,end;
if(data==NULL||data_len<=0)
{
@@ -1528,7 +1731,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
free(sp->scan_buff);
sp->scan_buff=NULL;
}
sp->scan_buff=(char*)malloc(sp->caching_size+data_len);
sp->scan_buff=ALLOC(char, sp->caching_size+data_len);
memcpy(sp->scan_buff,sp->last_cache,sp->caching_size);
memcpy(sp->scan_buff+sp->caching_size,data,data_len);
region_scan_data.text_data.text=sp->scan_buff;
@@ -1543,7 +1746,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
if(sp->last_cache==NULL&&sp->max_cross_size>0)
{
assert(sp->caching_size==0);
sp->last_cache=(char*)malloc(sizeof(char)*sp->max_cross_size);
sp->last_cache=ALLOC(char, sp->max_cross_size);
}
if(sp->max_cross_size>0)
{
@@ -1589,7 +1792,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
}
if(hit_region_cnt>0)
{
alignment_int64_array_add(p_table->hit_cnt, sp->thread_num,1);
alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1);
_mid=grab_mid(mid, sp->feather,sp->thread_num, 1);
compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region,
@@ -1628,11 +1831,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
if(sp->feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end,sp->thread_num);
maat_stat_table(table_rt,data_len,&start, &end,sp->thread_num);
}
else
{
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL,sp->thread_num);
maat_stat_table(table_rt,data_len,NULL, NULL,sp->thread_num);
}
if(compile_ret==0&&hit_region_cnt>0)
{
@@ -1657,8 +1860,8 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para)
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id];
alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1);
struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id];
alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1);
if(sp->rs_stream_para!=NULL)
{
if(scanner!=NULL&&sp->version>=sp->feather->last_full_version)
@@ -1695,14 +1898,14 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* scanner=NULL;
sfh_instance_t * tmp_fuzzy_handle=NULL;
struct _Maat_table_info_t *p_table=NULL;
struct Maat_table_desc *p_table=NULL;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
return NULL;
}
struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1);
}
struct _stream_para_t* sp=ALLOC(struct _stream_para_t, 1);
scanner=_feather->scanner;
sp->feather=_feather;
sp->version=_feather->maat_version;
@@ -1710,7 +1913,8 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
if(scanner==NULL)
{
return sp;
}
}
struct Maat_table_runtime* table_rt=scanner->table_rt[table_id];
tmp_fuzzy_handle=SFH_instance(total_len);
if(tmp_fuzzy_handle==NULL)
{
@@ -1724,7 +1928,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
sp->total_len=total_len;
sp->fuzzy_hash_handle=tmp_fuzzy_handle;
pthread_mutex_init(&(sp->fuzzy_mutex),NULL);
alignment_int64_array_add(p_table->stream_num,thread_num,1);
alignment_int64_array_add(table_rt->stream_num,thread_num,1);
return sp;
}
@@ -1761,7 +1965,6 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
GIE_result_t query_result[MAX_SCANNER_HIT_NUM];
int hit_region_cnt=0,compile_ret=0;
_compile_result_t compile_result[rule_num];//dynamic array
_Maat_table_info_t* p_table=NULL;
if(data==NULL||data_len<=0)
{
return 0;
@@ -1770,7 +1973,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{
return 0;
}
GIE_handle_t* GIE_handle=sp->feather->scanner->gie_aux[sp->table_id].gie_handle;
struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id];
GIE_handle_t* GIE_handle=table_rt->similar.gie_handle;
unsigned long long digest_len=0;
char* digest_buff=NULL;
struct _OUTER_scan_status_t* _mid=NULL;
@@ -1779,7 +1983,6 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
p_table=sp->feather->p_table_info[sp->table_id];
alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1);
pthread_mutex_lock(&(sp->fuzzy_mutex));
sp->process_offset+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset);
@@ -1796,7 +1999,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{
goto fast_out;
}
digest_buff=(char*)malloc(sizeof(char)*digest_len);
digest_buff=ALLOC(char, digest_len);
pthread_mutex_lock(&(sp->fuzzy_mutex));
SFH_digest(sp->fuzzy_hash_handle,digest_buff, digest_len);
pthread_mutex_unlock(&(sp->fuzzy_mutex));
@@ -1816,7 +2019,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
}
if(hit_region_cnt>0)
{
alignment_int64_array_add(p_table->hit_cnt, sp->thread_num, 1);
alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1);
_mid=grab_mid(mid,sp->feather, sp->thread_num,1);
compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region,
@@ -1834,11 +2037,11 @@ fast_out:
if(sp->feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end,sp->thread_num);
maat_stat_table(table_rt, data_len, &start, &end, sp->thread_num);
}
else
{
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL,sp->thread_num);
maat_stat_table(table_rt, data_len, NULL, NULL, sp->thread_num);
}
if(compile_ret==0&&hit_region_cnt>0)
{
@@ -1850,8 +2053,8 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para)
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id];
alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1);
struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id];
alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1);
if(scanner!=NULL)
{
if(sp->version==sp->feather->maat_version)
@@ -1946,7 +2149,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
_compile_result_t compile_result[rule_num];
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* my_scanner=NULL;
_Maat_table_info_t* p_table=NULL;
Maat_table_desc* p_table=NULL;
struct timespec start,end;
if(_feather->perf_on==1)
{
@@ -1958,25 +2161,20 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++;
return -1;
}
if(p_table->cfg_num==0)
{
return 0;
}
my_scanner=_feather->scanner;
if(my_scanner==NULL)
{
return 0;
}
GIE_handle_t* gie_handle=my_scanner->gie_aux[table_id].gie_handle;
struct _region_stat_t * region_stat=NULL;
region_stat=&(my_scanner->region_counter[p_table->table_id]);
if(region_stat->cfg_num==0)
struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id];
if(table_rt->origin_rule_num==0)
{
return 0;
}
GIE_handle_t* gie_handle=table_rt->similar.gie_handle;
INC_SCANNER_REF(my_scanner,thread_num);
alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
INC_SCANNER_REF(my_scanner,thread_num);
region_ret=GIE_query(gie_handle, data, data_len,region_result, MAX_SCANNER_HIT_NUM);
if(region_ret<0)
{
@@ -1986,7 +2184,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
}
else if(region_ret>0)
{
alignment_int64_array_add(p_table->hit_cnt, thread_num,1);
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
@@ -2005,11 +2203,11 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,0,&start, &end,thread_num);
maat_stat_table(table_rt,0,&start, &end,thread_num);
}
else
{
maat_stat_table(p_table,0,NULL, NULL,thread_num);
maat_stat_table(table_rt,0,NULL, NULL,thread_num);
}
if(compile_ret==0&&region_ret>0)
{
@@ -2062,6 +2260,20 @@ int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* valu
case MAAT_STATE_LAST_UPDATING_TABLE:
*int_val=_feather->is_last_plugin_table_updating;
break;
case MAAT_STATE_IN_UPDATING:
if(size!=sizeof(int))
{
return -1;
}
if(0==pthread_mutex_trylock(&(_feather->backgroud_update_mutex)))
{
*int_val=0;
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
}
else
{
*int_val=1;
}
default:
return -1;
break;

122
src/entry/Maat_command.cpp
View File

@@ -182,61 +182,51 @@ enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p)
}
int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq)
{
unsigned int offset=0;
unsigned int i=0,j=0;
switch(type)
size_t offset=0, len=0;
unsigned int column_seq=0, ret=0;
switch(type)
{
case TABLE_TYPE_EXPR:
offset=7;
column_seq=7;
break;
case TABLE_TYPE_IP:
offset=14;
column_seq=14;
break;
case TABLE_TYPE_COMPILE:
offset=8;
column_seq=8;
break;
case TABLE_TYPE_PLUGIN:
if(valid_column_seq<0)
{
return -1;
}
offset=(unsigned int)valid_column_seq;
column_seq=(unsigned int)valid_column_seq;
break;
case TABLE_TYPE_INTERVAL:
offset=5;
column_seq=5;
break;
case TABLE_TYPE_DIGEST:
offset=6;
column_seq=6;
break;
case TABLE_TYPE_SIMILARITY:
offset=5;
column_seq=5;
break;
case TABLE_TYPE_EXPR_PLUS:
offset=8;
column_seq=8;
break;
case TABLE_TYPE_GROUP:
offset=3;
column_seq=3;
break;
default:
assert(0);
}
for(i=0;i<strlen(line);i++)
{
if(line[i]==' '||line[i]=='\t')
{
j++;
}
if(j==offset-1)
{
break;
}
}
i++;
if(i>=strlen(line)||(line[i]!='1'&&line[i]!='0'))// 0 is also a valid value for some non-MAAT producer.
ret=get_column_pos(line, column_seq, &offset, &len);
if(ret<0||offset>=strlen(line)||(line[offset]!='1'&&line[offset]!='0'))// 0 is also a valid value for some non-MAAT producer.
{
return -1;
}
return i;
return offset;
}
int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq)
{
@@ -1074,28 +1064,34 @@ long long _exec_serial_rule_begin(redisContext* ctx,int rule_num, int renew_rule
}
return ret;
}
redisReply* _exec_serial_rule_end(redisContext* ctx,long long maat_redis_version, long long server_time, int renew_allowed)
redisReply* _exec_serial_rule_end(redisContext* ctx,long long maat_redis_version, long long server_time, int renew_allowed, struct expected_reply_t* expect_reply, unsigned int *cnt)
{
redisReply* data_reply=NULL;
if(renew_allowed==1)
{
redlock_unlock(ctx, rm_expire_lock);
expect_reply[*cnt].srule_seq=-1;
(*cnt)++;
}
if(maat_redis_version>0)
{
data_reply=_wrap_redisCommand(ctx,"ZADD %s NX %d %d",rm_version_sset,server_time,maat_redis_version);
freeReplyObject(data_reply);
expect_reply[*cnt].srule_seq=-1;
(*cnt)++;
data_reply=_wrap_redisCommand(ctx,"INCRBY MAAT_VERSION 1");
freeReplyObject(data_reply);
expect_reply[*cnt].srule_seq=-1;
expect_reply[*cnt].reply.type=REDIS_REPLY_INTEGER;
(*cnt)++;
}
data_reply=_wrap_redisCommand(ctx,"EXEC");
return data_reply;
}
void _exec_serial_rule(redisContext* ctx, long long version, struct serial_rule_t* s_rule, int rule_num, struct expected_reply_t* expect_reply, unsigned int *cnt, int offset,int renew_allowed)
void _exec_serial_rule(redisContext* ctx, long long version, struct serial_rule_t* s_rule, unsigned int rule_num, struct expected_reply_t* expect_reply, unsigned int *cnt, int offset,int renew_allowed)
{
int i=0;
redisReply* data_reply=NULL;
int append_cmd_cnt=0;
unsigned int append_cmd_cnt=0, i=0;
for(i=0;i<rule_num;i++)
{
switch(s_rule[i].op)
@@ -1223,21 +1219,21 @@ void _exec_serial_rule(redisContext* ctx, long long version, struct serial_rule_
return;
}
int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_rule_num, long long server_time, void* logger)
int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule, unsigned int serial_rule_num, long long server_time, void* logger)
{
int max_redis_batch=1*1024,batch_cnt=0;
int success_cnt=0,j=0,renew_allowed=0,last_failed=-1;
unsigned int max_redis_batch=1*1024, batch_cnt=0;
int renew_allowed=0,last_failed=-1;
redisReply*transaction_reply=NULL,*p=NULL;
unsigned int i=0;
unsigned int multi_cmd_cnt=0;
unsigned int multi_cmd_cnt=0, success_cnt=0;
const int MAX_REDIS_OP_PER_SRULE=8;
unsigned int max_multi_cmd_num=MAX_REDIS_OP_PER_SRULE*serial_rule_num+2;// 2 for operation in _exec_serial_rule_end()
struct expected_reply_t *expected_reply=(struct expected_reply_t*)calloc(sizeof(struct expected_reply_t), max_multi_cmd_num);
long long new_version=0;
int renew_num=0,ret=0;
for(i=0;i<(unsigned int)serial_rule_num;i++)
for(i=0;i<serial_rule_num;i++)
{
if(s_rule[i].op==MAAT_OP_RENEW_TIMEOUT)
{
@@ -1253,15 +1249,15 @@ int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_r
}
while(success_cnt<serial_rule_num)
{
batch_cnt=MIN(serial_rule_num-success_cnt,max_redis_batch);
batch_cnt=MIN(serial_rule_num-success_cnt, max_redis_batch);
_exec_serial_rule(ctx,new_version,s_rule+success_cnt,batch_cnt,expected_reply, &multi_cmd_cnt,success_cnt,renew_allowed);
assert(multi_cmd_cnt<max_multi_cmd_num);
success_cnt+=batch_cnt;
}
transaction_reply=_exec_serial_rule_end(ctx,new_version,server_time, renew_allowed);
transaction_reply=_exec_serial_rule_end(ctx,new_version,server_time, renew_allowed, expected_reply, &multi_cmd_cnt);
if(1==mr_transaction_success(transaction_reply))
{
assert(transaction_reply->elements-2<=multi_cmd_cnt);
assert(transaction_reply->elements==multi_cmd_cnt);
for(i=0;i<multi_cmd_cnt;i++)
{
p=transaction_reply->element[i];
@@ -1272,11 +1268,11 @@ int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_r
{
continue;
}
assert(j<serial_rule_num);
assert(i<(unsigned int)serial_rule_num);
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_command
,"%s %s %d failed, rule id maybe conflict or not exist."
,rm_op_str[s_rule[j].op]
,s_rule[j].table_name,s_rule[j].rule_id);
,rm_op_str[s_rule[i].op]
,s_rule[i].table_name,s_rule[i].rule_id);
success_cnt--;
last_failed=expected_reply[i].srule_seq;
}
@@ -1589,7 +1585,8 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list,
{
int ret=0, table_id=0, i=0;
int rule_with_foreign_key=0;
struct _Maat_table_info_t* p_table=NULL;
struct Maat_table_desc* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
for(i=0; i<rule_num; i++)
{
if(rule_list[i].table_line==NULL)
@@ -1602,11 +1599,12 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list,
continue;
}
p_table=feather->p_table_info[table_id];
if(p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->n_foreign==0)
plugin_desc= &(p_table->plugin);
if(p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->n_foreign==0)
{
continue;
}
_get_foregin_keys(rule_list+i, p_table->foreign_columns, p_table->n_foreign, dir, logger);
_get_foregin_keys(rule_list+i, plugin_desc->foreign_columns, plugin_desc->n_foreign, dir, logger);
rule_with_foreign_key++;
}
return rule_with_foreign_key;
@@ -1736,12 +1734,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
const unsigned char* dec_key,
_Maat_feather_t* feather)
{
int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0;
int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0, valid_column=-1;
int ret=0;
struct serial_rule_t* rule_list=NULL;
int update_type=CM_UPDATE_TYPE_INC;
long long new_version=0;
enum MAAT_TABLE_TYPE table_type;
const struct plugin_table_desc* plugin_desc=NULL;
void* logger=feather->logger;
if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write
@@ -1830,7 +1829,16 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
}
table_type=feather->p_table_info[table_id]->table_type;
if(rule_list[i].op==MAAT_OP_DEL)
{
{
if(table_type==TABLE_TYPE_PLUGIN)
{
plugin_desc=&(feather->p_table_info[table_id]->plugin);
valid_column=plugin_desc->valid_flag_column;
}
else
{
valid_column=-1;
}
ret=invalidate_line(rule_list[i].table_line, table_type, valid_column);
if(ret<0)
{
@@ -2015,7 +2023,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
int i=0, j=0;
_Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int ret=0, table_id=0,success_cnt=0;
struct serial_rule_t *s_rule=NULL;
struct serial_rule_t *s_rule=NULL;
struct Maat_table_desc* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
long long server_time=0,absolute_expire_time=0;
const char* p_foreign=NULL;
@@ -2039,6 +2048,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
ret=-1;
goto error_out;
}
p_table=_feather->p_table_info[table_id];
plugin_desc=&(p_table->plugin);
if(TABLE_TYPE_PLUGIN!=p_table->table_type)
{
@@ -2052,7 +2062,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
if(op==MAAT_OP_ADD)
{
ret=get_valid_flag_offset(line_rule[i]->table_line
, p_table->table_type
, p_table->table_type
, plugin_desc->valid_flag_column);
if(ret<0||
(op==MAAT_OP_ADD&&line_rule[i]->table_line[ret]!='1'))
@@ -2071,17 +2081,17 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
if(line_rule[i]->expire_after>0)
{
absolute_expire_time=server_time+line_rule[i]->expire_after;
}
}
if(plugin_desc->n_foreign>0)
{
{
for(j=0;j<plugin_desc->n_foreign;j++)
{
{
p_foreign=find_Nth_column(line_rule[i]->table_line, plugin_desc->foreign_columns[j], &foreign_key_size);
if(p_foreign==NULL)
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL, maat_command
, "Command set line %s %d failed: No %dth column."
, line_rule[i]->table_name, line_rule[i]->rule_id
, line_rule[i]->table_name, line_rule[i]->rule_id
, plugin_desc->foreign_columns[j]);
ret=-1;
goto error_out;
@@ -2096,7 +2106,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
}
}
}
}
set_serial_rule(s_rule+i, op,line_rule[i]->rule_id,line_rule[i]->label_id, line_rule[i]->table_name,
line_rule[i]->table_line, absolute_expire_time);
}
success_cnt=exec_serial_rule(write_ctx,s_rule, line_num,server_time,_feather->logger);
@@ -2126,6 +2137,11 @@ int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value,
int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value, size_t size, enum MAAT_OPERATION op)
{
struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather;
redisContext* ctx=_feather->mr_ctx.write_ctx;
if(ctx==NULL)
{
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, "%s failed: Redis is not connected.", __FUNCTION__);
return -1;
}
const char *arg_vec[3];
size_t len_vec[3];

789
src/entry/Maat_rule.cpp
View File

File diff suppressed because it is too large Load Diff

164
src/entry/Maat_stat.cpp
View File

@@ -42,7 +42,7 @@ void maat_stat_init(struct _Maat_feather_t* feather)
{
int value=0;
int i=0,j=0,offset=0;
struct _Maat_table_info_t* p_table=NULL;
struct Maat_table_desc* p_table=NULL;
char conj_table_name[(MAX_TABLE_NAME_LEN+1)*MAX_CONJUNCTION_TABLE_NUM]={0};
feather->stat_handle=FS_create_handle();
@@ -84,7 +84,7 @@ void maat_stat_init(struct _Maat_feather_t* feather)
feather->fs_status_id[STATUS_CMD_LINE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_SPEED,"line_cmd/s");
feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule");
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"regex");
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"reg/v6");
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"stream");
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"IN_Bps");
if(feather->perf_on==1)
@@ -149,20 +149,20 @@ void maat_stat_init(struct _Maat_feather_t* feather)
FS_start(feather->stat_handle);
return;
}
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num)
void maat_stat_table(struct Maat_table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num)
{
alignment_int64_array_add(p_table->scan_cnt,thread_num,1);
alignment_int64_array_add(p_table->input_bytes,thread_num,scan_len);
alignment_int64_array_add(p->scan_cnt,thread_num,1);
alignment_int64_array_add(p->input_bytes,thread_num,scan_len);
if(start!=NULL&&end!=NULL)
{
alignment_int64_array_add(p_table->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec);
alignment_int64_array_add(p->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec);
}
return;
}
void maat_stat_output(struct _Maat_feather_t* feather)
{
long value=0;
long long total_cfg_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0;
long long total_cfg_num=0, table_regex_ipv6_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0;
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0;
long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
long long outer_mid_cnt=0,inner_mid_cnt=0;
@@ -171,7 +171,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
long long compile_rule_num=0,group_rule_num=0,plugin_cache_num=0,plugin_acc_num=0;
int i=0;
time_t now;
struct _Maat_table_info_t* p_table=NULL;
struct Maat_table_desc* p_table=NULL;
struct Maat_table_runtime* table_rt=NULL;
time(&now);
active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num);
@@ -201,86 +202,105 @@ void maat_stat_output(struct _Maat_feather_t* feather)
table_input_bytes=0;
table_scan_cpu_time=0;
table_hit_cnt=0;
table_regex_ipv6_num=0;
p_table=feather->p_table_info[i];
if(p_table==NULL)
{
continue;
}
table_rt=feather->scanner->table_rt[i];
switch(p_table->table_type)
{
case TABLE_TYPE_PLUGIN:
plugin_cache_num+=p_table->cb_info->cache_line_num;
plugin_acc_num+=p_table->cb_info->acc_line_num;
plugin_cache_num+=table_rt->plugin.cache_line_num;
plugin_acc_num+=table_rt->plugin.acc_line_num;
break;
case TABLE_TYPE_GROUP:
group_rule_num+=p_table->cfg_num;
group_rule_num+=table_rt->origin_rule_num;
break;
case TABLE_TYPE_COMPILE:
compile_rule_num+=p_table->cfg_num;
compile_rule_num+=table_rt->origin_rule_num;
break;
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
table_regex_ipv6_num=table_rt->expr.regex_rule_cnt;
total_iconv_error=p_table->expr.iconv_err_cnt;
break;
case TABLE_TYPE_IP:
table_regex_ipv6_num=table_rt->ip.ipv6_rule_cnt;
break;
default:
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
FS_OP_SET,
p_table->cfg_num);
total_cfg_num+=p_table->cfg_num;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM],
FS_OP_SET,
p_table->regex_rule_cnt);
total_regex_num+= p_table->regex_rule_cnt;
table_stream_num=alignment_int64_array_sum(p_table->stream_num,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
FS_OP_SET,
table_stream_num);
total_stream_cnt+= table_stream_num;
table_scan_cnt=alignment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_SET,
table_scan_cnt);
total_scan_cnt+=table_scan_cnt;
table_input_bytes=alignment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_SET,
table_input_bytes);
total_input_bytes+=table_input_bytes;
if(feather->perf_on==1)
{
table_scan_cpu_time=alignment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num);
table_scan_cpu_time/=1000;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_SET,
table_scan_cpu_time);
total_cpu_time+=table_scan_cpu_time;
}
table_hit_cnt=alignment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_SET,
table_hit_cnt);
total_update_error+=p_table->udpate_err_cnt;
total_iconv_error+=p_table->iconv_err_cnt;
//total hit count stat in region_compile
break;
}
}
if(p_table->table_type==TABLE_TYPE_PLUGIN||
p_table->table_type==TABLE_TYPE_GROUP||
p_table->table_type==TABLE_TYPE_COMPILE)
{
continue;
}
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
FS_OP_SET,
table_rt->origin_rule_num);
total_cfg_num+=table_rt->origin_rule_num;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM],
FS_OP_SET,
table_regex_ipv6_num);
total_regex_num+=table_regex_ipv6_num;
table_stream_num=alignment_int64_array_sum(table_rt->stream_num,feather->scan_thread_num);
alignment_int64_array_reset(table_rt->stream_num,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
FS_OP_ADD,
table_stream_num);
total_stream_cnt+= table_stream_num;
table_scan_cnt=alignment_int64_array_sum(table_rt->scan_cnt,feather->scan_thread_num);
alignment_int64_array_reset(table_rt->scan_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_ADD,
table_scan_cnt);
total_scan_cnt+=table_scan_cnt;
table_input_bytes=alignment_int64_array_sum(table_rt->input_bytes,feather->scan_thread_num);
alignment_int64_array_reset(table_rt->input_bytes,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_ADD,
table_input_bytes);
total_input_bytes+=table_input_bytes;
if(feather->perf_on==1)
{
table_scan_cpu_time=alignment_int64_array_sum(table_rt->scan_cpu_time,feather->scan_thread_num);
alignment_int64_array_reset(table_rt->scan_cpu_time,feather->scan_thread_num);
table_scan_cpu_time/=1000;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_ADD,
table_scan_cpu_time);
total_cpu_time+=table_scan_cpu_time;
}
table_hit_cnt=alignment_int64_array_sum(table_rt->hit_cnt,feather->scan_thread_num);
alignment_int64_array_reset(table_rt->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_ADD,
table_hit_cnt);
total_update_error+=p_table->udpate_err_cnt;
}
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
@@ -314,7 +334,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_OP_SET,
total_cpu_time);
}
total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt,feather->scan_thread_num);
total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt, feather->scan_thread_num);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],

24
src/entry/Maat_utils.cpp
View File

@@ -164,6 +164,30 @@ char* str_unescape(char* s)
s[j]='\0';
return s;
}
int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len)
{
const char* seps=" \t";
char* saveptr=NULL, *subtoken=NULL, *str=NULL;
char* dup_line=_maat_strdup(line);
int i=0, ret=-1;
for (str = dup_line; ; str = NULL)
{
subtoken = strtok_r(str, seps, &saveptr);
if (subtoken == NULL)
break;
if(i==column_seq-1)
{
*offset=subtoken-dup_line;
*len=strlen(subtoken);
ret=0;
break;
}
i++;
}
free(dup_line);
return ret;
}
#define MAX_SYSTEM_CMD_LEN 512
int system_cmd_mkdir(const char* path)

10
src/entry/dynamic_array.cpp
View File

@@ -2,11 +2,11 @@
#include <stdlib.h>
#include <string.h>
int dynamic_array_VERSION_20141202=0;
struct dynamic_array_t* dynamic_array_create(int size,int step)
struct dynamic_array_t* dynamic_array_create(long long init_size, long long step)
{
struct dynamic_array_t* d_array=(struct dynamic_array_t*)calloc(sizeof(struct dynamic_array_t),1);
d_array->array=(void**)calloc(sizeof(void*),size);
d_array->size=size;
d_array->array=(void**)calloc(sizeof(void*),init_size);
d_array->size=init_size;
d_array->enlarge_step=step;
return d_array;
}
@@ -23,7 +23,7 @@ void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(vo
free(d_array->array);
free(d_array);
}
void* dynamic_array_read(struct dynamic_array_t* d_array,int i)
void* dynamic_array_read(struct dynamic_array_t* d_array, long long i)
{
if(i<d_array->size)
{
@@ -34,7 +34,7 @@ void* dynamic_array_read(struct dynamic_array_t* d_array,int i)
return NULL;
}
}
void dynamic_array_write(struct dynamic_array_t* d_array,int i,void* data)
void dynamic_array_write(struct dynamic_array_t* d_array, long long i, void* data)
{
int new_size=0;
if(i<d_array->size)

1
src/entry/json2iris.cpp
View File

@@ -12,6 +12,7 @@
#include "cJSON.h"
#include "hiredis.h"
#include "map_str2int.h"
#include "Maat_table_description.h"
#include "Maat_rule_internal.h"
#include "Maat_utils.h"

213
src/inc_internal/Maat_rule_internal.h
View File

@@ -3,6 +3,7 @@
#include "Maat_rule.h"
#include "Maat_command.h"
#include "Maat_table_description.h"
#include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h>
@@ -22,130 +23,24 @@
extern const char *maat_module;
#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411)
#define atomic_inc(x) __sync_add_and_fetch((x),1)
#define atomic_dec(x) __sync_sub_and_fetch((x),1)
#define atomic_add(x,y) __sync_add_and_fetch((x),(y))
#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y))
typedef int atomic_t;
#define ATOMIC_INIT(i) { (i) }
#define atomic_read(x) __sync_add_and_fetch((x),0)
#define atomic_set(x,y) __sync_lock_test_and_set((x),y)
#else
#include <alsa/iatomic.h>
#endif
#define TRUE 1
#define FALSE 0
#define MAX_TABLE_NUM 256
#define MAX_CONJUNCTION_TABLE_NUM 8
#define MAX_CHARSET_NUM 16
#define MAX_TABLE_NAME_LEN 256
#define MAX_TABLE_LINE_SIZE (1024*16)
#define MAX_EXPR_KEYLEN 1024
#define MAX_DISTRICT_LEN 64
#define MAX_PLUGIN_PER_TABLE 32
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_SCANNER_HIT_NUM 64
#define MAX_COMPILE_EX_DATA_NUM 2
#define MAX_GROUP_CACHE 128
#define MAX_FAILED_NUM 128
#define MAX_MAAT_STAT_NUM 64
#ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
#endif
#ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif
#ifndef container_of
#define container_of(ptr, type, member) ({ \
const typeof( ((type *)0)->member ) *__mptr = (ptr); \
(type *)( (char *)__mptr - offsetof(type,member) );})
#endif
typedef void* rule_scanner_t;
enum MAAT_TABLE_TYPE
{
TABLE_TYPE_EXPR=0,
TABLE_TYPE_IP,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_DIGEST,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_GROUP,
TABLE_TYPE_COMPILE,
TABLE_TYPE_PLUGIN
};
enum USER_REGION_ENCODE
{
USER_REGION_ENCODE_NONE=0,
USER_REGION_ENCODE_ESCAPE,
USER_REGION_ENCODE_BASE64
};
struct compile_ex_data_idx
{
Maat_rule_EX_new_func_t *new_func;
Maat_rule_EX_free_func_t* free_func;
Maat_rule_EX_dup_func_t* dup_func;
long argl;
void *argp;
int idx;
int table_id;
};
struct _Maat_table_info_t
{
unsigned short table_id;
unsigned short conj_cnt;
unsigned short updating_name;
char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
enum MAAT_TABLE_TYPE table_type;
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cfg_num;
int cross_cache_size;
int quick_expr_switch;
union
{
int expr_rule_cnt; //expr_type=0,1,3
int ipv4_rule_cnt;
};
union
{
int regex_rule_cnt; //expr_type=2
int ipv6_rule_cnt;
};
struct _plugin_table_info *cb_info;
int valid_flag_column; //for plugin table
int rule_tag_column; //for plugin table;
int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table;
int n_foreign;
//for compile table
enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num;
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
unsigned long long unmatch_tag_cnt;
unsigned long long iconv_err_cnt;
int stat_line_id;
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
};
struct db_str_rule_t
{
@@ -252,7 +147,7 @@ struct _Maat_compile_inner_t
dynamic_array_t *groups;
int is_valid;
int compile_id;//equal to db_c_rule->m_rule.config_id
const struct _Maat_table_info_t* ref_table;
const struct Maat_table_desc* ref_table;
int group_boundary;
int group_cnt;
MAAT_RULE_EX_DATA* ads;
@@ -263,41 +158,6 @@ struct _compile_result_t
int compile_id;
universal_bool_expr_t group_set;
};
struct _callback_plugin
{
Maat_start_callback_t *start;
Maat_update_callback_t *update;
Maat_finish_callback_t *finish;
void* u_para;
};
struct _plugin_table_info
{
int cb_plug_cnt;
struct _callback_plugin cb_plug[MAX_PLUGIN_PER_TABLE];
dynamic_array_t *cache_lines;
int cache_line_num;
int acc_line_num;
int update_type;
long cache_size;
};
struct _region_stat_t
{
int cfg_num;
union
{
int expr_rule_cnt; //expr_type=0,1,3
int ipv4_rule_cnt;
};
union
{
int regex_rule_cnt; //expr_type=2
int ipv6_rule_cnt;
};
};
struct _INNER_scan_status_t
{
int cur_hit_cnt;
@@ -349,12 +209,49 @@ struct _stream_para_t
pthread_mutex_t fuzzy_mutex;
unsigned char query_point[8];
};
struct GIE_aux_t
struct similar_runtime
{
enum MAAT_TABLE_TYPE table_type;
GIE_handle_t* gie_handle;
MESA_lqueue_head update_q;
};
struct plugin_runtime
{
dynamic_array_t *cache_lines;
long long cache_line_num;
long long acc_line_num;
long long cache_size;
pthread_rwlock_t rwlock;
MESA_htable_handle key2ex_hash;
};
struct expr_runtime
{
long long expr_rule_cnt; //expr_type=0,1,3
long long regex_rule_cnt; //expr_type=2
};
struct ip_runtime
{
long long ipv4_rule_cnt;
long long ipv6_rule_cnt;
};
struct Maat_table_runtime
{
enum MAAT_TABLE_TYPE table_type;
long origin_rule_num;
union
{
struct similar_runtime similar; //for digest and similarity
struct plugin_runtime plugin;
struct expr_runtime expr;
struct ip_runtime ip;
void * other;
};
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
};
struct rule_tag
{
char* tag_name;
@@ -364,10 +261,15 @@ struct _Maat_scanner_t
{
long long version;
time_t last_update_time;
long long *ref_cnt; //optimized for cache_alignment 64
mcore_long_t ref_cnt;
rule_scanner_t region;
long gie_total_q_size;
struct GIE_aux_t gie_aux[MAX_TABLE_NUM];
struct Maat_table_runtime* table_rt[MAX_TABLE_NUM];
/*
struct similar_runtime gie_aux[MAX_TABLE_NUM];
struct plugin_runtime plugin_aux[MAX_TABLE_NUM];
*/
MESA_htable_handle region_hash;
MESA_htable_handle group_hash;
MESA_htable_handle compile_hash;
@@ -378,10 +280,10 @@ struct _Maat_scanner_t
unsigned int exprid_generator;
unsigned int dedup_expr_num;
MESA_lqueue_head region_update_q;
void * expr_compiler;
void * bool_macher_expr_compiler;
scan_result_t *region_rslt_buff;
MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q
struct _region_stat_t region_counter[MAX_TABLE_NUM];
// struct _region_stat_t region_counter[MAX_TABLE_NUM];
int max_thread_num;
iconv_t iconv_handle[MAX_CHARSET_NUM][MAX_CHARSET_NUM];//iconv_handle[to][from]
};
@@ -435,7 +337,7 @@ struct _Maat_feather_t
int cumulative_update_off;
int stat_on;
int perf_on;
struct _Maat_table_info_t *p_table_info[MAX_TABLE_NUM];
struct Maat_table_desc *p_table_info[MAX_TABLE_NUM];
MESA_htable_handle map_tablename2id;
void* logger;
long long maat_version;
@@ -534,7 +436,7 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set);
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
void maat_start_cb(long long new_version,int update_type,void*u_para);
int maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para);
@@ -546,7 +448,7 @@ int HASH_add_by_id(MESA_htable_handle hash,int id,void*data);
int HASH_delete_by_id(MESA_htable_handle hash,int id);
void maat_read_full_config(_Maat_feather_t* _feather);
void maat_stat_init(struct _Maat_feather_t* feather);
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num);
void maat_stat_table(struct Maat_table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num);
void maat_stat_output(struct _Maat_feather_t* feather);
redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...);
@@ -559,11 +461,14 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p);
void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len);
MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc);
void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc);
MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index);
int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger);
int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger);
void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout);
void empty_serial_rules(struct serial_rule_t* rule);
int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_rule_num, long long server_time, void* logger);
int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,unsigned int serial_rule_num, long long server_time, void* logger);
long long redis_server_time(redisContext* ctx);
redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db, void* logger);
char* md5_file(const char* filename, char* md5string);

113
src/inc_internal/Maat_table_description.h Normal file
View File

@@ -0,0 +1,113 @@
#pragma once
#include "Maat_rule.h"
#define MAX_COMPILE_EX_DATA_NUM 2
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_PLUGIN_PER_TABLE 32
#define MAX_CHARSET_NUM 16
#define MAX_CONJUNCTION_TABLE_NUM 8
#define MAX_TABLE_NAME_LEN 256
enum USER_REGION_ENCODE
{
USER_REGION_ENCODE_NONE=0,
USER_REGION_ENCODE_ESCAPE,
USER_REGION_ENCODE_BASE64
};
enum MAAT_TABLE_TYPE
{
TABLE_TYPE_EXPR=0,
TABLE_TYPE_IP,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_DIGEST,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_GROUP,
TABLE_TYPE_COMPILE,
TABLE_TYPE_PLUGIN
};
struct compile_ex_data_idx
{
Maat_rule_EX_new_func_t *new_func;
Maat_rule_EX_free_func_t* free_func;
Maat_rule_EX_dup_func_t* dup_func;
long argl;
void *argp;
int idx;
int table_id;
};
struct compile_table_desc
{
enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num;
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
};
struct plugin_table_callback_desc
{
Maat_start_callback_t *start;
Maat_update_callback_t *update;
Maat_finish_callback_t *finish;
void* u_para;
};
struct plugin_table_ex_data_desc
{
Maat_plugin_EX_new_func_t* new_func;
Maat_plugin_EX_free_func_t* free_func;
Maat_plugin_EX_dup_func_t* dup_func;
Maat_plugin_EX_key2index_func_t* key2index_func;
long argl;
void *argp;
};
struct plugin_table_desc
{
int key_column;
int valid_flag_column;
int rule_tag_column;
int n_foreign;
int foreign_columns[MAX_FOREIGN_CLMN_NUM];
int cb_plug_cnt;
int have_exdata;
long long estimate_size;
struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE];
struct plugin_table_ex_data_desc ex_desc;
};
struct expr_table_desc
{
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cross_cache_size;
int quick_expr_switch;
long long iconv_err_cnt;
};
struct ip_table_desc
{
int ipv4_rule_cnt;
int ipv6_rule_cnt;
};
struct Maat_table_desc
{
int table_id;
int conj_cnt;
int updating_name;
char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
enum MAAT_TABLE_TYPE table_type;
union
{
struct compile_table_desc compile;
struct expr_table_desc expr;
struct ip_table_desc ip;
struct plugin_table_desc plugin;
void* others;//group, interval and digest don't have sperate description info.
};
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
unsigned long long unmatch_tag_cnt;
int stat_line_id;
};

29
src/inc_internal/Maat_utils.h
View File

@@ -9,9 +9,20 @@
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
#define FREE(p) {free(*p);*p=NULL;}
#define ATOMIC_INC(x) __atomic_fetch_add(x,1,__ATOMIC_RELAXED)
#define ATOMIC_READ(x) __atomic_fetch_add(x,0,__ATOMIC_RELAXED)
#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411)
#define atomic_inc(x) __sync_add_and_fetch((x),1)
#define atomic_dec(x) __sync_sub_and_fetch((x),1)
#define atomic_add(x,y) __sync_add_and_fetch((x),(y))
#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y))
typedef int atomic_t;
#define ATOMIC_INIT(i) { (i) }
#define atomic_read(x) __sync_add_and_fetch((x),0)
#define atomic_set(x,y) __sync_lock_test_and_set((x),y)
#else
#include <alsa/iatomic.h>
#endif
#define TRUE 1
#define FALSE 0
#ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
@@ -21,6 +32,16 @@
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif
#ifndef container_of
#define container_of(ptr, type, member) ({ \
const typeof( ((type *)0)->member ) *__mptr = (ptr); \
(type *)( (char *)__mptr - offsetof(type,member) );})
#endif
#define UNUSED __attribute__((unused))
const char* module_name_str(const char*name);
#define maat_module (module_name_str("MAAT_Frame"))
@@ -47,4 +68,6 @@ int system_cmd_rm(const char* src_file);
int system_cmd_mv(const char* src_file,const char*dst_file);
int system_cmd_cp(const char* src_file,const char*dst_file);
char* md5_file(const char* filename, char* md5string);
int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len);

5
src/inc_internal/alignment_int64.h
View File

@@ -43,6 +43,11 @@ inline long long alignment_int64_array_cnt(mcore_long_t array,int size)
}
return cnt;
}
inline void alignment_int64_array_reset(mcore_long_t array,int size)
{
memset(array, 0, CPU_CACHE_ALIGMENT*size);
return;
}
inline void alignment_int64_array_free(mcore_long_t array)
{
free(array);

10
src/inc_internal/dynamic_array.h
View File

@@ -3,11 +3,11 @@
struct dynamic_array_t
{
void ** array;
int size;
int enlarge_step;
long long size;
long long enlarge_step;
};
struct dynamic_array_t* dynamic_array_create(int size,int step);
struct dynamic_array_t* dynamic_array_create(long long init_size, long long step);
void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(void *));
void* dynamic_array_read(struct dynamic_array_t* d_array,int i);
void dynamic_array_write(struct dynamic_array_t* d_array,int i,void* data);
void* dynamic_array_read(struct dynamic_array_t* d_array,long long i);
void dynamic_array_write(struct dynamic_array_t* d_array, long long i,void* data);
#endif //_DYNAMIC_ARRAY_H_INCLUDE_

19
test/maat_demo.cpp
View File

@@ -675,35 +675,36 @@ int test_table_conjunction(Maat_feather_t feather,const char* table_name,const c
}
return 0;
}
#define TEST_CMD_LINE_NUM 4
#define TEST_CMD_LINE_NUM 200*1000
void test_set_cmd_line(Maat_feather_t feather)
{
const struct Maat_line_t *p_line[TEST_CMD_LINE_NUM];
struct Maat_line_t line_rule[TEST_CMD_LINE_NUM];
char table_line[TEST_CMD_LINE_NUM][128];
struct Maat_line_t **p_line=(struct Maat_line_t **)calloc(sizeof(struct Maat_line_t *), TEST_CMD_LINE_NUM);
struct Maat_line_t *line_rule=(struct Maat_line_t *)calloc(sizeof(struct Maat_line_t), TEST_CMD_LINE_NUM);
int i=0;
const char* line="1\t192.168.0.1\t4444444444\t1";
memset(&line_rule,0,sizeof(line_rule));
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
line_rule[i].label_id=0;
line_rule[i].rule_id=(int)Maat_cmd_incrby(feather,"TEST_PLUG_SEQ", 1);
line_rule[i].table_name="QD_ENTRY_INFO";
snprintf(table_line[i],sizeof(table_line[i]),"1\t192.168.0.1\t%d\t1",100+i);
line_rule[i].table_line=table_line[i];
line_rule[i].table_line=line;
// asprintf(&(line_rule[i].table_line),"1\t192.168.0.1\t%d\t1",100+i);
line_rule[i].expire_after=0;
p_line[i]=line_rule+i;
}
Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
Maat_cmd_set_lines(feather,(const struct Maat_line_t **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
usleep(WAIT_FOR_EFFECTIVE_US);
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
line_rule[i].table_line=NULL;
}
Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
Maat_cmd_set_lines(feather, (const struct Maat_line_t **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
free(p_line);
free(line_rule);
return;
}

3
test/table_info.conf
View File

@@ -28,5 +28,6 @@
9 SIM_URL similar --
10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff
11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} --
12 TEST_FOREIGN_KEY plugin {"valid":4,"foreign":"6,8","tag":3} --
12 TEST_FOREIGN_KEY plugin {"valid":4,"foreign":[6,8],"tag":3} --
13 COMPILE_ALIAS compile escape --
14 TEST_PLUGIN_EXDATA_TABLE plugin {"key":2,"valid":4,"tag":5,"estimate_size":1024} --

369
test/test_maatframe.cpp
View File

@@ -38,6 +38,19 @@ int g_iThreadNum=4;
const char* table_info_path="./table_info.conf";
int scan_interval_ms=1;
int effective_interval_ms=0;
void wait_for_cmd_effective(Maat_feather_t feather, long long version_before)
{
long long version_after=version_before;
int is_updating=1;
while(is_updating||version_before==version_after)
{
Maat_read_state(feather,MAAT_STATE_IN_UPDATING, &is_updating, sizeof(is_updating));
Maat_read_state(feather,MAAT_STATE_VERSION, &version_after, sizeof(version_after));
usleep(1000*100);//waiting for commands go into effect
}
}
void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old)
{
@@ -480,7 +493,7 @@ TEST(SimilarScan, Pure)
const char* scan_data="mwss.xiu.youku.com/live/hls/v1/0000000000000000000000001526a0a8/714.ts?&token=98765";
const char* table_name="SIM_URL";
scan_status_t mid=NULL;
memset(result, 0 , sizeof(result));
table_id=Maat_table_register(g_feather,table_name);
ASSERT_GT(table_id, 0);
@@ -640,7 +653,7 @@ struct rule_ex_param
pthread_mutex_t lock;
};
void ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
void compile_ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
MAAT_RULE_EX_DATA* ad, long argl, void *argp)
{
int *counter=(int*)argp;
@@ -657,7 +670,7 @@ void ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_l
*ad=param;
return;
}
void ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp)
void compile_ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp)
{
if(*ad==NULL)
{
@@ -674,7 +687,7 @@ void ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_
free(param);
return;
}
void ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp)
void compile_ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp)
{
struct rule_ex_param* from_param=*((struct rule_ex_param**)from);
pthread_mutex_lock(&(from_param->lock));
@@ -686,9 +699,9 @@ void ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long
TEST(EX_DATA_INDEX, all)
TEST(EX_DATA_INDEX, MaatRuleEXData)
{
#define EX_data_index
#define rule_EX_data_index
int ret=0;
int table_id=0, ex_data_counter=0;
@@ -701,7 +714,7 @@ TEST(EX_DATA_INDEX, all)
ASSERT_GT(table_id, 0);
int ex_param_idx=Maat_rule_get_ex_new_index(g_feather, "COMPILE_ALIAS",
ex_param_new, ex_param_free, ex_param_dup,
compile_ex_param_new, compile_ex_param_free, compile_ex_param_dup,
0, &ex_data_counter);
ASSERT_TRUE(ex_param_idx>=0);
EXPECT_EQ(ex_data_counter, 1);
@@ -716,7 +729,7 @@ TEST(EX_DATA_INDEX, all)
struct rule_ex_param* param=(struct rule_ex_param*)ex_data;
EXPECT_EQ(param->id, 7799);
EXPECT_EQ(strcmp(param->name, expect_name),0);
ex_param_free(0, NULL, NULL, &ex_data, 0, NULL);
compile_ex_param_free(0, NULL, NULL, &ex_data, 0, NULL);
Maat_clean_status(&mid);
return;
}
@@ -1033,6 +1046,95 @@ int del_command(Maat_feather_t feather,int config_id)
Maat_free_cmd(cmd);
return 0;
}
TEST_F(MaatCmdTest, SetIP)
{
struct Maat_cmd_t* cmd=NULL;
struct Maat_rule_t rule;
int config_id=0,timeout=4;
long long version_before=0;
const char* region_table="IP_CONFIG";
struct Maat_region_t region;
int group_num=1,ret=0;
memset(&rule,0,sizeof(rule));
Maat_feather_t feather=MaatCmdTest::_shared_feather;
//MUST acquire by Maat_cmd_incrby to guarantee a unique compile ID.
config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
rule.config_id=config_id;
strcpy(rule.service_defined,"maat_command");
//MUST acqire by function, because Maat_cmd_t has some hidden members.
cmd=Maat_create_cmd(&rule, group_num);
cmd->expire_after=timeout;
cmd->label_id=0; //no lable
memset(&region,0,sizeof(region));
region.region_type=REGION_IP;
region.table_name=region_table;
region.ip_rule.addr_type=ADDR_TYPE_IPv4;
region.ip_rule.direction=ADDR_DIR_DOUBLE;
region.ip_rule.src_ip="172.0.0.1";
region.ip_rule.mask_src_ip="255.255.255.255";
region.ip_rule.src_port=53331;
region.ip_rule.mask_src_port=0;//means any port should hit.
region.ip_rule.dst_ip="172.0.0.2";
region.ip_rule.mask_dst_ip="255.255.255.255";
region.ip_rule.dst_port=80;
region.ip_rule.mask_dst_port=65535;
region.ip_rule.protocol=0;//means any protocol should hit.
Maat_add_region2cmd(cmd, 0, &region);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
EXPECT_EQ(ret, 0);
ret=Maat_cmd(feather, cmd, MAAT_OP_ADD);
EXPECT_GE(ret, 0);
Maat_free_cmd(cmd);
cmd=NULL;
wait_for_cmd_effective(feather, version_before);
struct ipaddr ipv4_addr;
struct stream_tuple4_v4 v4_addr;
ipv4_addr.addrtype=ADDR_TYPE_IPV4;
inet_pton(AF_INET,region.ip_rule.src_ip,&(v4_addr.saddr));
v4_addr.source=htons(region.ip_rule.src_port+1);//Not use the exactly port for testing port mask.
inet_pton(AF_INET,region.ip_rule.dst_ip,&(v4_addr.daddr));
v4_addr.dest=htons(region.ip_rule.dst_port);
ipv4_addr.v4=&v4_addr;
int table_id=0;
struct Maat_rule_t result;
memset(&result, 0, sizeof(result));
scan_status_t mid=NULL;
table_id=Maat_table_register(feather,region_table);
ASSERT_GE(table_id, 0);
ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,&result,1, &mid,0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result.config_id, config_id);
Maat_clean_status(&mid);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
//reset timeout.
cmd=Maat_create_cmd(&rule, 0);
cmd->expire_after=10;
ret=Maat_cmd(feather, cmd, MAAT_OP_RENEW_TIMEOUT);
EXPECT_EQ(ret ,1);
// wait_for_cmd_effective(feather, version_before);
Maat_free_cmd(cmd);
cmd=NULL;
sleep(timeout+1);
ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,&result,1, &mid,0);
EXPECT_EQ(ret, 1);
Maat_clean_status(&mid);
return;
}
TEST_F(MaatCmdTest, SetExpr)
{
@@ -1107,6 +1209,8 @@ TEST_F(MaatCmdTest, SetExpr)
&mid, 0);
EXPECT_EQ(ret, 0);
}
TEST_F(MaatCmdTest, SetLines)
{
const int TEST_CMD_LINE_NUM=4;
@@ -1142,95 +1246,96 @@ TEST_F(MaatCmdTest, SetLines)
return;
}
TEST_F(MaatCmdTest, SetIP)
//Following tests must be coded/tested at last, for they stalled the maat update thread and interrupt other tests.
TEST_F(MaatCmdTest, SetExprOneMillion)
{
const int CMD_EXPR_NUM=1000*1000;
const char* table_name="HTTP_URL";
const char* keywords1="Hiredis";
const char* keywords2="C Client";
char escape_buff1[256],escape_buff2[256];
char keywords[256];
int label_id=5210, config_id=0,ret=0, output_id_cnt=0;
Maat_feather_t feather=MaatCmdTest::_shared_feather;
long long version_before=0;
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
Maat_str_escape(escape_buff1, sizeof(escape_buff1),keywords1);
Maat_str_escape(escape_buff2, sizeof(escape_buff2),keywords2);
snprintf(keywords,sizeof(keywords),"%s&%s",escape_buff1,escape_buff2);
config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", CMD_EXPR_NUM);
int i=0;
for(i=0; i<CMD_EXPR_NUM;i++)
{
test_add_expr_command(feather,table_name,config_id-i, 0, label_id, keywords);
}
ret=Maat_cmd_commit(feather);
EXPECT_TRUE(ret>=0);
wait_for_cmd_effective(feather, version_before);
struct Maat_cmd_t* cmd=NULL;
struct Maat_rule_t rule;
int config_id=0,timeout=4;
long long version_before=0,version_after=0;
const char* region_table="IP_CONFIG";
struct Maat_region_t region;
int group_num=1,ret=0;
memset(&rule,0,sizeof(rule));
int *output_ids=(int*)malloc(sizeof(int)*CMD_EXPR_NUM);
output_id_cnt=Maat_cmd_select(feather,label_id, output_ids, CMD_EXPR_NUM);
EXPECT_EQ(output_id_cnt, CMD_EXPR_NUM);
for(i=0; i<CMD_EXPR_NUM;i++)
{
memset(&rule,0,sizeof(rule));
rule.config_id=output_ids[i];
cmd=Maat_create_cmd(&rule, 0);
ret=Maat_cmd_append(feather, cmd, MAAT_OP_DEL);
EXPECT_EQ(ret, 0);
Maat_free_cmd(cmd);
}
ret=Maat_cmd_commit(feather);
EXPECT_EQ(ret, CMD_EXPR_NUM);
free(output_ids);
}
TEST_F(MaatCmdTest, SetLinesOneMillion)
{
const int TEST_CMD_LINE_NUM=1000*1000;
Maat_feather_t feather=MaatCmdTest::_shared_feather;
struct Maat_line_t **p_line=(struct Maat_line_t **)calloc(sizeof(struct Maat_line_t *), TEST_CMD_LINE_NUM);
struct Maat_line_t *line_rule=(struct Maat_line_t *)calloc(sizeof(struct Maat_line_t), TEST_CMD_LINE_NUM);
int i=0;
const char* line="1\t192.168.0.1\t4444444444\t1";
int seq=(int)Maat_cmd_incrby(feather,"TEST_PLUG_SEQ", TEST_CMD_LINE_NUM);
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
line_rule[i].label_id=0;
line_rule[i].rule_id=seq-i;
line_rule[i].table_name="QD_ENTRY_INFO";
line_rule[i].table_line=line;
// asprintf(&(line_rule[i].table_line),"1\t192.168.0.1\t%d\t1",100+i);
line_rule[i].expire_after=0;
p_line[i]=line_rule+i;
}
//MUST acquire by Maat_cmd_incrby to guarantee a unique compile ID.
config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
rule.config_id=config_id;
long long version_before=0;
Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
Maat_cmd_set_lines(feather,(const struct Maat_line_t **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
strcpy(rule.service_defined,"maat_command");
//MUST acqire by function, because Maat_cmd_t has some hidden members.
cmd=Maat_create_cmd(&rule, group_num);
cmd->expire_after=timeout;
cmd->label_id=0; //no lable
memset(&region,0,sizeof(region));
region.region_type=REGION_IP;
region.table_name=region_table;
region.ip_rule.addr_type=ADDR_TYPE_IPv4;
region.ip_rule.direction=ADDR_DIR_DOUBLE;
region.ip_rule.src_ip="172.0.0.1";
region.ip_rule.mask_src_ip="255.255.255.255";
region.ip_rule.src_port=53331;
region.ip_rule.mask_src_port=0;//means any port should hit.
wait_for_cmd_effective(feather, version_before);
region.ip_rule.dst_ip="172.0.0.2";
region.ip_rule.mask_dst_ip="255.255.255.255";
region.ip_rule.dst_port=80;
region.ip_rule.mask_dst_port=65535;
region.ip_rule.protocol=0;//means any protocol should hit.
Maat_add_region2cmd(cmd, 0, &region);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
EXPECT_EQ(ret, 0);
ret=Maat_cmd(feather, cmd, MAAT_OP_ADD);
EXPECT_GE(ret, 0);
Maat_free_cmd(cmd);
cmd=NULL;
//TEST if the command go into effective.
usleep(WAIT_FOR_EFFECTIVE_US); //waiting for commands go into effect
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_after, sizeof(version_after));
struct ipaddr ipv4_addr;
struct stream_tuple4_v4 v4_addr;
ipv4_addr.addrtype=ADDR_TYPE_IPV4;
inet_pton(AF_INET,region.ip_rule.src_ip,&(v4_addr.saddr));
v4_addr.source=htons(region.ip_rule.src_port+1);//Not use the exactly port for testing port mask.
inet_pton(AF_INET,region.ip_rule.dst_ip,&(v4_addr.daddr));
v4_addr.dest=htons(region.ip_rule.dst_port);
ipv4_addr.v4=&v4_addr;
int table_id=0;
struct Maat_rule_t result;
memset(&result, 0, sizeof(result));
scan_status_t mid=NULL;
table_id=Maat_table_register(feather,region_table);
ASSERT_GE(table_id, 0);
ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,&result,1, &mid,0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result.config_id, config_id);
Maat_clean_status(&mid);
//reset timeout.
cmd=Maat_create_cmd(&rule, 0);
cmd->expire_after=10;
ret=Maat_cmd(feather, cmd, MAAT_OP_RENEW_TIMEOUT);
EXPECT_EQ(ret ,1);
usleep(2*1000*1000+WAIT_FOR_EFFECTIVE_US);//wait for commands expired.
Maat_free_cmd(cmd);
cmd=NULL;
ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,&result,1, &mid,0);
EXPECT_EQ(ret, 1);
Maat_clean_status(&mid);
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
line_rule[i].table_line=NULL;
}
Maat_cmd_set_lines(feather, (const struct Maat_line_t **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
free(p_line);
free(line_rule);
return;
}
int g_test_update_paused=0;
void pause_update_test_entry_cb(int table_id,const char* table_line,void* u_para)
{
@@ -1418,6 +1523,98 @@ TEST_F(MaatCmdTest, SetFile)
return;
}
struct user_info
{
char name[256];
char ip_addr[32];
int id;
int ref_cnt;
};
void plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{
int *counter=(int *)argp;
struct user_info* u=(struct user_info*)calloc(sizeof(struct user_info), 1);
int valid=0, tag=0, ret;
ret=sscanf(table_line, "%d\t%s\t%s%d\t%d", &(u->id), u->ip_addr, u->name, &valid, &tag);
EXPECT_EQ(ret, 5);
u->ref_cnt=1;
*ad=u;
(*counter)++;
return;
}
void plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{
struct user_info* u=(struct user_info*)(*ad);
u->ref_cnt--;
if(u->ref_cnt>0) return;
free(u);
*ad=NULL;
}
void plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
{
struct user_info* u=(struct user_info*)(*from);
u->ref_cnt++;
*to=u;
}
TEST_F(MaatCmdTest, PluginEXData)
{
#define plugin_EX_data_index
Maat_feather_t feather=MaatCmdTest::_shared_feather;
int ret=0, i=0;
int table_id=0, ex_data_counter=0;
const char* table_name="TEST_PLUGIN_EXDATA_TABLE";
const int TEST_CMD_LINE_NUM=4;
const struct Maat_line_t *p_line[TEST_CMD_LINE_NUM];
struct Maat_line_t line_rule[TEST_CMD_LINE_NUM];
const char* table_line[TEST_CMD_LINE_NUM]={"1\t192.168.0.1\tmahuateng\t1\t0",
"2\t192.168.0.2\tliuqiangdong\t1\t0",
"3\t192.168.0.3\tmayun\t1\t0",
"4\t192.168.0.4\tliyanhong\t1\t0"};
table_id=Maat_table_register(feather, table_name);
ASSERT_GT(table_id, 0);
memset(&line_rule,0,sizeof(line_rule));
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
line_rule[i].label_id=0;
line_rule[i].rule_id=(int)Maat_cmd_incrby(feather,"TEST_PLUG_SEQ", 1);
line_rule[i].table_name=table_name;
line_rule[i].table_line=table_line[i];
line_rule[i].expire_after=0;
p_line[i]=line_rule+i;
}
ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
EXPECT_GT(ret, 0);
usleep(WAIT_FOR_EFFECTIVE_US);
ret=Maat_plugin_EX_register(feather, table_id,
plugin_EX_new_cb,
plugin_EX_free_cb,
plugin_EX_dup_cb,
NULL,
0, &ex_data_counter);
ASSERT_TRUE(ret>=0);
EXPECT_EQ(ex_data_counter, TEST_CMD_LINE_NUM);
struct user_info* uinfo=NULL;
uinfo=(struct user_info*)Maat_plugin_get_EX_data(feather, table_id, "192.168.0.2");
ASSERT_TRUE(uinfo!=NULL);
EXPECT_EQ(0, strcmp(uinfo->name, "liuqiangdong"));
EXPECT_EQ(uinfo->id, 2);
plugin_EX_free_cb(table_id, (void**)&uinfo, 0, NULL);
ret=Maat_cmd_set_lines(feather, p_line+1,1, MAAT_OP_DEL);
EXPECT_GT(ret, 0);
usleep(WAIT_FOR_EFFECTIVE_US);
uinfo=(struct user_info*)Maat_plugin_get_EX_data(feather, table_id, "192.168.0.2");
ASSERT_TRUE(uinfo==NULL);
return;
}
int main(int argc, char ** argv)
{

6
tools/maat_redis_tool.cpp
View File

@@ -173,7 +173,7 @@ clean_up:
}
int count_line_num(const char* table_name,const char* line,void *u_para)
{
(*((int *)u_para))++;
(*((unsigned int *)u_para))++;
return 0;
}
int line_idx=0;
@@ -221,7 +221,7 @@ int make_serial_rule(const char* table_name,const char* line,void *u_para)
#define WORK_MODE_JSON 1
int main(int argc, char * argv[])
{
int oc=0,ret=0, i=0,success_cnt=0;
int oc=0,ret=0;
int model=0;
char redis_ip[64];
int redis_port=6379;
@@ -230,7 +230,7 @@ int main(int argc, char * argv[])
char dump_dir[128], json_file[128], tmp_iris_path[128];
strncpy(dump_dir,redis_dump_dir,sizeof(dump_dir));
redisContext * ctx=NULL;
int total_line_cnt=0;
unsigned int total_line_cnt=0, success_cnt=0, i=0;
int timeout=0;
FILE* json_fp=NULL;
cJSON *json=NULL, *tmp_obj=NULL;