gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

重构取column的函数

Browse Source
This commit is contained in:
zhengchao
2018-12-04 23:26:59 +08:00
parent 6971f4cb56
commit 7b5baacf62
11 changed files with 310 additions and 248 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
inc/Maat_rule.h
View File

@@ -259,8 +259,8 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa
//Following functions are similar to Maat_rule_get_ex_data, except they are effective on plugin table.
typedef void* MAAT_PLUGIN_EX_DATA;
typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);
typedef void Maat_plugin_EX_free_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);
typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, size_t key_sz, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);
typedef void Maat_plugin_EX_free_func_t(int table_id, const char* key, size_t key_sz, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);
typedef void Maat_plugin_EX_dup_func_t(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp);
typedef int Maat_plugin_EX_key2index_func_t(const char* key);

176
src/entry/Maat_api.cpp
View File

@@ -16,9 +16,9 @@
#include "rulescan.h"
#include "json2iris.h"
struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type)
struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type)
{
struct _Maat_table_info_t *p_table=NULL;
struct Maat_table_desc *p_table=NULL;
if(table_id>MAX_TABLE_NUM)
{
return NULL;
@@ -918,7 +918,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
int idx=0,i=0;
struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id];
struct Maat_table_desc *p_table=_feather->p_table_info[table_id];
struct plugin_table_desc *plugin_desc=&(p_table->plugin);
struct plugin_runtime* plugin_aux=NULL;
const char* lines=NULL;
@@ -999,8 +999,8 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table
{
return -1;
}
struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id];
if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL)
struct Maat_table_desc *p_table=_feather->p_table_info[table_id];
if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL || dup_func==NULL)
{
return -1;
}
@@ -1051,6 +1051,47 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa
pthread_rwlock_unlock(&(compile_inner->rwlock));
return ad;
}
MESA_htable_handle plugin_EX_htable_new(const struct plugin_table_desc* plugin_desc,
struct dynamic_array_t* lines, size_t line_cnt)
{
MESA_htable_handle key2ex_hash=NULL;
unsigned int slot_size=1;
const char* line=NULL;
size_t key_offset=0, key_len=0;
long long estimate_size=plugin_desc->estimate_size, i=0;
int ret=0;
MAAT_RULE_EX_DATA exdata=NULL;
while(estimate_size>0)
{
estimate_size=estimate_size<<1;
slot_size*=2;
}
if(slot_size==1)
{
slot_size=4096;
}
MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs));
hargs.thread_safe=1;
hargs.hash_slot_size = slot_size;
hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
hargs.expire_time = 0;
hargs.key_comp = NULL;
hargs.key2index = plugin_desc->ex_desc.key2index_func;
hargs.recursive = 1;
hargs.data_free = NULL ;
hargs.data_expire_with_condition = NULL;
key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(key2ex_hash, 0);
for(i=0; i< line_cnt; i++)
{
line=(const char*)dynamic_array_read(lines, i);
ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
exdata=plugin_desc->ex_desc.new_func(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);;
MESA_htable_add(key2ex_hash, line+key_offset, key_len, const void * data)
}
}
int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
@@ -1059,6 +1100,27 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
long argl, void *argp)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_table_desc *table_desc=_feather->p_table_info[table_id];
struct plugin_table_desc* plugin_desc=&(table_desc->plugin);
struct Maat_table_runtime* table_rt=_feather->scanner->table_rt[table_id];
MESA_htable_handle key2ex_hash=NULL;
if(table_desc->table_type!=TABLE_TYPE_PLUGIN || new_func==NULL || free_func==NULL || dup_func==NULL)
{
assert(0);
return -1;
}
plugin_desc->ex_desc.new_func=new_func;
plugin_desc->ex_desc.free_func=free_func;
plugin_desc->ex_desc.dup_func=dup_func;
plugin_desc->ex_desc.key2index_func=key2index_func;
plugin_desc->ex_desc.argl=argl;
plugin_desc->ex_desc.argp=argp;
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
table_rt->plugin.key2ex_hash=plugin_EX_htable_new(plugin_desc,table_rt->plugin.cache_lines, table_rt->plugin.cache_line_num);
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return 0;
}
@@ -1074,7 +1136,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];//dynamic array
struct _Maat_table_info_t *p_table=NULL;
struct Maat_table_desc *p_table=NULL;
struct expr_table_desc* expr_desc=NULL;
struct timespec start,end;
_Maat_scanner_t* my_scanner=NULL;
@@ -1117,8 +1179,8 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{
return 0;
}
struct table_runtime* table_aux=my_scanner->table_rt[table_id];
if(table_aux->origin_rule_num==0)
struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id];
if(table_rt->origin_rule_num==0)
{
return 0;
}
@@ -1126,7 +1188,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
INC_SCANNER_REF(my_scanner, thread_num);
if(table_aux->expr.expr_rule_cnt>0)
if(table_rt->expr.expr_rule_cnt>0)
{
scan_data.rule_type=RULETYPE_STR;
scan_data.sub_type=sub_type;
@@ -1136,7 +1198,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
hit_region_cnt+=region_ret;
}
}
if(table_aux->expr.regex_rule_cnt>0)
if(table_rt->expr.regex_rule_cnt>0)
{
scan_data.rule_type=RULETYPE_REG;
scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0);
@@ -1153,7 +1215,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
}
if(hit_region_cnt>0)
{
alignment_int64_array_add(table_aux->hit_cnt, thread_num,1);
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
_mid=grab_mid(mid,_feather,thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
@@ -1177,11 +1239,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_aux, data_len, &start, &end, thread_num);
maat_stat_table(table_rt, data_len, &start, &end, thread_num);
}
else
{
maat_stat_table(table_aux, data_len, NULL, NULL, thread_num);
maat_stat_table(table_rt, data_len, NULL, NULL, thread_num);
}
DEC_SCANNER_REF(my_scanner, thread_num);
if(compile_ret==0&&hit_region_cnt>0)
@@ -1217,7 +1279,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
intval_scan_data.rule_type=RULETYPE_INT;
intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0);
intval_scan_data.int_data=intval;
_Maat_table_info_t* p_table=NULL;
Maat_table_desc* p_table=NULL;
struct timespec start,end;
if(_feather->perf_on==1)
{
@@ -1234,8 +1296,8 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
{
return 0;
}
struct table_runtime* table_aux=my_scanner->table_rt[table_id];
if(table_aux->origin_rule_num==0)
struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id];
if(table_rt->origin_rule_num==0)
{
return 0;
}
@@ -1253,7 +1315,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
}
else if(region_ret>0)
{
alignment_int64_array_add(table_aux->hit_cnt, thread_num,1);
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
@@ -1271,11 +1333,11 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_aux, 0, &start, &end, thread_num);
maat_stat_table(table_rt, 0, &start, &end, thread_num);
}
else
{
maat_stat_table(table_aux, 0, NULL, NULL, thread_num);
maat_stat_table(table_rt, 0, NULL, NULL, thread_num);
}
DEC_SCANNER_REF(my_scanner,thread_num);
@@ -1297,7 +1359,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
scan_data_t ip_scan_data;
scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];
_Maat_table_info_t* p_table=NULL;
Maat_table_desc* p_table=NULL;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* my_scanner=NULL;
@@ -1317,17 +1379,17 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
{
return 0;
}
struct table_runtime* table_aux=my_scanner->table_rt[table_id];
if(table_aux->origin_rule_num==0)
struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id];
if(table_rt->origin_rule_num==0)
{
return 0;
}
if(table_aux->ip.ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4)
if(table_rt->ip.ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4)
{
return 0;
}
if(table_aux->ip.ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6)
if(table_rt->ip.ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6)
{
return 0;
}
@@ -1372,7 +1434,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
}
else if(region_ret>0)
{
alignment_int64_array_add(table_aux->hit_cnt, thread_num,1);
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
@@ -1391,11 +1453,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_aux, 0, &start, &end, thread_num);
maat_stat_table(table_rt, 0, &start, &end, thread_num);
}
else
{
maat_stat_table(table_aux, 0, NULL, NULL, thread_num);
maat_stat_table(table_rt, 0, NULL, NULL, thread_num);
}
if(compile_ret==0&&region_ret>0)
{
@@ -1420,7 +1482,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* scanner=NULL;
struct _Maat_table_info_t *p_table=NULL;
struct Maat_table_desc *p_table=NULL;
assert(thread_num<_feather->scan_thread_num);
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR);
if(p_table==NULL)
@@ -1445,8 +1507,8 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
{
return sp;
}
struct table_runtime* table_aux=scanner->table_rt[table_id];
if(table_aux->origin_rule_num==0)
struct Maat_table_runtime* table_rt=scanner->table_rt[table_id];
if(table_rt->origin_rule_num==0)
{
return 0;
}
@@ -1462,15 +1524,15 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
{
sp->do_merge=1;
}
if(table_aux->expr.expr_rule_cnt>0)
if(table_rt->expr.expr_rule_cnt>0)
{
sp->do_expr=1;
}
if(table_aux->expr.regex_rule_cnt>0)
if(table_rt->expr.regex_rule_cnt>0)
{
sp->do_regex=1;
}
alignment_int64_array_add(table_aux->stream_num,thread_num,1);
alignment_int64_array_add(table_rt->stream_num,thread_num,1);
sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num);
return sp;
}
@@ -1489,8 +1551,8 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
scan_result_t *region_result;
_compile_result_t compile_result[rule_num];//dynamic array
scan_data_t region_scan_data;
_Maat_table_info_t* p_table=NULL;
struct table_runtime* table_aux=scanner->table_rt[sp->table_id];
Maat_table_desc* p_table=NULL;
struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id];
struct timespec start,end;
if(data==NULL||data_len<=0)
{
@@ -1598,7 +1660,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
}
if(hit_region_cnt>0)
{
alignment_int64_array_add(table_aux->hit_cnt, sp->thread_num,1);
alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1);
_mid=grab_mid(mid, sp->feather,sp->thread_num, 1);
compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region,
@@ -1637,11 +1699,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
if(sp->feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_aux,data_len,&start, &end,sp->thread_num);
maat_stat_table(table_rt,data_len,&start, &end,sp->thread_num);
}
else
{
maat_stat_table(table_aux,data_len,NULL, NULL,sp->thread_num);
maat_stat_table(table_rt,data_len,NULL, NULL,sp->thread_num);
}
if(compile_ret==0&&hit_region_cnt>0)
{
@@ -1666,8 +1728,8 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para)
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct table_runtime* table_aux=scanner->table_rt[sp->table_id];
alignment_int64_array_add(table_aux->stream_num, sp->thread_num, -1);
struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id];
alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1);
if(sp->rs_stream_para!=NULL)
{
if(scanner!=NULL&&sp->version>=sp->feather->last_full_version)
@@ -1704,7 +1766,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* scanner=NULL;
sfh_instance_t * tmp_fuzzy_handle=NULL;
struct _Maat_table_info_t *p_table=NULL;
struct Maat_table_desc *p_table=NULL;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST);
if(p_table==NULL)
{
@@ -1720,7 +1782,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
{
return sp;
}
struct table_runtime* table_aux=scanner->table_rt[table_id];
struct Maat_table_runtime* table_rt=scanner->table_rt[table_id];
tmp_fuzzy_handle=SFH_instance(total_len);
if(tmp_fuzzy_handle==NULL)
{
@@ -1734,7 +1796,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
sp->total_len=total_len;
sp->fuzzy_hash_handle=tmp_fuzzy_handle;
pthread_mutex_init(&(sp->fuzzy_mutex),NULL);
alignment_int64_array_add(table_aux->stream_num,thread_num,1);
alignment_int64_array_add(table_rt->stream_num,thread_num,1);
return sp;
}
@@ -1779,8 +1841,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{
return 0;
}
struct table_runtime *table_aux=sp->feather->scanner->table_rt[sp->table_id];
GIE_handle_t* GIE_handle=table_aux->similar.gie_handle;
struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id];
GIE_handle_t* GIE_handle=table_rt->similar.gie_handle;
unsigned long long digest_len=0;
char* digest_buff=NULL;
struct _OUTER_scan_status_t* _mid=NULL;
@@ -1825,7 +1887,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
}
if(hit_region_cnt>0)
{
alignment_int64_array_add(table_aux->hit_cnt, sp->thread_num, 1);
alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1);
_mid=grab_mid(mid,sp->feather, sp->thread_num,1);
compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region,
@@ -1843,11 +1905,11 @@ fast_out:
if(sp->feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_aux, data_len, &start, &end, sp->thread_num);
maat_stat_table(table_rt, data_len, &start, &end, sp->thread_num);
}
else
{
maat_stat_table(table_aux, data_len, NULL, NULL, sp->thread_num);
maat_stat_table(table_rt, data_len, NULL, NULL, sp->thread_num);
}
if(compile_ret==0&&hit_region_cnt>0)
{
@@ -1859,8 +1921,8 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para)
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct table_runtime *table_aux=sp->feather->scanner->table_rt[sp->table_id];
alignment_int64_array_add(table_aux->stream_num, sp->thread_num,-1);
struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id];
alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1);
if(scanner!=NULL)
{
if(sp->version==sp->feather->maat_version)
@@ -1955,7 +2017,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
_compile_result_t compile_result[rule_num];
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* my_scanner=NULL;
_Maat_table_info_t* p_table=NULL;
Maat_table_desc* p_table=NULL;
struct timespec start,end;
if(_feather->perf_on==1)
{
@@ -1972,12 +2034,12 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
{
return 0;
}
struct table_runtime* table_aux=my_scanner->table_rt[table_id];
if(table_aux->origin_rule_num==0)
struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id];
if(table_rt->origin_rule_num==0)
{
return 0;
}
GIE_handle_t* gie_handle=table_aux->similar.gie_handle;
GIE_handle_t* gie_handle=table_rt->similar.gie_handle;
INC_SCANNER_REF(my_scanner,thread_num);
alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
@@ -1990,7 +2052,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
}
else if(region_ret>0)
{
alignment_int64_array_add(table_aux->hit_cnt, thread_num,1);
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
@@ -2009,11 +2071,11 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_aux,0,&start, &end,thread_num);
maat_stat_table(table_rt,0,&start, &end,thread_num);
}
else
{
maat_stat_table(table_aux,0,NULL, NULL,thread_num);
maat_stat_table(table_rt,0,NULL, NULL,thread_num);
}
if(compile_ret==0&&region_ret>0)
{

44
src/entry/Maat_command.cpp
View File

@@ -182,61 +182,51 @@ enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p)
}
int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq)
{
unsigned int offset=0;
unsigned int i=0,j=0;
size_t offset=0, len=0;
unsigned int column_seq=0, ret=0;
switch(type)
{
case TABLE_TYPE_EXPR:
offset=7;
column_seq=7;
break;
case TABLE_TYPE_IP:
offset=14;
column_seq=14;
break;
case TABLE_TYPE_COMPILE:
offset=8;
column_seq=8;
break;
case TABLE_TYPE_PLUGIN:
if(valid_column_seq<0)
{
return -1;
}
offset=(unsigned int)valid_column_seq;
column_seq=(unsigned int)valid_column_seq;
break;
case TABLE_TYPE_INTERVAL:
offset=5;
column_seq=5;
break;
case TABLE_TYPE_DIGEST:
offset=6;
column_seq=6;
break;
case TABLE_TYPE_SIMILARITY:
offset=5;
column_seq=5;
break;
case TABLE_TYPE_EXPR_PLUS:
offset=8;
column_seq=8;
break;
case TABLE_TYPE_GROUP:
offset=3;
column_seq=3;
break;
default:
assert(0);
}
for(i=0;i<strlen(line);i++)
{
if(line[i]==' '||line[i]=='\t')
{
j++;
}
if(j==offset-1)
{
break;
}
}
i++;
if(i>=strlen(line)||(line[i]!='1'&&line[i]!='0'))// 0 is also a valid value for some non-MAAT producer.
ret=get_column_pos(const char* line, int column_seq, &offset, &len);
if(ret<0||offset>=strlen(line)||(line[offset]!='1'&&line[offset]!='0'))// 0 is also a valid value for some non-MAAT producer.
{
return -1;
}
return i;
return offset;
}
int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq)
{
@@ -1589,7 +1579,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list,
{
int ret=0, table_id=0, i=0;
int rule_with_foreign_key=0;
struct _Maat_table_info_t* p_table=NULL;
struct Maat_table_desc* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
for(i=0; i<rule_num; i++)
{
@@ -2027,7 +2017,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
int i=0, j=0;
_Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int ret=0, table_id=0,success_cnt=0;
struct serial_rule_t *s_rule=NULL;
struct serial_rule_t *s_rule=NULL;
struct Maat_table_desc* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
long long server_time=0,absolute_expire_time=0;

206
src/entry/Maat_rule.cpp
View File

@@ -63,6 +63,8 @@ int is_valid_match_method(enum MAAT_MATCH_METHOD match_method)
return 0;
}
}
iconv_t maat_iconv_open(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from)
{
const char *from_s=CHARSET_STRING[from];
@@ -521,7 +523,7 @@ void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* s
}
int read_expr_table_info(const char* line, struct _Maat_table_info_t* table, MESA_htable_handle string2int_map)
int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map)
{
int j=0,ret[4]={0};
char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0};
@@ -575,13 +577,13 @@ int read_expr_table_info(const char* line, struct _Maat_table_info_t* table, MES
}
return 0;
}
_Maat_table_info_t* table_info_new(int max_thread_num)
Maat_table_desc* table_info_new(int max_thread_num)
{
struct _Maat_table_info_t*p=ALLOC(struct _Maat_table_info_t, 1);
struct Maat_table_desc*p=ALLOC(struct Maat_table_desc, 1);
p->conj_cnt=1;
return p;
}
void table_info_free(struct _Maat_table_info_t*p)
void table_info_free(struct Maat_table_desc*p)
{
free(p);
return;
@@ -600,7 +602,7 @@ int _read_integer_arrary(char* string, int *array, int size)
return i;
}
int read_plugin_table_info(const char* line, struct _Maat_table_info_t* p)
int read_plugin_table_info(const char* line, struct Maat_table_desc* p)
{
int i=0,ret=0;
@@ -659,6 +661,12 @@ int read_plugin_table_info(const char* line, struct _Maat_table_info_t* p)
assert(tmp->type==cJSON_Number);
plugin_desc->rule_tag_column=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "estimate_size");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
plugin_desc->estimate_size=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "foreign");
if(tmp!=NULL)
{
@@ -685,15 +693,15 @@ error_out:
free(copy_line);
return -1;
}
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger)
int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger)
{
FILE*fp=NULL;
char line[MAX_TABLE_LINE_SIZE];
int i=0,ret=0,table_cnt=0;
char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0};
MESA_htable_handle string2int_map=map_create();
struct _Maat_table_info_t*p=NULL;
struct _Maat_table_info_t*conj_table=NULL;
struct Maat_table_desc*p=NULL;
struct Maat_table_desc*conj_table=NULL;
map_register(string2int_map,"expr", TABLE_TYPE_EXPR);
map_register(string2int_map,"ip", TABLE_TYPE_IP);
@@ -988,7 +996,7 @@ struct _Maat_compile_inner_t * create_compile_rule(int compile_id)
}
void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule)
{
const struct _Maat_table_info_t* table=compile_rule->ref_table;
const struct Maat_table_desc* table=compile_rule->ref_table;
const struct compile_table_desc* compile_desc=&(table->compile);
struct db_compile_rule_t* db_compile_rule=compile_rule->db_c_rule;
int i=0;
@@ -1185,9 +1193,9 @@ void destroy_digest_rule(GIE_digest_t*rule)
}
struct table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_thread_num)
struct Maat_table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_thread_num)
{
struct table_runtime* p= ALLOC(struct table_runtime, 1);
struct Maat_table_runtime* p= ALLOC(struct Maat_table_runtime, 1);
p->table_type=type;
switch(type)
{
@@ -1209,7 +1217,7 @@ struct table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_threa
p->hit_cnt=alignment_int64_array_alloc(max_thread_num);
return p;
}
void table_runtime_free(struct table_runtime* p)
void table_runtime_free(struct Maat_table_runtime* p)
{
long q_cnt=0,data_size=0;
int i=0;
@@ -1244,9 +1252,9 @@ void table_runtime_free(struct table_runtime* p)
case TABLE_TYPE_PLUGIN:
dynamic_array_destroy(p->plugin.cache_lines, free);
p->plugin.cache_lines=NULL;
if(p->plugin.ex_data_hash!=NULL)
if(p->plugin.key2ex_hash!=NULL)
{
MESA_htable_destroy(p->plugin.ex_data_hash, NULL);
MESA_htable_destroy(p->plugin.key2ex_hash, NULL);
}
break;
default:
@@ -1266,8 +1274,8 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
{
int scan_thread_num=feather->scan_thread_num;
// int rs_scan_type=feather->rule_scan_type;
struct _Maat_table_info_t ** pp_table_desc=feather->p_table_info;
struct table_runtime* table_aux=NULL;
struct Maat_table_desc ** pp_table_desc=feather->p_table_info;
struct Maat_table_runtime* table_rt=NULL;
const struct expr_table_desc* expr_desc=NULL;
int i=0,j=0;
unsigned int sub_type=0;
@@ -1336,7 +1344,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
{
continue;
}
table_aux=table_runtime_new(pp_table_desc[i]->table_type, feather->scan_thread_num);
table_rt=table_runtime_new(pp_table_desc[i]->table_type, feather->scan_thread_num);
if(pp_table_desc[i]->table_type==TABLE_TYPE_EXPR||pp_table_desc[i]->table_type==TABLE_TYPE_EXPR_PLUS)
{
expr_desc=&(pp_table_desc[i]->expr);
@@ -1354,7 +1362,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
}
}
}
scanner->table_rt[i]=table_aux;
scanner->table_rt[i]=table_rt;
}
return scanner;
}
@@ -1502,7 +1510,7 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,
unsigned long long update_interval=0;
struct _region_stat_t region_counter[MAX_TABLE_NUM];
memset(region_counter, 0, sizeof(region_counter));
struct table_runtime* table_aux=NULL;
struct Maat_table_runtime* table_rt=NULL;
if(q_cnt==0)
{
return;
@@ -1558,28 +1566,28 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,
//update scanner's region cnt;
for(i=0;i<MAX_TABLE_NUM;i++)
{
table_aux=maat_scanner->table_rt[i];
if(table_aux==NULL)
table_rt=maat_scanner->table_rt[i];
if(table_rt==NULL)
{
continue;
}
switch(table_aux->table_type)
switch(table_rt->table_type)
{
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
table_aux->expr.expr_rule_cnt+=region_counter[i].expr_rule_cnt;
table_aux->expr.regex_rule_cnt+=region_counter[i].regex_rule_cnt;
assert(table_aux->expr.expr_rule_cnt>=0);
assert(table_aux->expr.regex_rule_cnt>=0);
table_rt->expr.expr_rule_cnt+=region_counter[i].expr_rule_cnt;
table_rt->expr.regex_rule_cnt+=region_counter[i].regex_rule_cnt;
assert(table_rt->expr.expr_rule_cnt>=0);
assert(table_rt->expr.regex_rule_cnt>=0);
break;
case TABLE_TYPE_IP:
table_aux->ip.ipv4_rule_cnt+=region_counter[i].ipv4_rule_cnt;
table_aux->ip.ipv6_rule_cnt+=region_counter[i].ipv6_rule_cnt;
table_rt->ip.ipv4_rule_cnt+=region_counter[i].ipv4_rule_cnt;
table_rt->ip.ipv6_rule_cnt+=region_counter[i].ipv6_rule_cnt;
break;
default:
break;
}
assert(table_aux->origin_rule_num>=0);
assert(table_rt->origin_rule_num>=0);
}
for(i=0;i<q_cnt;i++)
{
@@ -1607,7 +1615,7 @@ void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*log
{
return;
}
struct table_runtime* table_aux=maat_scanner->table_rt[table_id];
struct Maat_table_runtime* table_rt=maat_scanner->table_rt[table_id];
update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt);
for(i=0;i<q_cnt;i++)
{
@@ -1628,11 +1636,11 @@ void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*log
{
if(update_array[i]->operation==GIE_INSERT_OPT)
{
table_aux->origin_rule_num++;
table_rt->origin_rule_num++;
}
else
{
table_aux->origin_rule_num--;
table_rt->origin_rule_num--;
}
destroy_digest_rule(update_array[i]);
update_array[i]=NULL;
@@ -1877,7 +1885,7 @@ int get_district_id(_Maat_scanner_t *scanner,const char* district_str)
}
return district_id;
}
int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger)
int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger)
{
unsigned int i=0,j=0;
char* p=NULL,*saveptr=NULL,*region_string=NULL;
@@ -2141,7 +2149,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
}
return 0;
}
int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger)
int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger)
{
struct _Maat_group_inner_t* group_rule=NULL;
scan_rule_t* p_rule=NULL;
@@ -2173,7 +2181,7 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule
MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*));
return 0;
}
int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger)
int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger)
{
struct _Maat_group_inner_t* group_rule=NULL;
scan_rule_t* p_rule=NULL;
@@ -2204,12 +2212,12 @@ int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* in
MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*));
return 0;
}
int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db_digest_rule,struct _Maat_scanner_t *scanner,void* logger)
int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule_t* db_digest_rule,struct _Maat_scanner_t *scanner,void* logger)
{
struct _Maat_group_inner_t* group_rule=NULL;
GIE_digest_t* digest_rule=NULL;
struct _Maat_group_inner_t* u_para=NULL;
struct table_runtime * table_aux=scanner->table_rt[table->table_id];
struct Maat_table_runtime * table_rt=scanner->table_rt[table->table_id];
int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id);
@@ -2232,11 +2240,11 @@ int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db
,db_digest_rule->digest_string
,db_digest_rule->confidence_degree
,group_rule);
MESA_lqueue_join_tail(table_aux->similar.update_q, &digest_rule, sizeof(void*));
MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*));
scanner->gie_total_q_size++;
return 0;
}
int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id,int rule_type,struct _Maat_scanner_t *maat_scanner,void* logger)
int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int rule_type,struct _Maat_scanner_t *maat_scanner,void* logger)
{
int i=0;
unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0};
@@ -2300,7 +2308,7 @@ int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id,
}
return 0;
}
int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger)
int add_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger)
{
struct _Maat_group_inner_t* group_rule=NULL;
struct _Maat_compile_inner_t*compile_rule=NULL;
@@ -2335,7 +2343,7 @@ int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_g
return 0;
}
void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger)
void del_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger)
{
struct _Maat_compile_inner_t*compile_rule=NULL;
struct _Maat_group_inner_t* group_rule=NULL;
@@ -2370,7 +2378,7 @@ void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_
}
return;
}
int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger)
int add_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger)
{
struct _Maat_compile_inner_t *compile_rule=NULL;
struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head);
@@ -2402,7 +2410,7 @@ int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t*
return 0;
}
int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger)
int del_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger)
{
struct _Maat_compile_inner_t *compile_rule=NULL;
compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id);
@@ -2425,10 +2433,10 @@ int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t*
}
return 1;
}
void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger)
void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger)
{
struct db_group_rule_t db_group_rule;
struct table_runtime* table_aux=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
int ret=0;
ret=sscanf(table_line,"%d\t%d\t%d",&(db_group_rule.group_id)
,&(db_group_rule.compile_id)
@@ -2447,7 +2455,7 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s
//leave no trace when compatible_group_update calling
if(table->table_type==TABLE_TYPE_GROUP)
{
table_aux->origin_rule_num--;
table_rt->origin_rule_num--;
}
}
else
@@ -2466,14 +2474,14 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s
//no need to free db_group_rule,it was saved in scanner->compile_hash
if(table->table_type==TABLE_TYPE_GROUP)
{
table_aux->origin_rule_num++;
table_rt->origin_rule_num++;
}
}
}
return;
}
void compatible_group_udpate(struct _Maat_table_info_t* table,int region_id,int compile_id,int is_valid,struct _Maat_scanner_t *scanner,void* logger)
void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct _Maat_scanner_t *scanner,void* logger)
{
char virtual_group_line[256];
snprintf(virtual_group_line,sizeof(virtual_group_line),
@@ -2481,12 +2489,12 @@ void compatible_group_udpate(struct _Maat_table_info_t* table,int region_id,int
update_group_rule(table, virtual_group_line,scanner,logger);
return;
}
void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
{
struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1);
int ret=0,db_hexbin=0,rule_type=0;
const struct expr_table_desc* expr_desc=&(table->expr);
struct table_runtime* table_aux=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
switch(table->table_type)
{
case TABLE_TYPE_EXPR:
@@ -2611,7 +2619,7 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st
}
else
{
table_aux->origin_rule_num--;
table_rt->origin_rule_num--;
}
}
else
@@ -2645,7 +2653,7 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st
}
else
{
table_aux->origin_rule_num++;
table_rt->origin_rule_num++;
}
}
@@ -2653,11 +2661,11 @@ error_out:
free(maat_str_rule);
maat_str_rule=NULL;
}
void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
void update_ip_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
{
struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1);
char src_ip[40],mask_src_ip[40],dst_ip[40],mask_dst_ip[40];
struct table_runtime* table_aux=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
unsigned short i_src_port,i_sport_mask,i_dst_port,i_dport_mask;
int protocol=0,direction=0;
int ret=0,rule_type=0;
@@ -2772,7 +2780,7 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru
}
else
{
table_aux->origin_rule_num--;
table_rt->origin_rule_num--;
}
}
else
@@ -2788,7 +2796,7 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru
}
else
{
table_aux->origin_rule_num++;
table_rt->origin_rule_num++;
}
}
@@ -2797,10 +2805,10 @@ error_out:
ip_rule=NULL;
}
void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
void update_intval_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
{
struct db_intval_rule_t* intval_rule=ALLOC(struct db_intval_rule_t, 1);
struct table_runtime* table_aux=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
int ret=0;
ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id)
,&(intval_rule->group_id)
@@ -2847,7 +2855,7 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line,
}
else
{
table_aux->origin_rule_num--;
table_rt->origin_rule_num--;
}
}
@@ -2863,7 +2871,7 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line,
}
else
{
table_aux->origin_rule_num++;
table_rt->origin_rule_num++;
}
}
@@ -2872,10 +2880,10 @@ error_out:
intval_rule=NULL;
}
void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger)
void update_compile_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger)
{
struct compile_table_desc* compile_desc=&(table->compile);
struct table_runtime* table_aux=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
struct db_compile_rule_t *p_compile=ALLOC(struct db_compile_rule_t, 1);
struct _head_Maat_rule_t* p_m_rule=&(p_compile->m_rule_head);
@@ -2932,7 +2940,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line
if(p_compile->is_valid==FALSE)
{
ret=del_compile_rule(table,p_compile,scanner, logger);
table_aux->origin_rule_num--;
table_rt->origin_rule_num--;
goto no_save;
}
else
@@ -2946,7 +2954,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line
table->udpate_err_cnt++;
goto no_save;
}
table_aux->origin_rule_num++;
table_rt->origin_rule_num++;
}
return;
@@ -2958,9 +2966,9 @@ no_save:
return;
}
void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
void update_digest_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
{
struct table_runtime* table_aux=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
struct db_digest_rule_t* digest_rule=ALLOC(struct db_digest_rule_t, 1);
int ret=0;
char digest_buff[MAX_TABLE_LINE_SIZE]={'\0'};
@@ -3025,7 +3033,7 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,
}
else
{
table_aux->origin_rule_num--;
table_rt->origin_rule_num--;
}
}
@@ -3041,7 +3049,7 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,
}
else
{
table_aux->origin_rule_num++;
table_rt->origin_rule_num++;
}
}
@@ -3162,12 +3170,12 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger)
q_cnt,bury_cnt);
}
}
void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line,_Maat_scanner_t* scanner, const struct rule_tag* tags, int n_tags, void* logger)
void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_Maat_scanner_t* scanner, const struct rule_tag* tags, int n_tags, void* logger)
{
int i=0, ret=1;
unsigned int len=strlen(table_line)+1;
struct plugin_table_desc* plugin_desc=&(table->plugin);
struct table_runtime* table_aux=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
char *p=NULL;
char* copy=NULL;
@@ -3204,7 +3212,7 @@ void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line
}
}
table_aux->plugin.acc_line_num++;
table_rt->plugin.acc_line_num++;
if(plugin_desc->cb_plug_cnt>0)
{
@@ -3217,16 +3225,16 @@ void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line
{
p=ALLOC(char, len);
memcpy(p,table_line,len);
table_aux->plugin.cache_size+=len;
dynamic_array_write(table_aux->plugin.cache_lines,table_aux->plugin.cache_line_num,p);
table_aux->plugin.cache_line_num++;
table_rt->plugin.cache_size+=len;
dynamic_array_write(table_rt->plugin.cache_lines,table_rt->plugin.cache_line_num,p);
table_rt->plugin.cache_line_num++;
}
}
void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger)
{
void *tmp1=NULL,*tmp2=NULL;
MESA_htable_handle tmp_map=NULL;
struct table_runtime* table_aux=NULL;
struct Maat_table_runtime* table_rt=NULL;
int i=0;
long q_cnt;
GIE_create_para_t para;
@@ -3252,23 +3260,23 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_
,scanner);
for(i=0;i<MAX_TABLE_NUM;i++)
{
table_aux=scanner->table_rt[i];
if(table_aux==NULL)
table_rt=scanner->table_rt[i];
if(table_rt==NULL)
{
continue;
}
switch(table_aux->table_type)
switch(table_rt->table_type)
{
case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY:
q_cnt=MESA_lqueue_get_count(table_aux->similar.update_q);
q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q);
if(q_cnt==0)
{
continue;
}
if(table_aux->similar.gie_handle==NULL)
if(table_rt->similar.gie_handle==NULL)
{
if(table_aux->table_type==TABLE_TYPE_SIMILARITY)
if(table_rt->table_type==TABLE_TYPE_SIMILARITY)
{
para.ED_reexamine=1;
para.format=GIE_INPUT_FORMAT_PLAIN;
@@ -3278,10 +3286,10 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_
para.ED_reexamine=0;
para.format=GIE_INPUT_FORMAT_SFH;
}
table_aux->similar.gie_handle=GIE_create(&para);
table_rt->similar.gie_handle=GIE_create(&para);
}
digest_batch_update(table_aux->similar.gie_handle,
table_aux->similar.update_q,
digest_batch_update(table_rt->similar.gie_handle,
table_rt->similar.update_q,
logger,
scanner,
i);
@@ -3306,25 +3314,11 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_
return;
}
void clear_plugin_table_info(struct _plugin_table_info *cb_info)
{
int i=0;
void *line=NULL;
for(i=0;i<cb_info->cache_line_num;i++)
{
line=dynamic_array_read(cb_info->cache_lines,i);
free(line);
dynamic_array_write(cb_info->cache_lines,i,NULL);
}
cb_info->cache_line_num=0;
cb_info->cache_size=0;
cb_info->acc_line_num=0;
return;
}
void maat_start_cb(long long new_version,int update_type,void*u_para)
{
struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para;
struct _Maat_table_info_t* p_table=NULL;
struct Maat_table_desc* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
int i=0,j=0;
feather->new_version=new_version;
@@ -3369,14 +3363,14 @@ void maat_start_cb(long long new_version,int update_type,void*u_para)
long long scanner_rule_num(struct _Maat_scanner_t *scanner)
{
long long total=0;
struct table_runtime* table_aux=NULL;
struct Maat_table_runtime* table_rt=NULL;
int i=0;
for(i=0;i<MAX_TABLE_NUM;i++)
{
table_aux=scanner->table_rt[i];
if(table_aux!=NULL)
table_rt=scanner->table_rt[i];
if(table_rt!=NULL)
{
total+=table_aux->origin_rule_num;
total+=table_rt->origin_rule_num;
}
}
return total;
@@ -3384,7 +3378,7 @@ long long scanner_rule_num(struct _Maat_scanner_t *scanner)
void maat_finish_cb(void* u_para)
{
struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para;
struct _Maat_table_info_t* p_table=NULL;
struct Maat_table_desc* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
long expr_wait_q_cnt=0;
int i=0, j=0;
@@ -3472,7 +3466,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
int ret=-1,i=0;
int table_id=-1;
_Maat_scanner_t* scanner=NULL;
struct _Maat_table_info_t* p_table=NULL;
struct Maat_table_desc* p_table=NULL;
if(feather->update_tmp_scanner!=NULL)
{
scanner=feather->update_tmp_scanner;

48
src/entry/Maat_stat.cpp
View File

@@ -42,7 +42,7 @@ void maat_stat_init(struct _Maat_feather_t* feather)
{
int value=0;
int i=0,j=0,offset=0;
struct _Maat_table_info_t* p_table=NULL;
struct Maat_table_desc* p_table=NULL;
char conj_table_name[(MAX_TABLE_NAME_LEN+1)*MAX_CONJUNCTION_TABLE_NUM]={0};
feather->stat_handle=FS_create_handle();
@@ -84,7 +84,7 @@ void maat_stat_init(struct _Maat_feather_t* feather)
feather->fs_status_id[STATUS_CMD_LINE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_SPEED,"line_cmd/s");
feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule");
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"regex");
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"reg/v6");
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"stream");
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"IN_Bps");
if(feather->perf_on==1)
@@ -149,7 +149,7 @@ void maat_stat_init(struct _Maat_feather_t* feather)
FS_start(feather->stat_handle);
return;
}
void maat_stat_table(struct table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num)
void maat_stat_table(struct Maat_table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num)
{
alignment_int64_array_add(p->scan_cnt,thread_num,1);
alignment_int64_array_add(p->input_bytes,thread_num,scan_len);
@@ -171,8 +171,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
long long compile_rule_num=0,group_rule_num=0,plugin_cache_num=0,plugin_acc_num=0;
int i=0;
time_t now;
struct _Maat_table_info_t* p_table=NULL;
struct table_runtime* table_aux=NULL;
struct Maat_table_desc* p_table=NULL;
struct Maat_table_runtime* table_rt=NULL;
time(&now);
active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num);
@@ -208,26 +208,26 @@ void maat_stat_output(struct _Maat_feather_t* feather)
{
continue;
}
table_aux=feather->scanner->table_rt[i];
table_rt=feather->scanner->table_rt[i];
switch(p_table->table_type)
{
case TABLE_TYPE_PLUGIN:
plugin_cache_num+=table_aux->plugin.cache_line_num;
plugin_acc_num+=table_aux->plugin.acc_line_num;
plugin_cache_num+=table_rt->plugin.cache_line_num;
plugin_acc_num+=table_rt->plugin.acc_line_num;
break;
case TABLE_TYPE_GROUP:
group_rule_num+=table_aux->origin_rule_num;
group_rule_num+=table_rt->origin_rule_num;
break;
case TABLE_TYPE_COMPILE:
compile_rule_num+=table_aux->origin_rule_num;
compile_rule_num+=table_rt->origin_rule_num;
break;
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
table_regex_ipv6_num=table_aux->expr.regex_rule_cnt;
table_regex_ipv6_num=table_rt->expr.regex_rule_cnt;
total_iconv_error=p_table->expr.iconv_err_cnt;
break;
case TABLE_TYPE_IP:
table_regex_ipv6_num=table_aux->ip.ipv6_rule_cnt;
table_regex_ipv6_num=table_rt->ip.ipv6_rule_cnt;
break;
default:
break;
@@ -242,8 +242,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
FS_OP_SET,
table_aux->origin_rule_num);
total_cfg_num+=table_aux->origin_rule_num;
table_rt->origin_rule_num);
total_cfg_num+=table_rt->origin_rule_num;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
@@ -252,8 +252,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
table_regex_ipv6_num);
total_regex_num+=table_regex_ipv6_num;
table_stream_num=alignment_int64_array_sum(table_aux->stream_num,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->stream_num,feather->scan_thread_num);
table_stream_num=alignment_int64_array_sum(table_rt->stream_num,feather->scan_thread_num);
alignment_int64_array_reset(table_rt->stream_num,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
@@ -262,8 +262,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
total_stream_cnt+= table_stream_num;
table_scan_cnt=alignment_int64_array_sum(table_aux->scan_cnt,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->scan_cnt,feather->scan_thread_num);
table_scan_cnt=alignment_int64_array_sum(table_rt->scan_cnt,feather->scan_thread_num);
alignment_int64_array_reset(table_rt->scan_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
@@ -271,8 +271,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
table_scan_cnt);
total_scan_cnt+=table_scan_cnt;
table_input_bytes=alignment_int64_array_sum(table_aux->input_bytes,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->input_bytes,feather->scan_thread_num);
table_input_bytes=alignment_int64_array_sum(table_rt->input_bytes,feather->scan_thread_num);
alignment_int64_array_reset(table_rt->input_bytes,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
@@ -281,8 +281,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
total_input_bytes+=table_input_bytes;
if(feather->perf_on==1)
{
table_scan_cpu_time=alignment_int64_array_sum(table_aux->scan_cpu_time,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->scan_cpu_time,feather->scan_thread_num);
table_scan_cpu_time=alignment_int64_array_sum(table_rt->scan_cpu_time,feather->scan_thread_num);
alignment_int64_array_reset(table_rt->scan_cpu_time,feather->scan_thread_num);
table_scan_cpu_time/=1000;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
@@ -292,8 +292,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
total_cpu_time+=table_scan_cpu_time;
}
table_hit_cnt=alignment_int64_array_sum(table_aux->hit_cnt,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->hit_cnt,feather->scan_thread_num);
table_hit_cnt=alignment_int64_array_sum(table_rt->hit_cnt,feather->scan_thread_num);
alignment_int64_array_reset(table_rt->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],

24
src/entry/Maat_utils.cpp
View File

@@ -164,6 +164,30 @@ char* str_unescape(char* s)
s[j]='\0';
return s;
}
int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len)
{
const char* seps=" \t";
char* saveptr=NULL, *subtoken=NULL, *str=NULL;
char* dup_line=_maat_strdup(line);
int i=0, ret=-1;
for (str = dup_line; ; str = NULL)
{
subtoken = strtok_r(str, seps, &saveptr);
if (subtoken == NULL)
break;
if(i==column_seq-1)
{
*offset=subtoken-dup_line;
*len=strlen(subtoken);
ret=0
break;
}
i++;
}
free(dup_line);
return ret;
}
#define MAX_SYSTEM_CMD_LEN 512
int system_cmd_mkdir(const char* path)

10
src/entry/dynamic_array.cpp
View File

@@ -2,11 +2,11 @@
#include <stdlib.h>
#include <string.h>
int dynamic_array_VERSION_20141202=0;
struct dynamic_array_t* dynamic_array_create(int size,int step)
struct dynamic_array_t* dynamic_array_create(long long init_size, long long step)
{
struct dynamic_array_t* d_array=(struct dynamic_array_t*)calloc(sizeof(struct dynamic_array_t),1);
d_array->array=(void**)calloc(sizeof(void*),size);
d_array->size=size;
d_array->array=(void**)calloc(sizeof(void*),init_size);
d_array->size=init_size;
d_array->enlarge_step=step;
return d_array;
}
@@ -23,7 +23,7 @@ void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(vo
free(d_array->array);
free(d_array);
}
void* dynamic_array_read(struct dynamic_array_t* d_array,int i)
void* dynamic_array_read(struct dynamic_array_t* d_array, long long i)
{
if(i<d_array->size)
{
@@ -34,7 +34,7 @@ void* dynamic_array_read(struct dynamic_array_t* d_array,int i)
return NULL;
}
}
void dynamic_array_write(struct dynamic_array_t* d_array,int i,void* data)
void dynamic_array_write(struct dynamic_array_t* d_array, long long i, void* data)
{
int new_size=0;
if(i<d_array->size)

30
src/inc_internal/Maat_rule_internal.h
View File

@@ -123,18 +123,8 @@ struct plugin_table_ex_data_desc
Maat_plugin_EX_key2index_func_t* key2index_func;
long argl;
void *argp;
MESA_htable_handle key2ex_hash;
};
struct _plugin_table_info
{
int cb_plug_cnt;
struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE];
dynamic_array_t *cache_lines;
int cache_line_num;
int acc_line_num;
int update_type;
long cache_size;
};
struct plugin_table_desc
{
int key_column;
@@ -143,9 +133,9 @@ struct plugin_table_desc
int n_foreign;
int foreign_columns[MAX_FOREIGN_CLMN_NUM];
int cb_plug_cnt;
long long estimate_size;
struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE];
struct plugin_table_ex_data_desc ex_desc;
int acc_line_num;
};
struct expr_table_desc
{
@@ -163,7 +153,7 @@ struct ip_table_desc
int ipv6_rule_cnt;
};
struct _Maat_table_info_t
struct Maat_table_desc
{
unsigned short table_id;
unsigned short conj_cnt;
@@ -317,7 +307,7 @@ struct _Maat_compile_inner_t
dynamic_array_t *groups;
int is_valid;
int compile_id;//equal to db_c_rule->m_rule.config_id
const struct _Maat_table_info_t* ref_table;
const struct Maat_table_desc* ref_table;
int group_boundary;
int group_cnt;
MAAT_RULE_EX_DATA* ads;
@@ -390,7 +380,7 @@ struct plugin_runtime
long long cache_line_num;
long long acc_line_num;
long long cache_size;
MESA_htable_handle ex_data_hash;
MESA_htable_handle key2ex_hash;
};
struct expr_runtime
{
@@ -403,7 +393,7 @@ struct ip_runtime
long long ipv6_rule_cnt;
};
struct table_runtime
struct Maat_table_runtime
{
enum MAAT_TABLE_TYPE table_type;
long origin_rule_num;
@@ -434,7 +424,7 @@ struct _Maat_scanner_t
rule_scanner_t region;
long gie_total_q_size;
struct table_runtime* table_rt[MAX_TABLE_NUM];
struct Maat_table_runtime* table_rt[MAX_TABLE_NUM];
/*
struct similar_runtime gie_aux[MAX_TABLE_NUM];
struct plugin_runtime plugin_aux[MAX_TABLE_NUM];
@@ -506,7 +496,7 @@ struct _Maat_feather_t
int cumulative_update_off;
int stat_on;
int perf_on;
struct _Maat_table_info_t *p_table_info[MAX_TABLE_NUM];
struct Maat_table_desc *p_table_info[MAX_TABLE_NUM];
MESA_htable_handle map_tablename2id;
void* logger;
long long maat_version;
@@ -605,7 +595,7 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set);
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
void maat_start_cb(long long new_version,int update_type,void*u_para);
int maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para);
@@ -617,7 +607,7 @@ int HASH_add_by_id(MESA_htable_handle hash,int id,void*data);
int HASH_delete_by_id(MESA_htable_handle hash,int id);
void maat_read_full_config(_Maat_feather_t* _feather);
void maat_stat_init(struct _Maat_feather_t* feather);
void maat_stat_table(struct table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num);
void maat_stat_table(struct Maat_table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num);
void maat_stat_output(struct _Maat_feather_t* feather);
redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...);

2
src/inc_internal/Maat_utils.h
View File

@@ -47,4 +47,6 @@ int system_cmd_rm(const char* src_file);
int system_cmd_mv(const char* src_file,const char*dst_file);
int system_cmd_cp(const char* src_file,const char*dst_file);
char* md5_file(const char* filename, char* md5string);
int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len);

10
src/inc_internal/dynamic_array.h
View File

@@ -3,11 +3,11 @@
struct dynamic_array_t
{
void ** array;
int size;
int enlarge_step;
long long size;
long long enlarge_step;
};
struct dynamic_array_t* dynamic_array_create(int size,int step);
struct dynamic_array_t* dynamic_array_create(long long init_size, long long step);
void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(void *));
void* dynamic_array_read(struct dynamic_array_t* d_array,int i);
void dynamic_array_write(struct dynamic_array_t* d_array,int i,void* data);
void* dynamic_array_read(struct dynamic_array_t* d_array,long long i);
void dynamic_array_write(struct dynamic_array_t* d_array, long long i,void* data);
#endif //_DYNAMIC_ARRAY_H_INCLUDE_

2
test/table_info.conf
View File

@@ -28,5 +28,5 @@
9 SIM_URL similar --
10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff
11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} --
12 TEST_FOREIGN_KEY plugin {"valid":4,"foreign":"6,8","tag":3} --
12 TEST_FOREIGN_KEY plugin {"key":2,"valid":4,"foreign":[6,8],"tag":3, "estimate_size":4096} --
13 COMPILE_ALIAS compile escape --