gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

重构Maat table相关代码。

Browse Source
This commit is contained in:
zhengchao
2019-07-25 14:49:11 +06:00
parent c189b90e6d
commit 2909cb1997
10 changed files with 562 additions and 439 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/CMakeLists.txt
View File

@@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC) add_definitions(-fPIC)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp) set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
include_directories(/opt/MESA/include/MESA/) include_directories(/opt/MESA/include/MESA/)

149
src/entry/Maat_api.cpp
View File

@@ -16,9 +16,9 @@
#include "rulescan.h" #include "rulescan.h"
#include "json2iris.h" #include "json2iris.h"
struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int table_id, enum MAAT_TABLE_TYPE expect_type) struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id)
{ {
struct Maat_table_desc *p_table=NULL; struct Maat_table_desc *p_table=NULL, *p_real_table=NULL;
if(table_id>MAX_TABLE_NUM) if(table_id>MAX_TABLE_NUM)
{ {
return NULL; return NULL;
@@ -32,7 +32,17 @@ struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int tab
{ {
return NULL; return NULL;
} }
if(p_table->table_type!=expect_type) if(p_table->table_type==TABLE_TYPE_VIRTUAL)
{
p_real_table=_feather->p_table_info[p_table->virtual_table.real_table_id];
*virutal_table_id=table_id;
}
else
{
p_real_table=p_table;
*virutal_table_id=0;
}
if(p_real_table->table_type!=expect_type)
{ {
if((expect_type==TABLE_TYPE_EXPR && p_table->table_type!=TABLE_TYPE_EXPR_PLUS)|| if((expect_type==TABLE_TYPE_EXPR && p_table->table_type!=TABLE_TYPE_EXPR_PLUS)||
(expect_type==TABLE_TYPE_IP && p_table->table_type!=TABLE_TYPE_IP_PLUS)) (expect_type==TABLE_TYPE_IP && p_table->table_type!=TABLE_TYPE_IP_PLUS))
@@ -40,7 +50,7 @@ struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int tab
return NULL; return NULL;
} }
} }
return p_table; return p_real_table;
} }
inline void INC_SCANNER_REF(Maat_scanner*scanner,int thread_num) inline void INC_SCANNER_REF(Maat_scanner*scanner,int thread_num)
{ {
@@ -537,40 +547,18 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
return NULL; return NULL;
} }
_Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1); _Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1);
feather->table_cnt=read_table_description(feather->p_table_info, MAX_TABLE_NUM, table_info_path, max_thread_num, logger); feather->table_cnt=Maat_table_read_table_info(feather->p_table_info, MAX_TABLE_NUM, table_info_path, logger);
if(feather->table_cnt==0) if(feather->table_cnt==0)
{ {
free(feather); goto failed;
return NULL;
} }
feather->map_tablename2id=map_create(); feather->map_tablename2id=Maat_table_build(feather->p_table_info, MAX_TABLE_NUM,
int i=0,j=0,ret=0; feather->compile_tn, sizeof(feather->compile_tn),
for(i=0;i<MAX_TABLE_NUM;i++) feather->group_tn, sizeof(feather->group_tn),
logger);
if(feather->map_tablename2id==NULL)
{ {
if(feather->p_table_info[i]!=NULL) goto failed;
{
if(feather->p_table_info[i]->table_type==TABLE_TYPE_GROUP)
{
feather->GROUP_MODE_ON=1;
strncpy(feather->group_tn,feather->p_table_info[i]->table_name[0],sizeof(feather->group_tn));
}
if(feather->p_table_info[i]->table_type==TABLE_TYPE_COMPILE)
{
strncpy(feather->compile_tn,feather->p_table_info[i]->table_name[0],sizeof(feather->compile_tn));
}
for(j=0;j<feather->p_table_info[i]->conj_cnt;j++)
{
ret=map_register(feather->map_tablename2id,feather->p_table_info[i]->table_name[j],feather->p_table_info[i]->table_id);
if(ret<0)
{
MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module ,
"Duplicate table name %s of table id %d"
,feather->p_table_info[i]->table_name[j]
,feather->p_table_info[i]->table_id);
continue;
}
}
}
} }
feather->logger=logger; feather->logger=logger;
feather->scan_thread_num=max_thread_num; feather->scan_thread_num=max_thread_num;
@@ -594,6 +582,9 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
pthread_mutex_init(&(feather->background_update_mutex),NULL); pthread_mutex_init(&(feather->background_update_mutex),NULL);
snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path); snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path);
return feather; return feather;
failed:
free(feather);
return NULL;
} }
int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size) int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size)
{ {
@@ -1369,6 +1360,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{ {
int region_ret=0,compile_ret=0,hit_region_cnt=0; int region_ret=0,compile_ret=0,hit_region_cnt=0;
unsigned int sub_type=0; unsigned int sub_type=0;
int virtual_table_id=0;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _OUTER_scan_status_t* _mid=(struct _OUTER_scan_status_t*)(*mid); struct _OUTER_scan_status_t* _mid=(struct _OUTER_scan_status_t*)(*mid);
@@ -1392,7 +1384,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
clock_gettime(CLOCK_MONOTONIC,&start); clock_gettime(CLOCK_MONOTONIC,&start);
} }
_mid=grab_mid(mid,_feather, thread_num, 0); _mid=grab_mid(mid,_feather, thread_num, 0);
p_table=acqurie_table(_feather, table_id,TABLE_TYPE_EXPR); p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR, &virtual_table_id);
if(p_table==NULL) if(p_table==NULL)
{ {
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
@@ -1406,11 +1398,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
} }
if(expr_desc->do_charset_merge==1) if(expr_desc->do_charset_merge==1)
{ {
sub_type=make_sub_type(table_id,CHARSET_NONE,0); sub_type=make_sub_type(p_table->table_id, CHARSET_NONE,0);
} }
else else
{ {
sub_type=make_sub_type(table_id,charset,0); sub_type=make_sub_type(p_table->table_id, charset,0);
} }
alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
scan_data_t scan_data; scan_data_t scan_data;
@@ -1418,7 +1410,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
scan_data.text_data.tlen=data_len; scan_data.text_data.tlen=data_len;
scan_data.text_data.toffset=0; scan_data.text_data.toffset=0;
struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id];
if(table_rt->origin_rule_num==0) if(table_rt->origin_rule_num==0)
{ {
return 0; return 0;
@@ -1440,7 +1432,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
if(table_rt->expr.regex_rule_cnt>0) if(table_rt->expr.regex_rule_cnt>0)
{ {
scan_data.rule_type=RULETYPE_REG; scan_data.rule_type=RULETYPE_REG;
scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0); scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE,0);
region_ret=rulescan_search(my_scanner->region, thread_num, &scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt); region_ret=rulescan_search(my_scanner->region, thread_num, &scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt);
if(region_ret>0) if(region_ret>0)
{ {
@@ -1460,7 +1452,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
} }
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
struct scan_region_hit_wraper region_hit_wraper; struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, 0); scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, virtual_table_id);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
&region_hit_wraper, &region_hit_wraper,
result,compile_result,rule_num, result,compile_result,rule_num,
@@ -1518,16 +1510,15 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
_compile_result_t compile_result[rule_num]; _compile_result_t compile_result[rule_num];
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_scanner* my_scanner=NULL; struct Maat_scanner* my_scanner=NULL;
intval_scan_data.rule_type=RULETYPE_INT;
intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0);
intval_scan_data.int_data=intval;
Maat_table_desc* p_table=NULL; Maat_table_desc* p_table=NULL;
struct timespec start,end; struct timespec start,end;
if(_feather->perf_on==1) if(_feather->perf_on==1)
{ {
clock_gettime(CLOCK_MONOTONIC,&start); clock_gettime(CLOCK_MONOTONIC,&start);
} }
p_table=acqurie_table(_feather,table_id,TABLE_TYPE_INTERVAL); int virutal_table_id=0;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_INTERVAL, &virutal_table_id);
if(p_table==NULL) if(p_table==NULL)
{ {
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
@@ -1538,11 +1529,15 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
{ {
return 0; return 0;
} }
struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id];
if(table_rt->origin_rule_num==0) if(table_rt->origin_rule_num==0)
{ {
return 0; return 0;
} }
intval_scan_data.rule_type=RULETYPE_INT;
intval_scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE, 0);
intval_scan_data.int_data=intval;
alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
@@ -1563,7 +1558,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
} }
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
struct scan_region_hit_wraper region_hit_wraper; struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, region_ret, _mid->is_last_region, 0); scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, region_ret, _mid->is_last_region, virutal_table_id);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
&region_hit_wraper, &region_hit_wraper,
result,compile_result,rule_num, result,compile_result,rule_num,
@@ -1613,7 +1608,8 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
{ {
clock_gettime(CLOCK_MONOTONIC,&start); clock_gettime(CLOCK_MONOTONIC,&start);
} }
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_IP); int virtual_table_id=0;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_IP, &virtual_table_id);
if(p_table==NULL) if(p_table==NULL)
{ {
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
@@ -1624,7 +1620,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
{ {
return 0; return 0;
} }
struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id];
if(table_rt->origin_rule_num==0) if(table_rt->origin_rule_num==0)
{ {
return 0; return 0;
@@ -1641,7 +1637,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
ip_scan_data.rule_type=RULETYPE_IPv4; ip_scan_data.rule_type=RULETYPE_IPv4;
ip_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); ip_scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE, 0);
switch(addr->addrtype) switch(addr->addrtype)
{ {
case ADDR_TYPE_IPV4: case ADDR_TYPE_IPV4:
@@ -1685,7 +1681,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
} }
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
struct scan_region_hit_wraper region_hit_wraper; struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, region_ret, _mid->is_last_region, 0); scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, region_ret, _mid->is_last_region, virtual_table_id);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
&region_hit_wraper, &region_hit_wraper,
result,compile_result,rule_num, result,compile_result,rule_num,
@@ -1730,8 +1726,9 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
struct Maat_scanner* scanner=NULL; struct Maat_scanner* scanner=NULL;
struct Maat_table_desc *p_table=NULL; struct Maat_table_desc *p_table=NULL;
int virtual_table_id=0;
assert(thread_num<_feather->scan_thread_num); assert(thread_num<_feather->scan_thread_num);
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR); p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR, &virtual_table_id);
if(p_table==NULL) if(p_table==NULL)
{ {
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
@@ -1742,6 +1739,8 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
struct _stream_para_t* sp=ALLOC(struct _stream_para_t ,1); struct _stream_para_t* sp=ALLOC(struct _stream_para_t ,1);
scanner=_feather->scanner; scanner=_feather->scanner;
sp->feather=_feather; sp->feather=_feather;
sp->p_real_table=p_table;
sp->virtual_table_id=virtual_table_id;
sp->version=_feather->maat_version; sp->version=_feather->maat_version;
sp->process_offset=0; sp->process_offset=0;
sp->rs_stream_para=NULL; sp->rs_stream_para=NULL;
@@ -1749,14 +1748,13 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
{ {
return sp; return sp;
} }
struct Maat_table_runtime* table_rt=scanner->table_rt[table_id]; struct Maat_table_runtime* table_rt=scanner->table_rt[sp->p_real_table->table_id];
if(table_rt->origin_rule_num==0) if(table_rt->origin_rule_num==0)
{ {
return sp; return sp;
} }
INC_SCANNER_REF(scanner, thread_num); INC_SCANNER_REF(scanner, thread_num);
sp->table_id=table_id;
sp->thread_num=thread_num; sp->thread_num=thread_num;
sp->max_cross_size=expr_desc->cross_cache_size; sp->max_cross_size=expr_desc->cross_cache_size;
sp->caching_size=0; sp->caching_size=0;
@@ -1793,16 +1791,15 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
scan_result_t *region_result; scan_result_t *region_result;
_compile_result_t compile_result[rule_num];//dynamic array _compile_result_t compile_result[rule_num];//dynamic array
scan_data_t region_scan_data; scan_data_t region_scan_data;
Maat_table_desc* p_table=NULL; struct timespec start,end;
struct timespec start,end;
if(data==NULL||data_len<=0||scanner==NULL) if(data==NULL||data_len<=0||scanner==NULL)
{ {
return 0; return 0;
} }
struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id]; struct Maat_table_runtime* table_rt=scanner->table_rt[sp->p_real_table->table_id];
if(sp->feather->perf_on==1) if(sp->feather->perf_on==1)
{ {
clock_gettime(CLOCK_MONOTONIC,&start); clock_gettime(CLOCK_MONOTONIC, &start);
} }
_mid=grab_mid(mid, sp->feather, sp->thread_num,0); _mid=grab_mid(mid, sp->feather, sp->thread_num,0);
@@ -1810,11 +1807,10 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
{ {
return 0; return 0;
} }
p_table=sp->feather->p_table_info[sp->table_id]; //table rule num is already in Maat_stream_scan_string_start
//table rule num is already judged in Maat_stream_scan_string_start
if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) if(sp->p_real_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
{ {
sp->feather->scan_err_cnt++; sp->feather->scan_err_cnt++;
return -1; return -1;
@@ -1824,11 +1820,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
*detail_ret=0; *detail_ret=0;
if(sp->do_merge==1) if(sp->do_merge==1)
{ {
sub_type=make_sub_type(sp->table_id,CHARSET_NONE,0); sub_type=make_sub_type(sp->p_real_table->table_id, CHARSET_NONE, 0);
} }
else else
{ {
sub_type=make_sub_type(sp->table_id,charset,0); sub_type=make_sub_type(sp->p_real_table->table_id, charset, 0);
} }
if(sp->max_cross_size>0&&sp->caching_size>0) if(sp->max_cross_size>0&&sp->caching_size>0)
{ {
@@ -1880,7 +1876,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
if(sp->do_regex==1) if(sp->do_regex==1)
{ {
region_scan_data.rule_type=RULETYPE_REG; region_scan_data.rule_type=RULETYPE_REG;
region_scan_data.sub_type=make_sub_type(sp->table_id,CHARSET_NONE,0); region_scan_data.sub_type=make_sub_type(sp->p_real_table->table_id, CHARSET_NONE,0);
region_ret=rulescan_searchstream(sp->rs_stream_para, &region_scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt); region_ret=rulescan_searchstream(sp->rs_stream_para, &region_scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt);
if(region_ret<0) if(region_ret<0)
{ {
@@ -1892,7 +1888,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
hit_region_cnt+=region_ret; hit_region_cnt+=region_ret;
} }
} }
if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS) if(hit_region_cnt>0&&sp->p_real_table->table_type==TABLE_TYPE_EXPR_PLUS)
{ {
hit_region_cnt=match_district(_mid, region_result, hit_region_cnt, scanner); hit_region_cnt=match_district(_mid, region_result, hit_region_cnt, scanner);
} }
@@ -1904,7 +1900,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
} }
_mid=grab_mid(mid, sp->feather,sp->thread_num, 1); _mid=grab_mid(mid, sp->feather,sp->thread_num, 1);
struct scan_region_hit_wraper region_hit_wraper; struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, 0); scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, sp->virtual_table_id);
compile_ret=region_compile(sp->feather,_mid->inner, compile_ret=region_compile(sp->feather,_mid->inner,
&region_hit_wraper, &region_hit_wraper,
@@ -1973,7 +1969,7 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para)
struct Maat_table_runtime* table_rt=NULL; struct Maat_table_runtime* table_rt=NULL;
if(scanner!=NULL) if(scanner!=NULL)
{ {
table_rt=scanner->table_rt[sp->table_id]; table_rt=scanner->table_rt[sp->p_real_table->table_id];
alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1); alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1);
} }
@@ -2014,7 +2010,8 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
struct Maat_scanner* scanner=NULL; struct Maat_scanner* scanner=NULL;
sfh_instance_t * tmp_fuzzy_handle=NULL; sfh_instance_t * tmp_fuzzy_handle=NULL;
struct Maat_table_desc *p_table=NULL; struct Maat_table_desc *p_table=NULL;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST); int virtual_table_id=0;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST, &virtual_table_id);
if(p_table==NULL) if(p_table==NULL)
{ {
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
@@ -2023,6 +2020,8 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
struct _stream_para_t* sp=ALLOC(struct _stream_para_t, 1); struct _stream_para_t* sp=ALLOC(struct _stream_para_t, 1);
scanner=_feather->scanner; scanner=_feather->scanner;
sp->feather=_feather; sp->feather=_feather;
sp->p_real_table=p_table;
sp->virtual_table_id=virtual_table_id;
sp->version=_feather->maat_version; sp->version=_feather->maat_version;
sp->process_offset=0; sp->process_offset=0;
if(scanner==NULL) if(scanner==NULL)
@@ -2038,7 +2037,6 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
} }
INC_SCANNER_REF(scanner, thread_num); INC_SCANNER_REF(scanner, thread_num);
sp->table_id=table_id;
sp->thread_num=thread_num; sp->thread_num=thread_num;
sp->total_len=total_len; sp->total_len=total_len;
sp->fuzzy_hash_handle=tmp_fuzzy_handle; sp->fuzzy_hash_handle=tmp_fuzzy_handle;
@@ -2088,7 +2086,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{ {
return 0; return 0;
} }
struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->p_real_table->table_id];
GIE_handle_t* GIE_handle=table_rt->similar.gie_handle; GIE_handle_t* GIE_handle=table_rt->similar.gie_handle;
unsigned long long digest_len=0; unsigned long long digest_len=0;
char* digest_buff=NULL; char* digest_buff=NULL;
@@ -2140,7 +2138,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
} }
_mid=grab_mid(mid,sp->feather, sp->thread_num,1); _mid=grab_mid(mid,sp->feather, sp->thread_num,1);
struct scan_region_hit_wraper region_hit_wraper; struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_GIE(&region_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, 0); scan_region_hit_wraper_build_with_GIE(&region_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, sp->virtual_table_id);
compile_ret=region_compile(sp->feather,_mid->inner, compile_ret=region_compile(sp->feather,_mid->inner,
&region_hit_wraper, &region_hit_wraper,
result,compile_result,rule_num, result,compile_result,rule_num,
@@ -2171,7 +2169,7 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para)
{ {
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct Maat_scanner* scanner=sp->feather->scanner; struct Maat_scanner* scanner=sp->feather->scanner;
struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->p_real_table->table_id];
alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1); alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1);
if(scanner!=NULL) if(scanner!=NULL)
{ {
@@ -2274,7 +2272,8 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
{ {
clock_gettime(CLOCK_MONOTONIC,&start); clock_gettime(CLOCK_MONOTONIC,&start);
} }
p_table=acqurie_table(_feather,table_id,TABLE_TYPE_SIMILARITY); int virtual_table_id=0;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id);
if(p_table==NULL) if(p_table==NULL)
{ {
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
@@ -2285,7 +2284,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
{ {
return 0; return 0;
} }
struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id];
if(table_rt->origin_rule_num==0) if(table_rt->origin_rule_num==0)
{ {
return 0; return 0;
@@ -2306,7 +2305,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
struct scan_region_hit_wraper region_hit_wraper; struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_GIE(&region_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, 0); scan_region_hit_wraper_build_with_GIE(&region_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, virtual_table_id);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
&region_hit_wraper, &region_hit_wraper,
result,compile_result,rule_num, result,compile_result,rule_num,

361
src/entry/Maat_rule.cpp
View File

@@ -34,8 +34,6 @@
int MAAT_FRAME_VERSION_2_7_20190629=1; int MAAT_FRAME_VERSION_2_7_20190629=1;
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};
int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type) int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type)
{ {
@@ -67,8 +65,8 @@ int is_valid_match_method(enum MAAT_MATCH_METHOD match_method)
iconv_t maat_iconv_open(struct Maat_scanner* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from) iconv_t maat_iconv_open(struct Maat_scanner* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from)
{ {
const char *from_s=CHARSET_STRING[from]; const char *from_s=charset_get_name(from);
const char *to_s=CHARSET_STRING[to]; const char *to_s=charset_get_name(to);
iconv_t cd; iconv_t cd;
if(from==CHARSET_GBK&&to==CHARSET_BIG5) if(from==CHARSET_GBK&&to==CHARSET_BIG5)
{ {
@@ -523,323 +521,6 @@ void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_
} }
int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map)
{
int j=0,ret[4]={0};
char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0};
char *token=NULL,*sub_token=NULL,*saveptr;
struct expr_table_desc* p=&(table->expr);
sscanf(line,"%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id)
,table->table_name[0]
,table_type
,src_charset
,dst_charset
,merge
,&(p->cross_cache_size)
,quick_str_scan);
memset(ret,0,sizeof(ret));
ret[0]=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type));
ret[1]=map_str2int(string2int_map,str_tolower(src_charset),(int*)&(p->src_charset));
ret[2]=map_str2int(string2int_map,str_tolower(merge),&(p->do_charset_merge));
if(strlen(quick_str_scan)>0)
{
ret[3]=map_str2int(string2int_map,str_tolower(quick_str_scan),&(p->quick_expr_switch));
}
memset(quick_str_scan,0,sizeof(quick_str_scan));
for(j=0;j<4;j++)
{
if(ret[j]<0)
{
return -1;
}
}
j=0;
for (token = dst_charset; ; token= NULL)
{
sub_token= strtok_r(token,"/", &saveptr);
if (sub_token == NULL)
break;
ret[3]=map_str2int(string2int_map,str_tolower(sub_token),(int*)&(p->dst_charset[j]));
if(ret[3]>0)
{
if(p->dst_charset[j]==p->src_charset)
{
p->src_charset_in_dst=TRUE;
}
j++;
}
else
{
return -1;
}
}
return 0;
}
Maat_table_desc* table_info_new(int max_thread_num)
{
struct Maat_table_desc*p=ALLOC(struct Maat_table_desc, 1);
p->conj_cnt=1;
return p;
}
void table_info_free(struct Maat_table_desc*p)
{
free(p);
return;
}
int _read_integer_arrary(char* string, int *array, int size)
{
char *token=NULL,*sub_token=NULL,*saveptr;
int i=0;
for (token = string, i=0; i<size ; token= NULL, i++)
{
sub_token= strtok_r(token,",", &saveptr);
if (sub_token == NULL)
break;
sscanf(sub_token, "%d", array+i);
}
return i;
}
#define COLUMN_PLUGIN_DESCR_JSON 4
int read_plugin_table_description(const char* line, struct Maat_table_desc* p)
{
int i=0,ret=0;
size_t offset=0, len=0;
cJSON* json=NULL, *tmp=NULL, *array_item=NULL;
char* copy_line=NULL, *plug_info=NULL;
struct plugin_table_desc* plugin_desc=&(p->plugin);
copy_line=_maat_strdup(line);
ret=get_column_pos(copy_line, COLUMN_PLUGIN_DESCR_JSON, &offset, &len);
if(i<0)
{
goto error_out;
}
if(offset+len<strlen(copy_line))
{
copy_line[offset+len+1]='\0';
}
plug_info=copy_line+offset;
if(NULL==strchr(plug_info,'{'))//For old version compatible.
{
ret=sscanf(plug_info, "%d", &(plugin_desc->valid_flag_column));
if(ret==0||ret==EOF)
{
plugin_desc->valid_flag_column=-1;
}
free(copy_line);
return 0;
}
json=cJSON_Parse(plug_info);
if(!json)
{
goto error_out;
}
tmp=cJSON_GetObjectItem(json, "key");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
plugin_desc->key_column=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "valid");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
plugin_desc->valid_flag_column=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "tag");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
plugin_desc->rule_tag_column=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "estimate_size");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
plugin_desc->estimate_size=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "foreign");
if(tmp!=NULL)
{
if(tmp->type==cJSON_String)
{
plugin_desc->n_foreign=_read_integer_arrary(tmp->valuestring, plugin_desc->foreign_columns, MAX_FOREIGN_CLMN_NUM);
}
else if(tmp->type==cJSON_Array)
{
plugin_desc->n_foreign= cJSON_GetArraySize(tmp);
for(i=0;i<plugin_desc->n_foreign; i++)
{
array_item=cJSON_GetArrayItem(tmp, i);
assert(array_item->type==cJSON_Number);
plugin_desc->foreign_columns[i]=array_item->valueint;
}
}
}
cJSON_Delete(json);
free(copy_line);
return 0;
error_out:
free(copy_line);
return -1;
}
int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger)
{
FILE*fp=NULL;
char line[MAX_TABLE_LINE_SIZE];
int i=0,ret=0,table_cnt=0;
char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0};
MESA_htable_handle string2int_map=NULL;;
struct Maat_table_desc*p=NULL;
struct Maat_table_desc*conj_table=NULL;
fp=fopen(table_info_path,"r");
if(fp==NULL)
{
fprintf(stderr,"Maat read table info %s error.\n",table_info_path);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s failed: %s.\n", table_info_path, strerror(errno));
return 0;
}
string2int_map=map_create();
map_register(string2int_map,"expr", TABLE_TYPE_EXPR);
map_register(string2int_map,"ip", TABLE_TYPE_IP);
map_register(string2int_map,"ip_plus", TABLE_TYPE_IP_PLUS);
map_register(string2int_map,"compile", TABLE_TYPE_COMPILE);
map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN);
map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL);
map_register(string2int_map,"digest", TABLE_TYPE_DIGEST);
map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS);
map_register(string2int_map,"group", TABLE_TYPE_GROUP);
map_register(string2int_map,"similar", TABLE_TYPE_SIMILARITY);
map_register(string2int_map,"quickoff",0);
map_register(string2int_map,"quickon",1);
map_register(string2int_map,"escape",USER_REGION_ENCODE_ESCAPE);
// map_register(string2int_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet
for(i=0;i<MAX_CHARSET_NUM;i++)
{
if(strlen(CHARSET_STRING[i])>0)
{
map_register(string2int_map,CHARSET_STRING[i], i);
}
else
{
break;
}
}
map_register(string2int_map,"yes", 1);
map_register(string2int_map,"no", 0);
i=0;
while(NULL!=fgets(line,sizeof(line),fp))
{
i++;
if(line[0]=='#'||line[0]==' '||line[0]=='\t'||strlen(line)<4)
{
continue;
}
p=table_info_new(max_thread_num);
ret=sscanf(line,"%d\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id)
,p->table_name[0]
,table_type_str
,not_care);
if(ret<3)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error: not enough column.",table_info_path,i);
continue;
}
ret=map_str2int(string2int_map,str_tolower(table_type_str),(int*)&(p->table_type));
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:invalid table type.",table_info_path,i);
goto invalid_table;
}
switch(p->table_type)
{
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
ret=read_expr_table_info(line, p, string2int_map);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal column.",table_info_path,i);
goto invalid_table;
}
break;
case TABLE_TYPE_PLUGIN:
ret=read_plugin_table_description(line, p);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal plugin info.",table_info_path,i);
goto invalid_table;
}
break;
case TABLE_TYPE_COMPILE:
ret=sscanf(not_care,"%[a-z0-9]",tmp_str);
if(ret>0)
{
ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding));
}
if(ret!=1)
{
p->compile.user_region_encoding=USER_REGION_ENCODE_NONE;
}
default:
break;
}
if(p->table_id>=num)
{
fprintf(stderr,"Maat read table info %s:%d error: table id %uh > %d.\n",table_info_path,i,p->table_id,num);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,num);
goto invalid_table;
}
if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table;
{
conj_table=p_table_info[p->table_id];
if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Maat read table info %s line %d error:reach tableid %d conjunction upper limit."
,table_info_path,i,p->table_id);
goto invalid_table;
}
memcpy(conj_table->table_name[conj_table->conj_cnt],p->table_name[0],MAX_TABLE_NAME_LEN);
conj_table->conj_cnt++;
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
"Maat read table info %s:%d:conjunction %s with %s (id=%d,total=%d)."
,table_info_path,i,p->table_name[0]
,conj_table->table_name[0],conj_table->table_id,conj_table->conj_cnt);
//use goto to free the conjunctioned table_info
goto invalid_table;
}
p_table_info[p->table_id]=p;
table_cnt++;
continue;
invalid_table:
table_info_free(p);
p=NULL;
}
fclose(fp);
map_destroy(string2int_map);
return table_cnt;
}
struct Maat_group_inner* create_group_rule(int group_id, int table_id, struct Maat_scanner *scanner) struct Maat_group_inner* create_group_rule(int group_id, int table_id, struct Maat_scanner *scanner)
{ {
int ret=0; int ret=0;
@@ -2222,10 +1903,11 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st
if(ret<0) if(ret<0)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"Table %s region cfg %d charset convert from %s to %s failed.",table->table_name "Table %s region cfg %d charset convert from %s to %s failed.",
,db_rule->region_id table->table_name,
,CHARSET_STRING[expr_desc->src_charset] db_rule->region_id,
,CHARSET_STRING[dst_charset]); charset_get_name(expr_desc->src_charset),
charset_get_name(dst_charset));
free(region_string); free(region_string);
op_expr->convert_failed++; op_expr->convert_failed++;
expr_desc->iconv_err_cnt++; expr_desc->iconv_err_cnt++;
@@ -2736,7 +2418,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru
return; return;
} }
void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger)
{ {
struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1);
int ret=0,db_hexbin=0,rule_type=0; int ret=0,db_hexbin=0,rule_type=0;
@@ -3024,7 +2706,7 @@ int ip_format2range(int ip_type, enum MAAT_IP_FORMAT format, const char* ip1, co
} }
return 0; return 0;
} }
void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger, int group_mode_on) void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger)
{ {
struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1);
char src_ip1[40]={0}, src_ip2[40]={0}, dst_ip1[40]={0}, dst_ip2[40]={0}; char src_ip1[40]={0}, src_ip2[40]={0}, dst_ip1[40]={0}, dst_ip2[40]={0};
@@ -3233,7 +2915,7 @@ error_out:
ip_rule=NULL; ip_rule=NULL;
} }
void update_intval_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) void update_intval_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger)
{ {
struct db_intval_rule* intval_rule=ALLOC(struct db_intval_rule, 1); struct db_intval_rule* intval_rule=ALLOC(struct db_intval_rule, 1);
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
@@ -3385,7 +3067,7 @@ error_out:
return; return;
} }
void update_digest_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) void update_digest_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger)
{ {
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
struct db_digest_rule* digest_rule=ALLOC(struct db_digest_rule, 1); struct db_digest_rule* digest_rule=ALLOC(struct db_digest_rule, 1);
@@ -4008,18 +3690,18 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
{ {
case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS: case TABLE_TYPE_EXPR_PLUS:
update_expr_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); update_expr_rule(feather->p_table_info[table_id], line, scanner, feather->logger);
break; break;
case TABLE_TYPE_IP: case TABLE_TYPE_IP:
case TABLE_TYPE_IP_PLUS: case TABLE_TYPE_IP_PLUS:
update_ip_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); update_ip_rule(feather->p_table_info[table_id], line, scanner, feather->logger);
break; break;
case TABLE_TYPE_INTERVAL: case TABLE_TYPE_INTERVAL:
update_intval_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); update_intval_rule(feather->p_table_info[table_id], line, scanner,feather->logger);
break; break;
case TABLE_TYPE_DIGEST: case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY: case TABLE_TYPE_SIMILARITY:
update_digest_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); update_digest_rule(feather->p_table_info[table_id], line, scanner,feather->logger);
break; break;
case TABLE_TYPE_COMPILE: case TABLE_TYPE_COMPILE:
update_compile_rule(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); update_compile_rule(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
@@ -4207,16 +3889,8 @@ void *thread_rule_monitor(void *arg)
garbage_bury(feather->garbage_q,0,feather->logger); garbage_bury(feather->garbage_q,0,feather->logger);
assert(0==MESA_lqueue_get_count(feather->garbage_q)); assert(0==MESA_lqueue_get_count(feather->garbage_q));
MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL); MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL);
int i=0; Maat_table_clear(feather->p_table_info, MAX_TABLE_NUM);
for(i=0;i<MAX_TABLE_NUM;i++)
{
if(feather->p_table_info[i]==NULL)
{
continue;
}
table_info_free(feather->p_table_info[i]);
feather->p_table_info[i]=NULL;
}
alignment_int64_array_free(feather->thread_call_cnt); alignment_int64_array_free(feather->thread_call_cnt);
alignment_int64_array_free(feather->inner_mid_cnt); alignment_int64_array_free(feather->inner_mid_cnt);
alignment_int64_array_free(feather->outer_mid_cnt); alignment_int64_array_free(feather->outer_mid_cnt);
@@ -4235,6 +3909,7 @@ void *thread_rule_monitor(void *arg)
feather->mr_ctx.write_ctx=NULL; feather->mr_ctx.write_ctx=NULL;
} }
} }
int i=0;
for(i=0; i<feather->n_tags; i++) for(i=0; i<feather->n_tags; i++)
{ {
free(feather->accept_tags[i].tag_name); free(feather->accept_tags[i].tag_name);

430
src/entry/Maat_table.cpp Normal file
View File

@@ -0,0 +1,430 @@
#include "Maat_table.h"
#include "map_str2int.h"
#include "Maat_utils.h"
#include "cJSON.h"
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <MESA/MESA_handle_logger.h>
int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map)
{
int j=0,ret[4]={0};
char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0};
char *token=NULL,*sub_token=NULL,*saveptr;
struct expr_table_desc* p=&(table->expr);
sscanf(line,"%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id)
,table->table_name[0]
,table_type
,src_charset
,dst_charset
,merge
,&(p->cross_cache_size)
,quick_str_scan);
memset(ret,0,sizeof(ret));
ret[0]=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type));
ret[1]=map_str2int(string2int_map,str_tolower(src_charset),(int*)&(p->src_charset));
ret[2]=map_str2int(string2int_map,str_tolower(merge),&(p->do_charset_merge));
if(strlen(quick_str_scan)>0)
{
ret[3]=map_str2int(string2int_map,str_tolower(quick_str_scan),&(p->quick_expr_switch));
}
memset(quick_str_scan,0,sizeof(quick_str_scan));
for(j=0;j<4;j++)
{
if(ret[j]<0)
{
return -1;
}
}
j=0;
for (token = dst_charset; ; token= NULL)
{
sub_token= strtok_r(token,"/", &saveptr);
if (sub_token == NULL)
break;
ret[3]=map_str2int(string2int_map,str_tolower(sub_token),(int*)&(p->dst_charset[j]));
if(ret[3]>0)
{
if(p->dst_charset[j]==p->src_charset)
{
p->src_charset_in_dst=TRUE;
}
j++;
}
else
{
return -1;
}
}
return 0;
}
int read_virtual_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map)
{
int ret=0;
char table_type[16];
ret=sscanf(line, "%d\t%s\t%s\t%s", &(table->table_id),
table->table_name[0],
table_type,
table->virtual_table.real_table_name);
if(ret!=4)
{
return -1;
}
ret=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type));
if(ret<0)
{
return -1;
}
return 0;
}
Maat_table_desc* table_info_new(void)
{
struct Maat_table_desc*p=ALLOC(struct Maat_table_desc, 1);
p->conj_cnt=1;
return p;
}
void table_info_free(struct Maat_table_desc*p)
{
free(p);
return;
}
int _read_integer_arrary(char* string, int *array, int size)
{
char *token=NULL,*sub_token=NULL,*saveptr;
int i=0;
for (token = string, i=0; i<size ; token= NULL, i++)
{
sub_token= strtok_r(token,",", &saveptr);
if (sub_token == NULL)
break;
sscanf(sub_token, "%d", array+i);
}
return i;
}
#define COLUMN_PLUGIN_DESCR_JSON 4
int read_plugin_table_description(const char* line, struct Maat_table_desc* p)
{
int i=0,ret=0;
size_t offset=0, len=0;
cJSON* json=NULL, *tmp=NULL, *array_item=NULL;
char* copy_line=NULL, *plug_info=NULL;
struct plugin_table_desc* plugin_desc=&(p->plugin);
copy_line=_maat_strdup(line);
ret=get_column_pos(copy_line, COLUMN_PLUGIN_DESCR_JSON, &offset, &len);
if(i<0)
{
goto error_out;
}
if(offset+len<strlen(copy_line))
{
copy_line[offset+len+1]='\0';
}
plug_info=copy_line+offset;
if(NULL==strchr(plug_info,'{'))//For old version compatible.
{
ret=sscanf(plug_info, "%d", &(plugin_desc->valid_flag_column));
if(ret==0||ret==EOF)
{
plugin_desc->valid_flag_column=-1;
}
free(copy_line);
return 0;
}
json=cJSON_Parse(plug_info);
if(!json)
{
goto error_out;
}
tmp=cJSON_GetObjectItem(json, "key");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
plugin_desc->key_column=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "valid");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
plugin_desc->valid_flag_column=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "tag");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
plugin_desc->rule_tag_column=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "estimate_size");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
plugin_desc->estimate_size=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "foreign");
if(tmp!=NULL)
{
if(tmp->type==cJSON_String)
{
plugin_desc->n_foreign=_read_integer_arrary(tmp->valuestring, plugin_desc->foreign_columns, MAX_FOREIGN_CLMN_NUM);
}
else if(tmp->type==cJSON_Array)
{
plugin_desc->n_foreign= cJSON_GetArraySize(tmp);
for(i=0;i<plugin_desc->n_foreign; i++)
{
array_item=cJSON_GetArrayItem(tmp, i);
assert(array_item->type==cJSON_Number);
plugin_desc->foreign_columns[i]=array_item->valueint;
}
}
}
cJSON_Delete(json);
free(copy_line);
return 0;
error_out:
free(copy_line);
return -1;
}
int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_table, const char* table_info_path, void* logger)
{
FILE*fp=NULL;
char line[MAX_TABLE_LINE_SIZE];
int i=0,ret=0,table_cnt=0;
char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0};
MESA_htable_handle string2int_map=NULL;;
struct Maat_table_desc*p=NULL;
struct Maat_table_desc*conj_table=NULL;
fp=fopen(table_info_path,"r");
if(fp==NULL)
{
fprintf(stderr,"Maat read table info %s error.\n",table_info_path);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s failed: %s.\n", table_info_path, strerror(errno));
return 0;
}
string2int_map=map_create();
map_register(string2int_map,"expr", TABLE_TYPE_EXPR);
map_register(string2int_map,"ip", TABLE_TYPE_IP);
map_register(string2int_map,"ip_plus", TABLE_TYPE_IP_PLUS);
map_register(string2int_map,"compile", TABLE_TYPE_COMPILE);
map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN);
map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL);
map_register(string2int_map,"digest", TABLE_TYPE_DIGEST);
map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS);
map_register(string2int_map,"group", TABLE_TYPE_GROUP);
map_register(string2int_map,"similar", TABLE_TYPE_SIMILARITY);
map_register(string2int_map,"virtual", TABLE_TYPE_VIRTUAL);
map_register(string2int_map,"quickoff", 0);
map_register(string2int_map,"quickon", 1);
map_register(string2int_map,"escape", USER_REGION_ENCODE_ESCAPE);
// map_register(string2int_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet
const char** charset_name_list=charset_get_all_name();
for(i=0;i<MAX_CHARSET_NUM;i++)
{
if(strlen(charset_name_list[i])>0)
{
map_register(string2int_map, charset_name_list[i], i);
}
else
{
break;
}
}
map_register(string2int_map,"yes", 1);
map_register(string2int_map,"no", 0);
i=0;
while(NULL!=fgets(line,sizeof(line),fp))
{
i++;
if(line[0]=='#'||line[0]==' '||line[0]=='\t'||strlen(line)<4)
{
continue;
}
p=table_info_new();
ret=sscanf(line,"%d\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id)
,p->table_name[0]
,table_type_str
,not_care);
if(ret<3)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error: not enough column.",table_info_path,i);
continue;
}
ret=map_str2int(string2int_map,str_tolower(table_type_str),(int*)&(p->table_type));
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:invalid table type.",table_info_path,i);
goto invalid_table;
}
switch(p->table_type)
{
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
ret=read_expr_table_info(line, p, string2int_map);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal column.",table_info_path,i);
goto invalid_table;
}
break;
case TABLE_TYPE_PLUGIN:
ret=read_plugin_table_description(line, p);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal plugin info.",table_info_path,i);
goto invalid_table;
}
break;
case TABLE_TYPE_VIRTUAL:
ret=read_virtual_table_info(line, p, string2int_map);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal virtual info.\n",table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal virtual info.",table_info_path,i);
goto invalid_table;
}
break;
case TABLE_TYPE_COMPILE:
ret=sscanf(not_care,"%[a-z0-9]",tmp_str);
if(ret>0)
{
ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding));
}
if(ret!=1)
{
p->compile.user_region_encoding=USER_REGION_ENCODE_NONE;
}
default:
break;
}
if((unsigned int)p->table_id>=n_table)
{
fprintf(stderr,"Maat read table info %s:%d error: table id %uh > %zu.\n",table_info_path,i,p->table_id,n_table);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,n_table);
goto invalid_table;
}
if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table;
{
conj_table=p_table_info[p->table_id];
if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Maat read table info %s line %d error:reach tableid %d conjunction upper limit."
,table_info_path,i,p->table_id);
goto invalid_table;
}
memcpy(conj_table->table_name[conj_table->conj_cnt],p->table_name[0],MAX_TABLE_NAME_LEN);
conj_table->conj_cnt++;
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
"Maat read table info %s:%d:conjunction %s with %s (id=%d,total=%d)."
,table_info_path,i,p->table_name[0]
,conj_table->table_name[0],conj_table->table_id,conj_table->conj_cnt);
//use goto to free the conjunctioned table_info
goto invalid_table;
}
p_table_info[p->table_id]=p;
table_cnt++;
continue;
invalid_table:
table_info_free(p);
p=NULL;
}
fclose(fp);
map_destroy(string2int_map);
return table_cnt;
}
MESA_htable_handle Maat_table_build(struct Maat_table_desc** p_table_info, size_t n_table, char* compile_tn, size_t n_ctn, char* group_tn, size_t n_gtn, void* logger)
{
MESA_htable_handle map_tablename2id=map_create();
size_t i=0;
int j=0, ret=0;
for(i=0;i<n_table;i++)
{
if(p_table_info[i]==NULL)
{
continue;
}
switch(p_table_info[i]->table_type)
{
case TABLE_TYPE_GROUP:
strncpy(group_tn, p_table_info[i]->table_name[0], n_gtn);
break;
case TABLE_TYPE_COMPILE:
strncpy(compile_tn, p_table_info[i]->table_name[0], n_ctn);
break;
case TABLE_TYPE_VIRTUAL:
ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id));
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Undefined real table %s, virtual table %s of table id %d.",
p_table_info[i]->virtual_table.real_table_name,
p_table_info[i]->table_name[j],
p_table_info[i]->table_id);
goto failed;
}
break;
default:
break;
}
for(j=0; j<p_table_info[i]->conj_cnt; j++)
{
ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Duplicate table %s of table id %d",
p_table_info[i]->table_name[j],
p_table_info[i]->table_id);
continue;
}
}
}
return map_tablename2id;
failed:
map_destroy(map_tablename2id);
return NULL;
}
void Maat_table_clear(struct Maat_table_desc** p_table_info, size_t n_table)
{
size_t i=0;
for(i=0;i<n_table;i++)
{
if(p_table_info[i]==NULL)
{
continue;
}
table_info_free(p_table_info[i]);
p_table_info[i]=NULL;
}
return;
}

10
src/entry/Maat_utils.cpp
View File

@@ -241,4 +241,14 @@ char* md5_file(const char* filename, char* md5string)
free(file_buff); free(file_buff);
return md5string; return md5string;
} }
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};
const char** charset_get_all_name(void)
{
return CHARSET_STRING;
}
const char* charset_get_name(enum MAAT_CHARSET charset)
{
return CHARSET_STRING[charset];
}

2
src/entry/json2iris.cpp
View File

@@ -12,7 +12,7 @@
#include "cJSON.h" #include "cJSON.h"
#include "hiredis.h" #include "hiredis.h"
#include "map_str2int.h" #include "map_str2int.h"
#include "Maat_table_description.h" #include "Maat_table.h"
#include "Maat_rule_internal.h" #include "Maat_rule_internal.h"
#include "Maat_utils.h" #include "Maat_utils.h"

12
src/inc_internal/Maat_limits.h Normal file
View File

@@ -0,0 +1,12 @@
#define MAX_TABLE_NUM 256
#define MAX_TABLE_LINE_SIZE (1024*16)
#define MAX_EXPR_KEYLEN 1024
#define MAX_DISTRICT_LEN 64
#define MAX_SCANNER_HIT_NUM 64
#define MAX_GROUP_CACHE 128
#define MAX_FAILED_NUM 128
#define MAX_MAAT_STAT_NUM 64

24
src/inc_internal/Maat_rule_internal.h
View File

@@ -1,9 +1,9 @@
#ifndef H_MAAT_RULE_INTERNAL_H_INCLUDE #pragma once
#define H_MAAT_RULE_INTERNAL_H_INCLUDE
#include "Maat_rule.h" #include "Maat_rule.h"
#include "Maat_command.h" #include "Maat_command.h"
#include "Maat_table_description.h" #include "Maat_limits.h"
#include "Maat_table.h"
#include <MESA/MESA_htable.h> #include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h> #include <MESA/MESA_list_queue.h>
@@ -27,17 +27,7 @@ extern const char *maat_module;
#define mr_group_id_var "SEQUENCE_GROUP" #define mr_group_id_var "SEQUENCE_GROUP"
#define MAX_TABLE_NUM 256
#define MAX_TABLE_LINE_SIZE (1024*16)
#define MAX_EXPR_KEYLEN 1024
#define MAX_DISTRICT_LEN 64
#define MAX_SCANNER_HIT_NUM 64
#define MAX_GROUP_CACHE 128
#define MAX_FAILED_NUM 128
#define MAX_MAAT_STAT_NUM 64
typedef void* rule_scanner_t; typedef void* rule_scanner_t;
@@ -210,11 +200,12 @@ struct iconv_handle_t
struct _stream_para_t struct _stream_para_t
{ {
struct _Maat_feather_t* feather; struct _Maat_feather_t* feather;
const struct Maat_table_desc* p_real_table;
int virtual_table_id;
int version; int version;
int thread_num; int thread_num;
int max_cross_size; int max_cross_size;
int caching_size; int caching_size;
unsigned short table_id;
char do_merge; char do_merge;
char do_expr:4; char do_expr:4;
char do_regex:4; char do_regex:4;
@@ -355,7 +346,6 @@ struct _Maat_feather_t
MESA_lqueue_head garbage_q; MESA_lqueue_head garbage_q;
int table_cnt; int table_cnt;
int DEFERRED_LOAD_ON; int DEFERRED_LOAD_ON;
int GROUP_MODE_ON;
int REDIS_MODE_ON; int REDIS_MODE_ON;
enum data_source input_mode; enum data_source input_mode;
union union
@@ -471,7 +461,6 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(struct Maat_compile_group_relation* compile_rule, struct bool_expr* a_set, unsigned char *has_not); void make_group_set(struct Maat_compile_group_relation* compile_rule, struct bool_expr* a_set, unsigned char *has_not);
int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
void maat_start_cb(long long new_version,int update_type,void*u_para); void maat_start_cb(long long new_version,int update_type,void*u_para);
int maat_update_cb(const char* table_name,const char* line,void *u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para); void maat_finish_cb(void* u_para);
@@ -517,6 +506,3 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
_Maat_feather_t* feather); _Maat_feather_t* feather);
#endif

8
src/inc_internal/Maat_table_description.h → src/inc_internal/Maat_table.h
View File

@@ -1,5 +1,8 @@
#pragma once #pragma once
#include <MESA/MESA_htable.h>
#include "Maat_rule.h" #include "Maat_rule.h"
#include "Maat_limits.h"
#define MAX_COMPILE_EX_DATA_NUM 2 #define MAX_COMPILE_EX_DATA_NUM 2
#define MAX_FOREIGN_CLMN_NUM 8 #define MAX_FOREIGN_CLMN_NUM 8
#define MAX_PLUGIN_PER_TABLE 32 #define MAX_PLUGIN_PER_TABLE 32
@@ -7,6 +10,7 @@
#define MAX_CONJUNCTION_TABLE_NUM 8 #define MAX_CONJUNCTION_TABLE_NUM 8
#define MAX_TABLE_NAME_LEN 256 #define MAX_TABLE_NAME_LEN 256
enum USER_REGION_ENCODE enum USER_REGION_ENCODE
{ {
USER_REGION_ENCODE_NONE=0, USER_REGION_ENCODE_NONE=0,
@@ -93,6 +97,7 @@ struct ip_table_desc
struct virtual_table_desc struct virtual_table_desc
{ {
int real_table_id; int real_table_id;
char real_table_name[MAX_TABLE_NAME_LEN];
}; };
struct Maat_table_desc struct Maat_table_desc
{ {
@@ -115,4 +120,7 @@ struct Maat_table_desc
unsigned long long unmatch_tag_cnt; unsigned long long unmatch_tag_cnt;
int stat_line_id; int stat_line_id;
}; };
int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_table, const char* table_info_path, void* logger);
MESA_htable_handle Maat_table_build(struct Maat_table_desc** p_table_info, size_t n_table, char* compile_tn, size_t n_ctn, char* group_tn, size_t n_gtn, void* logger);
void Maat_table_clear(struct Maat_table_desc** p_table_info, size_t n_table);

3
src/inc_internal/Maat_utils.h
View File

@@ -1,4 +1,5 @@
#pragma once #pragma once
#include "Maat_rule.h"
#include <arpa/inet.h> #include <arpa/inet.h>
#include <unistd.h> #include <unistd.h>
#include <sys/syscall.h> #include <sys/syscall.h>
@@ -69,5 +70,7 @@ int system_cmd_mv(const char* src_file,const char*dst_file);
int system_cmd_cp(const char* src_file,const char*dst_file); int system_cmd_cp(const char* src_file,const char*dst_file);
char* md5_file(const char* filename, char* md5string); char* md5_file(const char* filename, char* md5string);
int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len); int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len);
const char** charset_get_all_name(void);
const char* charset_get_name(enum MAAT_CHARSET charset);