gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add EXPR PLUS region table and Maat_set_scan_status function to indicate the district.

Browse Source
This commit is contained in:
zhengchao
2016-02-11 13:57:39 +08:00
parent 2a9bbc3c81
commit c2509da065
10 changed files with 611 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
inc/Maat_rule.h
View File

@@ -124,7 +124,8 @@ Maat_feather_t Maat_summon_feather_json(int max_thread_num,
Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger);
int Maat_initiate_feather(Maat_feather_t feather);
enum MAAT_INIT_OPT{
enum MAAT_INIT_OPT
{
MAAT_OPT_SCANDIR_INTERVAL_MS=1, //VALUE is interger,SIZE=sizeof(int). DEFAULT:1,000 milliseconds.
MAAT_OPT_EFFECT_INVERVAL_MS, //VALUE is interger,SIZE=sizeof(int). DEFAULT:60,000 milliseconds.
MAAT_OPT_FULL_CFG_DIR, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default.
@@ -149,7 +150,12 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
Maat_finish_callback_t *finish,//u_para
void* u_para);
enum MAAT_SCAN_OPT
{
MAAT_SET_SCAN_DISTRICT=1 //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default.
};
//return 0 if success, return -1 when failed;
int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT,const void* value,int size);
//Return hit rule number, return -1 when error occurs,return -2 when hit current region
//mid MUST set NULL before fist call

249
src/entry/Maat_api.cpp
View File

@@ -26,10 +26,18 @@ struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int t
return NULL;
}
p_table=_feather->p_table_info[table_id];
if(p_table==NULL||p_table->table_type!=expect_type)
if(p_table==NULL)
{
return NULL;
}
if(p_table->table_type!=expect_type)
{
if(expect_type!=TABLE_TYPE_EXPR||
p_table->table_type!=TABLE_TYPE_EXPR_PLUS)
{
return NULL;
}
}
return p_table;
}
inline void INC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num)
@@ -90,14 +98,14 @@ int pickup_hit_region_from_compile(universal_bool_expr_t *compile_hit,const unsi
}
return k;
}
int region_compile(struct _scan_status_t *_mid,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size)
int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size)
{
int scan_ret=0,result_cnt=0;
int ret=0,i=0,j=0;
int r_in_c_cnt=0;
void* expr_compiler=_mid->feather->scanner->expr_compiler;
int shortcut_avilable_cnt=0;
void* bool_matcher=feather->scanner->expr_compiler;
struct _Maat_group_rule_t* group_rule=NULL;
struct _Maat_compile_rule_t* array_mi_rule[MAX_SCANNER_HIT_NUM];
struct _Maat_compile_rule_t* _mi_rule=NULL;
@@ -130,7 +138,7 @@ int region_compile(struct _scan_status_t *_mid,void* region_hit,int region_type_
}
else
{
scan_ret=boolexpr_match(expr_compiler,_mid->thread_num,
scan_ret=boolexpr_match(bool_matcher,_mid->thread_num,
_mid->hitted_group_id,_mid->hit_group_cnt,
(void **)array_mi_rule, MAX_SCANNER_HIT_NUM);
}
@@ -161,14 +169,14 @@ int region_compile(struct _scan_status_t *_mid,void* region_hit,int region_type_
}
if(result_cnt>0)
{
_mid->feather->hit_cnt++;
feather->hit_cnt++;
}
return result_cnt;
}
int exprid2region_id(struct _Maat_group_rule_t* group_rule,int expr_id)
int exprid2region_id(struct _Maat_group_rule_t* group_rule,int expr_id,int* district_id)
{
int i=0,region_id=-1;;
int i=0,region_id=-1;
struct _Maat_region_rule_t* region_rule=NULL;
assert(group_rule->group_id>=0);
pthread_mutex_lock(&(group_rule->mutex));
@@ -182,11 +190,34 @@ int exprid2region_id(struct _Maat_group_rule_t* group_rule,int expr_id)
if(region_rule->expr_id==expr_id)
{
region_id=region_rule->region_id;
*district_id=region_rule->district_id;
}
}
pthread_mutex_unlock(&(group_rule->mutex));
return region_id;
}
int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num)
{
struct _Maat_group_rule_t* group_rule=NULL;
int i=0;
int district_id=-1,region_id=-1;
int ret_region_num=region_hit_num;
while(i<ret_region_num)
{
group_rule=(struct _Maat_group_rule_t*)(region_hit[i].tag);
region_id=exprid2region_id(group_rule, region_hit[i].expr_id,&district_id);
if(region_id>0&&district_id!=_mid->district_id)
{
ret_region_num--;
memmove(&(region_hit[i]),&(region_hit[i+1]),sizeof(scan_result_t)*(ret_region_num-i));
}
else
{
i++;
}
}
return ret_region_num;
}
int fill_regex_pos(struct regex_pos_t *regex_pos,int size,rule_result_t *rs_result,const char* buff)
@@ -255,7 +286,7 @@ int hit_pos_RS2Maat(struct sub_item_pos_t* maat_sub_item,int size,rule_result_t*
}
return k;
}
int fill_region_hit_detail(const char* scan_buff,const _scan_status_t* _mid,
int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mid,
scan_result_t *region_hit,int region_cnt,
_compile_result_t *compile_hit,int compile_cnt,
struct Maat_hit_detail_t *hit_detail,int detail_num)
@@ -265,6 +296,7 @@ int fill_region_hit_detail(const char* scan_buff,const _scan_status_t* _mid,
int region_pos[MAX_SCANNER_HIT_NUM],pos=0;
int r_in_c_cnt=0;
int region_id=-1;
int district_id=-1; //Indifferenc
memset(r_in_c_flag,0,sizeof(r_in_c_flag));
memset(region_pos,0,sizeof(region_pos));
@@ -281,7 +313,7 @@ int fill_region_hit_detail(const char* scan_buff,const _scan_status_t* _mid,
pos=region_pos[j];
r_in_c_flag[pos]=1;
group_rule=(struct _Maat_group_rule_t*)(region_hit[pos].tag);
region_id=exprid2region_id(group_rule,region_hit[pos].expr_id);
region_id=exprid2region_id(group_rule,region_hit[pos].expr_id,&district_id);
if(region_id<0)
{
continue;
@@ -303,7 +335,7 @@ int fill_region_hit_detail(const char* scan_buff,const _scan_status_t* _mid,
group_rule=(struct _Maat_group_rule_t*)(region_hit[k].tag);
hit_detail[j].config_id=-2;
hit_detail[j].hit_region_cnt=1;
hit_detail[j].region_pos[0].region_id=exprid2region_id(group_rule,region_hit[k].expr_id);
hit_detail[j].region_pos[0].region_id=exprid2region_id(group_rule,region_hit[k].expr_id,&district_id);
hit_detail[j].region_pos[0].sub_item_num=region_hit[k].rnum;
hit_pos_RS2Maat(hit_detail[j].region_pos[0].sub_item_pos,MAAT_MAX_EXPR_ITEM_NUM,
region_hit[k].result,region_hit[k].rnum,scan_buff);
@@ -312,20 +344,48 @@ int fill_region_hit_detail(const char* scan_buff,const _scan_status_t* _mid,
}
return j;
}
struct _scan_status_t* _Maat_make_status(struct _Maat_feather_t* feather,int thread_num)
struct _INNER_scan_status_t* _make_inner_status(int thread_num)
{
struct _scan_status_t* _mid=NULL;
_mid=(struct _scan_status_t*)calloc(sizeof(struct _scan_status_t),1);
_mid->feather=feather;
_mid->thread_num=thread_num;
_mid->cur_hit_cnt=0;
_mid->hit_group_cnt=0;
_mid->hit_group_size=4;
_mid->hitted_group_id=(unsigned int*)malloc(sizeof(unsigned int)*_mid->hit_group_size);
feather->mid_cnt++;
struct _INNER_scan_status_t* inner_mid=NULL;
inner_mid=(struct _INNER_scan_status_t*)calloc(sizeof(struct _INNER_scan_status_t),1);
inner_mid->thread_num=thread_num;
inner_mid->cur_hit_cnt=0;
inner_mid->hit_group_cnt=0;
inner_mid->hit_group_size=4;
inner_mid->hitted_group_id=(unsigned int*)malloc(sizeof(unsigned int)*inner_mid->hit_group_size);
return inner_mid;
}
struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather)
{
struct _OUTER_scan_status_t* outer_mid=NULL;
outer_mid=(struct _OUTER_scan_status_t*)calloc(sizeof(struct _OUTER_scan_status_t),1);
outer_mid->feather=feather;
outer_mid->district_id=-1;
feather->outer_mid_cnt++;
return outer_mid;
}
struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* feather,int thread_num,int is_hit_region)
{
struct _OUTER_scan_status_t* _mid=NULL;
if(*raw_mid!=NULL)
{
_mid=(struct _OUTER_scan_status_t*)(*raw_mid);
}
if(is_hit_region==1)
{
if(_mid==NULL)
{
_mid=_make_outer_status(feather);
*raw_mid=_mid;
}
if(_mid->inner==NULL)
{
_mid->inner=_make_inner_status(thread_num);
feather->inner_mid_cnt++;
}
}
return _mid;
}
int detain_last_data(char* buff,int buff_size,int detained_len,const char* data,int data_len)
{
int to_copy_size=0,foward_offset=0;
@@ -352,7 +412,6 @@ int detain_last_data(char* buff,int buff_size,int detained_len,const char* data,
}
return ret_len;
}
Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger)
{
_Maat_feather_t* feather=(_Maat_feather_t*)calloc(sizeof(struct _Maat_feather_t),1);
@@ -633,7 +692,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
int region_ret=0,compile_ret=0,hit_region_cnt=0;
unsigned int sub_type=0;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _scan_status_t* _mid=NULL;
struct _OUTER_scan_status_t* _mid=(struct _OUTER_scan_status_t*)(*mid);
scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];//dynamic array
@@ -643,6 +702,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{
gettimeofday(&start,NULL);
}
_mid=grab_mid(mid,_feather, thread_num, 0);
p_table=acqurie_table(_feather, table_id,TABLE_TYPE_EXPR);
if(p_table==NULL)
{
@@ -652,6 +712,10 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{
return 0;
}
if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
{
return -1;
}
if(p_table->do_charset_merge==1)
{
sub_type=make_sub_type(table_id,CHARSET_NONE,0);
@@ -696,25 +760,21 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
}
}
if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS)
{
hit_region_cnt=match_district(_mid,region_result,hit_region_cnt);
}
if(hit_region_cnt>0)
{
p_table->hit_cnt++;
if(*mid==NULL)
{
_mid=_Maat_make_status(_feather,thread_num);
*mid=_mid;
}
else
{
_mid=(struct _scan_status_t*)(*mid);
}
compile_ret=region_compile(_mid,
_mid=grab_mid(mid,_feather,thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
hit_region_cnt,
result,compile_result,rule_num);
if(hit_detail!=NULL)
{
*detail_ret=fill_region_hit_detail(data,_mid,
*detail_ret=fill_region_hit_detail(data,_mid->inner,
region_result,hit_region_cnt,
compile_result,compile_ret,
hit_detail,detail_num);
@@ -750,7 +810,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
,scan_status_t *mid,int thread_num)
{
int region_ret=0,compile_ret=0;
struct _scan_status_t* _mid=NULL;
struct _OUTER_scan_status_t* _mid=NULL;
scan_data_t intval_scan_data;
scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];
@@ -792,16 +852,8 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
else if(region_ret>0)
{
p_table->hit_cnt++;
if(*mid==NULL)
{
_mid=_Maat_make_status(_feather,thread_num);
*mid=_mid;
}
else
{
_mid=(struct _scan_status_t*)(*mid);
}
compile_ret=region_compile(_mid,
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
region_ret,
result,compile_result,rule_num);
@@ -827,7 +879,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
,scan_status_t *mid,int thread_num)
{
int region_ret=0,compile_ret=0;
struct _scan_status_t* _mid=NULL;
struct _OUTER_scan_status_t* _mid=NULL;
scan_data_t ip_scan_data;
scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];
@@ -892,16 +944,8 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
else if(region_ret>0)
{
p_table->hit_cnt++;
if(*mid==NULL)
{
_mid=_Maat_make_status(_feather,thread_num);
*mid=_mid;
}
else
{
_mid=(struct _scan_status_t*)(*mid);
}
compile_ret=region_compile(_mid,
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
region_ret,
result,compile_result,rule_num);
@@ -987,15 +1031,17 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
int sub_type=0;
int region_ret=0,hit_region_cnt=0,compile_ret=0;
struct _scan_status_t* _mid=(struct _scan_status_t*)(*mid);
struct _OUTER_scan_status_t* _mid=NULL;
scan_result_t *region_result;
_compile_result_t compile_result[rule_num];//dynamic array
scan_data_t region_scan_data;
_Maat_table_info_t* p_table=NULL;
struct timeval start,end;
if(sp->feather->perf_on==1)
{
gettimeofday(&start,NULL);
}
_mid=grab_mid(mid, sp->feather, sp->thread_num,0);
if(data==NULL||data_len==0)
{
return 0;
@@ -1008,10 +1054,15 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
{
return 0;
}
if(sp->feather->p_table_info[sp->table_id]->cfg_num==0)
p_table=sp->feather->p_table_info[sp->table_id];
if(p_table->cfg_num==0)
{
return 0;
}
if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
{
return -1;
}
region_result=scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*sp->thread_num;
*detail_ret=0;
if(sp->do_merge==1)
@@ -1081,20 +1132,15 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
hit_region_cnt+=region_ret;
}
}
if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS)
{
hit_region_cnt=match_district(_mid,region_result,hit_region_cnt);
}
if(hit_region_cnt>0)
{
sp->feather->p_table_info[sp->table_id]->hit_cnt++;
if(*mid==NULL)
{
_mid=_Maat_make_status(sp->feather,sp->thread_num);
*mid=_mid;
}
else
{
_mid=(struct _scan_status_t*)(*mid);
}
compile_ret=region_compile(_mid,
p_table->hit_cnt++;
_mid=grab_mid(mid, sp->feather,sp->thread_num, 1);
compile_ret=region_compile(sp->feather,_mid->inner,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
hit_region_cnt,
result,compile_result,rule_num);
@@ -1102,14 +1148,14 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
{
if(sp->scan_buff!=NULL)
{
*detail_ret=fill_region_hit_detail(sp->scan_buff,_mid,
*detail_ret=fill_region_hit_detail(sp->scan_buff,_mid->inner,
region_result,hit_region_cnt,
compile_result,compile_ret,
hit_detail,detail_num);
}
else
{
*detail_ret=fill_region_hit_detail(data,_mid,
*detail_ret=fill_region_hit_detail(data,_mid->inner,
region_result,hit_region_cnt,
compile_result,compile_ret,
hit_detail,detail_num);
@@ -1119,7 +1165,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
if(*detail_ret==0)
{
free(sp->scan_buff);
sp->scan_buff=0;
sp->scan_buff=NULL;
}
if(sp->feather->perf_on==1)
{
@@ -1244,7 +1290,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
GIE_handle_t* GIE_handle=sp->feather->scanner->digest_handle[sp->table_id];
unsigned long long digest_len=0;
char* digest_buff=NULL;
struct _scan_status_t* _mid=(struct _scan_status_t*)(*mid);
struct _OUTER_scan_status_t* _mid=NULL;
pthread_rwlock_t *GIE_rwlock=&(sp->feather->scanner->digest_rwlock[sp->table_id]);
struct timeval start,end;
if(sp->feather->perf_on==1)
@@ -1291,16 +1337,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
if(hit_region_cnt>0)
{
sp->feather->p_table_info[sp->table_id]->hit_cnt++;
if(*mid==NULL)
{
_mid=_Maat_make_status(sp->feather,sp->thread_num);
*mid=_mid;
}
else
{
_mid=(struct _scan_status_t*)(*mid);
}
compile_ret=region_compile(_mid,
_mid=grab_mid(mid,sp->feather, sp->thread_num,1);
compile_ret=region_compile(sp->feather,_mid->inner,
query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag),
hit_region_cnt,
result,compile_result,rule_num);
@@ -1340,17 +1378,52 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para)
return;
}
int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT type,const void* value,int size)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _OUTER_scan_status_t* _mid=NULL;
int ret=0,map_ret=-1;
_mid=grab_mid(mid,_feather, 0, 0);
if(*mid==NULL)
{
_mid=_make_outer_status(_feather);
*mid=_mid;
}
switch(type)
{
case MAAT_SET_SCAN_DISTRICT:
map_ret=map_str2int(_feather->scanner->district_map,(const char*)value,&(_mid->district_id));
if(map_ret<0)
{
//May be the district have not effect yet.
}
_mid->is_set_district=1;
ret=0;
break;
default:
ret=-1;
break;
}
return ret;
}
void Maat_clean_status(scan_status_t* mid)
{
struct _scan_status_t* _mid=NULL;
struct _OUTER_scan_status_t* _mid=NULL;
if(*mid==NULL)
{
return;
}
_mid=(struct _scan_status_t*)(*mid);
_mid->feather->mid_cnt--;
free(_mid->hitted_group_id);
_mid=(struct _OUTER_scan_status_t*)(*mid);
_mid->feather->outer_mid_cnt--;
if(_mid->inner!=NULL)
{
free(_mid->inner->hitted_group_id);
free(_mid->inner);
_mid->feather->inner_mid_cnt--;
}
_mid->feather=NULL;
free(_mid);
*mid=NULL;
return;
}

104
src/entry/Maat_rule.cpp
View File

@@ -25,7 +25,7 @@
#include "mesa_fuzzy.h"
#include "great_index_engine.h"
int MAAT_FRAME_VERSION_1_4_20160210=1;
int MAAT_FRAME_VERSION_1_5_20160211=1;
const char *maat_module="MAAT Frame";
const char* CHARSET_STRING[]={"CHARSET_NONE","GBK","BIG5","UNICODE","UTF-8"};
@@ -269,6 +269,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN);
map_register(string2int_map,"intval", TABLE_TYPE_INTVAL);
map_register(string2int_map,"digest", TABLE_TYPE_DIGEST);
map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS);
map_register(string2int_map,"group", TABLE_TYPE_GROUP);
map_register(string2int_map,"bin", CHARSET_NONE);
map_register(string2int_map,"gbk", CHARSET_GBK);
@@ -706,6 +707,8 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread
scanner->region_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(scanner->region_hash,0);
scanner->district_map=map_create();
scanner->version=version;
scanner->cfg_num=0;
@@ -738,7 +741,7 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner)
MESA_htable_destroy(scanner->compile_hash,NULL);
MESA_htable_destroy(scanner->group_hash, NULL);
MESA_htable_destroy(scanner->region_hash, NULL);
map_destroy(scanner->district_map);
destroy_bool_matcher((void*)scanner->expr_compiler);
q_cnt=MESA_lqueue_get_count(scanner->region_update_q);
for(i=0;i<q_cnt;i++)
@@ -918,11 +921,12 @@ void batch_update(struct _Maat_scanner_t *scanner,void*logger)
scanner->last_update_time=time(NULL);
return;
}
struct _Maat_group_rule_t* add_region_to_group(struct _Maat_group_rule_t* group,int region_id,int expr_id,enum MAAT_TABLE_TYPE region_type)
struct _Maat_group_rule_t* add_region_to_group(struct _Maat_group_rule_t* group,int region_id,int district_id,int expr_id,enum MAAT_TABLE_TYPE region_type)
{
struct _Maat_region_rule_t* region_rule=(struct _Maat_region_rule_t*)malloc(sizeof(struct _Maat_region_rule_t));
region_rule->region_id=region_id;
region_rule->expr_id=expr_id;
region_rule->district_id=district_id;
region_rule->region_type=region_type;
pthread_mutex_lock(&(group->mutex));
dynamic_array_write(group->region_rules,group->region_boundary,region_rule);
@@ -1076,12 +1080,24 @@ int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_n
}
return 1;
}
int get_district_id(_Maat_scanner_t *scanner,const char* district_str)
{
int map_ret=0,district_id=-1;
map_ret=map_str2int(scanner->district_map, district_str,&district_id);
if(map_ret<0)
{
district_id= scanner->district_num;
map_register(scanner->district_map,district_str, district_id);
scanner->district_num++;
}
return district_id;
}
int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger)
{
unsigned int i=0,j=0;
char* p=NULL,*saveptr=NULL,*region_string=NULL;
int region_str_len=0,ret=0,k=0;
int expr_id=0;
int expr_id=0,district_id=-1;
scan_rule_t*p_rule=NULL;
struct _Maat_group_rule_t* group_rule=NULL;
@@ -1096,7 +1112,12 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
int sub_expr_cnt=0;
struct op_expr_t *op_expr=NULL;
struct _Maat_group_rule_t* u_para=NULL;
if(table->table_type==TABLE_TYPE_EXPR_PLUS)
{
assert(strlen(db_rule->district)>0);
district_id=get_district_id(scanner, db_rule->district);
}
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id);
if(group_rule==NULL)
{
@@ -1168,7 +1189,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
break;
case EXPR_TYPE_REGEX://it's easy,no need to charset convert
expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule,db_rule->region_id,expr_id,TABLE_TYPE_EXPR);
u_para=add_region_to_group(group_rule,db_rule->region_id,district_id,expr_id,TABLE_TYPE_EXPR);
if(u_para==NULL)
{
return -1;
@@ -1234,7 +1255,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
break;
}
expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule, db_rule->region_id,expr_id, TABLE_TYPE_EXPR);
u_para=add_region_to_group(group_rule, db_rule->region_id,district_id,expr_id, table->table_type);
if(u_para==NULL)//duplicate
{
return -1;
@@ -1299,7 +1320,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
else
{
expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule, db_rule->region_id,expr_id, TABLE_TYPE_EXPR);
u_para=add_region_to_group(group_rule, db_rule->region_id,district_id,expr_id, table->table_type);
if(u_para==NULL)
{
return -1;
@@ -1335,7 +1356,7 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule
scan_rule_t* p_rule=NULL;
struct op_expr_t* op_expr=NULL;
struct _Maat_group_rule_t* u_para=NULL;
int expr_id=0;
int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id);
if(group_rule==NULL)
@@ -1345,7 +1366,7 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule
}
expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule,db_ip_rule->region_id,expr_id,TABLE_TYPE_IP);
u_para=add_region_to_group(group_rule,db_ip_rule->region_id,district_id,expr_id,TABLE_TYPE_IP);
if(u_para==NULL)
{
return -1;
@@ -1366,8 +1387,8 @@ int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* in
scan_rule_t* p_rule=NULL;
struct op_expr_t* op_expr=NULL;
struct _Maat_group_rule_t* u_para=NULL;
int expr_id=0;
int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id);
if(group_rule==NULL)
{
@@ -1375,7 +1396,7 @@ int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* in
HASH_add_by_id(scanner->group_hash, intval_rule->group_id, group_rule);
}
expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule,intval_rule->region_id,expr_id,TABLE_TYPE_INTVAL);
u_para=add_region_to_group(group_rule,intval_rule->region_id,district_id,expr_id,TABLE_TYPE_INTVAL);
if(u_para==NULL)
{
return -1;
@@ -1395,7 +1416,7 @@ int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db
struct _Maat_group_rule_t* group_rule=NULL;
GIE_digest_t* digest_rule=NULL;
struct _Maat_group_rule_t* u_para=NULL;
int expr_id=0;
int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id);
if(group_rule==NULL)
@@ -1404,7 +1425,7 @@ int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db
HASH_add_by_id(scanner->group_hash, db_digest_rule->group_id, group_rule);
}
expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule,db_digest_rule->region_id,expr_id,TABLE_TYPE_DIGEST);
u_para=add_region_to_group(group_rule,db_digest_rule->region_id,expr_id,district_id,TABLE_TYPE_DIGEST);
if(u_para==NULL)
{
return -1;
@@ -1643,20 +1664,46 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st
{
struct db_str_rule_t* maat_str_rule=(struct db_str_rule_t*)malloc(sizeof(struct db_str_rule_t));
int ret=0,db_hexbin=0;
ret=sscanf(table_line,"%d\t%d\t%s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id)
,&(maat_str_rule->group_id)
,maat_str_rule->keywords
,(int*)&(maat_str_rule->expr_type)
,(int*)&(maat_str_rule->match_method)
,&db_hexbin
,&(maat_str_rule->is_valid));
if(ret!=7)
switch(table->table_type)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid format of expr table %s:%s",table->table_name,table_line);
free(maat_str_rule);
maat_str_rule=NULL;
return;
case TABLE_TYPE_EXPR:
ret=sscanf(table_line,"%d\t%d\t%s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id)
,&(maat_str_rule->group_id)
,maat_str_rule->keywords
,(int*)&(maat_str_rule->expr_type)
,(int*)&(maat_str_rule->match_method)
,&db_hexbin
,&(maat_str_rule->is_valid));
if(ret!=7)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid format of expr table %s:%s",table->table_name,table_line);
free(maat_str_rule);
maat_str_rule=NULL;
return;
}
break;
case TABLE_TYPE_EXPR_PLUS:
ret=sscanf(table_line,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id)
,&(maat_str_rule->group_id)
,maat_str_rule->district
,maat_str_rule->keywords
,(int*)&(maat_str_rule->expr_type)
,(int*)&(maat_str_rule->match_method)
,&db_hexbin
,&(maat_str_rule->is_valid));
if(ret!=8)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid format of expr_plus table %s:%s",table->table_name,table_line);
free(maat_str_rule);
maat_str_rule=NULL;
return;
}
break;
default:
assert(0);
break;
}
switch(db_hexbin)
{
@@ -2305,6 +2352,7 @@ void maat_update_cb(const char* table_name,const char* line,void *u_para)
switch(feather->p_table_info[table_id]->table_type)
{
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
update_expr_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON);
break;
case TABLE_TYPE_IP:

20
src/entry/Maat_rule_internal.h
View File

@@ -40,6 +40,7 @@ typedef int atomic_t;
#define MAX_TABLE_NAME_LEN 256
#define MAX_TABLE_LINE_SIZE (1024*4)
#define MAX_EXPR_KEYLEN 1024
#define MAX_DISTRICT_LEN 64
#define MAX_PLUGING_NUM 32
#define MAX_SCANNER_HIT_NUM 64
@@ -76,6 +77,7 @@ enum MAAT_TABLE_TYPE
TABLE_TYPE_PLUGIN,
TABLE_TYPE_INTVAL,
TABLE_TYPE_DIGEST,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_GROUP
};
@@ -99,6 +101,7 @@ struct db_str_rule_t
int region_id;
int group_id;
char keywords[MAX_EXPR_KEYLEN];
char district[MAX_DISTRICT_LEN];
enum MAAT_EXPR_TYPE expr_type;
enum MAAT_MATCH_METHOD match_method;
int is_hexbin;
@@ -171,6 +174,7 @@ struct _Maat_region_rule_t
{
int region_id;
int expr_id;
int district_id;
enum MAAT_TABLE_TYPE region_type;
};
struct _Maat_group_rule_t
@@ -235,9 +239,9 @@ struct _Maat_table_info_t
long long stream_num;
long long hit_cnt;
};
struct _scan_status_t
struct _INNER_scan_status_t
{
struct _Maat_feather_t* feather;
int thread_num;
int cur_hit_cnt;
int hit_group_cnt;
@@ -245,6 +249,13 @@ struct _scan_status_t
unsigned int cur_hit_id[MAX_SCANNER_HIT_NUM];
unsigned int *hitted_group_id;
};
struct _OUTER_scan_status_t
{
struct _Maat_feather_t* feather;
int is_set_district;
int district_id;
struct _INNER_scan_status_t* inner;
};
enum maat_garbage_type
{
GARBAGE_SCANNER=0,
@@ -288,6 +299,8 @@ struct _Maat_scanner_t
MESA_htable_handle region_hash;
MESA_htable_handle group_hash;
MESA_htable_handle compile_hash;
MESA_htable_handle district_map;
unsigned int district_num;
unsigned int cfg_num;
unsigned int exprid_generator;
MESA_lqueue_head region_update_q;
@@ -324,7 +337,8 @@ struct _Maat_feather_t
int total_stat_id;
int fs_status_id[MAX_MAAT_STAT_NUM];
int fs_column_id[MAX_MAAT_STAT_NUM];
long long mid_cnt;
long long outer_mid_cnt;
long long inner_mid_cnt;
long long hit_cnt;
};
struct _maat_garbage_t

9
src/entry/Maat_stat.cpp
View File

@@ -3,7 +3,8 @@
enum MAAT_FS_STATUS{
STATUS_VERSION=0,
STATUS_TABLE_NUM,
STATUS_MID_NUM,
STATUS_OUTER_MID_NUM,
STATUS_INNER_MID_NUM,
STATUS_GARBAGE_QSIZE,
};
@@ -32,7 +33,8 @@ void maat_stat_init(struct _Maat_feather_t* feather)
feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"version");
feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"table_num");
feather->fs_status_id[STATUS_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"mid_num");
feather->fs_status_id[STATUS_OUTER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"outer_mid");
feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid");
feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"garbage_num");
feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule");
@@ -93,7 +95,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_MID_NUM], 0,FS_OP_SET,feather->mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,feather->outer_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,feather->inner_mid_cnt);
value=MESA_lqueue_get_count(feather->garbage_q);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value);

15
src/entry/json2iris.cpp
View File

@@ -110,8 +110,11 @@ int set_iris_descriptor(const char* json_file,cJSON *json,struct iris_descriptio
map_register(iris_cfg->str2int_map, "ip",TABLE_TYPE_IP);
map_register(iris_cfg->str2int_map, "string",TABLE_TYPE_EXPR);
map_register(iris_cfg->str2int_map, "expr",TABLE_TYPE_EXPR);
map_register(iris_cfg->str2int_map, "expr_plus",TABLE_TYPE_EXPR_PLUS);
map_register(iris_cfg->str2int_map, "intval",TABLE_TYPE_INTVAL);
map_register(iris_cfg->str2int_map, "digest",TABLE_TYPE_DIGEST);
map_register(iris_cfg->str2int_map, "ipv4",4);
map_register(iris_cfg->str2int_map, "ipv6",6);
@@ -364,7 +367,7 @@ int write_ip_rule(cJSON *region_json,struct iris_description_t *p_iris,const cha
return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
}
int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger)
int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,enum MAAT_TABLE_TYPE table_type,void * logger)
{
struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
int cmd_cnt=0;
@@ -378,6 +381,13 @@ int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const c
json_cmd[cmd_cnt].json_type=cJSON_Number;
cmd_cnt++;
if(table_type==TABLE_TYPE_EXPR_PLUS)
{
json_cmd[cmd_cnt].json_string="district";
json_cmd[cmd_cnt].json_type=cJSON_String;
cmd_cnt++;
}
json_cmd[cmd_cnt].json_string="keywords";
json_cmd[cmd_cnt].json_type=cJSON_String;
cmd_cnt++;
@@ -598,7 +608,8 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri
switch(table_type)
{
case TABLE_TYPE_EXPR:
ret=write_expr_rule(table_content, p_iris, table_info->table_path, logger);
case TABLE_TYPE_EXPR_PLUS:
ret=write_expr_rule(table_content, p_iris, table_info->table_path,table_type, logger);
break;
case TABLE_TYPE_IP:
ret=write_ip_rule(table_content, p_iris, table_info->table_path, logger);

34
src/entry/map_str2int.cpp
View File

@@ -1,21 +1,35 @@
#include <MESA/MESA_htable.h>
void map_tmp_free(void* ptr)
{
free(ptr);
}
long read_map_val(void *data, const uchar *key, uint size, void *user_arg)
{
if(data!=NULL)
{
*(int*)user_arg=*(int*)data;
return 1;
}
else
{
return 0;
}
}
MESA_htable_handle map_create(void)
{
MESA_htable_handle string2int_map;
MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs));
hargs.thread_safe=1;
hargs.hash_slot_size = 1024*1024;
hargs.hash_slot_size = 4*1024;
hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_LRU;
hargs.expire_time = 0;
hargs.key_comp = NULL;
hargs.key2index = NULL;
hargs.recursive = 0;
hargs.recursive = 1;
hargs.data_free = map_tmp_free;
hargs.data_expire_with_condition = NULL;
string2int_map=MESA_htable_create(&hargs, sizeof(hargs));
@@ -31,20 +45,28 @@ int map_register(MESA_htable_handle handle,const char* string,int value)
{
unsigned int size=strlen(string);
unsigned char *key=(unsigned char *)string;
int *data=(int*)malloc(sizeof(int));
int ret=0;
int * data= (int*)malloc(sizeof(int));
*data=value;
ret=MESA_htable_add(handle,key,size,data);
if(ret<0)
{
free(data);
}
return ret;
}
int map_str2int(MESA_htable_handle handle,const char* string,int* value)
{
int *data=NULL;
long cb_ret=0;
unsigned int size=strlen(string);
data=(int*)MESA_htable_search(handle,(unsigned char*)string,size);
if(data!=NULL)
data=(int*)MESA_htable_search_cb(handle,(unsigned char*)string,size,
read_map_val,value,&cb_ret);
// data=(int*)MESA_htable_search(handle,(unsigned char*)string,size);
if(cb_ret>0)
{
*value=*data;
return 1;
}
else

28
test/maat_json.json
View File

@@ -188,6 +188,34 @@
]
}
]
},
{
"compile_id": 128,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"effective_rage": 0,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "group_8",
"regions": [
{
"table_name": "HTTP_REGION",
"table_type": "expr_plus",
"table_content": {
"district": "URL",
"keywords": "abckkk&123",
"expr_type": "and",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
],
"plugin_table": [

445
test/maat_test.cpp
View File

@@ -37,26 +37,273 @@ void Maat_read_entry_finish_cb(void* u_para)
{
return;
}
void print_maat_ret(int ret)
{
switch(ret)
{
case -1:
printf("scan error.\n");
break;
case -2:
printf("hit current region,but not hit compile rule.\n");
break;
case 0:
printf("nothing hit\n");
break;
default://>0
printf("hit %d rules\n",ret);
break;
}
return;
}
int test_string_full_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int ret=0;
int table_id=0;
struct Maat_rule_t result[4];
int found_pos[4];
const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table %s register failed.\n",table_name);
return -1;
}
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
result,found_pos, 4,
mid, 0);
print_maat_ret(ret);
return ret;
}
int test_intval_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int table_id=0,ret=0;
int scan_val=2015;
struct Maat_rule_t result[4];
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table %s register failed.",table_name);
}
else
{
ret=Maat_scan_intval(feather, table_id, scan_val, result,4,mid, 0);
print_maat_ret(ret);
}
return ret;
}
int test_str_stream_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int table_id=0,ret=0;
struct Maat_rule_t result[4];
const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table %s register failed.\n",table_name);
return -1;
}
struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10);
stream_para_t sp=Maat_stream_scan_string_start(feather,table_id,0);
int detail_ret=0;
if(sp==NULL)
{
printf("stream scan start failed.\n");
return -1;
}
ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,"www.cyberessays.com", strlen("www.cyberessays.com")
,result,4,hit_detail,10
,&detail_ret,mid);
ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data, strlen(scan_data)
,result,4,hit_detail,10
,&detail_ret,mid);
Maat_stream_scan_string_end(&sp);
free(hit_detail);
print_maat_ret(ret);
return ret;
}
int test_ipv4_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int table_id=0,ret=0;
struct Maat_rule_t result[4];
struct ipaddr ipv4_addr;
struct stream_tuple4_v4 v4_addr;
ipv4_addr.addrtype=ADDR_TYPE_IPV4;
inet_pton(AF_INET,"10.0.6.205",&(v4_addr.saddr));
v4_addr.source=htons(50001);
inet_pton(AF_INET,"10.0.6.201",&(v4_addr.daddr));
v4_addr.dest=htons(80);
ipv4_addr.v4=&v4_addr;
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table %s register failed.\n",table_name);
}
else
{
ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,result,4, mid,0);
if(ret>0)
{
printf("ipv4 scan hit compile rule id %d.\n",result[0].config_id);
}
}
return ret;
}
int test_ipv6_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int table_id=0,ret=0;
struct Maat_rule_t result[4];
struct ipaddr ipv6_addr;
struct stream_tuple4_v6 v6_addr;
ipv6_addr.addrtype=ADDR_TYPE_IPV6;
inet_pton(AF_INET6,"2001:da8:205:1::101",&(v6_addr.saddr));
v6_addr.source=htons(50001);
inet_pton(AF_INET6,"2001:da8:205:1::102",&(v6_addr.daddr));
v6_addr.dest=htons(80);
ipv6_addr.v6=&v6_addr;
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table %s register failed.\n",table_name);
}
else
{
ret=Maat_scan_proto_addr(feather,table_id,&ipv6_addr,6,result,4, mid,0);
if(ret==-2)
{
printf("ipv6 scan hit region.\n");
}
else
{
printf("ipv6 scan result:%d ,shoulde be -2.\n",ret);
}
}
return ret;
}
int test_digest_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int table_id=0,ret=0;
const char* digest_test_file="./digest_test.data";
struct stat digest_fstat;
unsigned long long read_size=0,scan_offset=0;
char digest_test_buff[4096]={0};
struct Maat_rule_t result[4];
stream_para_t sp=NULL;
table_id=Maat_table_register(feather, table_name);
if(table_id<0)
{
printf("registe table %s error.\n",table_name);
return 0;
}
ret=stat(digest_test_file,&digest_fstat);
if(ret!=0)
{
printf("fstat %s error.\n",digest_test_file);
return 0;
}
FILE* fp=fopen(digest_test_file,"r");
if(fp!=NULL)
{
sp=Maat_stream_scan_digest_start(feather, table_id, digest_fstat.st_size, 0);
while(0==feof(fp))
{
read_size=fread(digest_test_buff,1,sizeof(digest_test_buff),fp);
ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,mid);
scan_offset+=read_size;
if(ret>0)
{
printf("digest scan hit %d.\n",result[0].config_id);
}
}
fclose(fp);
}
else
{
printf("fopen %s error.\n",digest_test_file);
}
Maat_stream_scan_string_end(&sp);
return ret;
}
int test_plugin_table(Maat_feather_t feather,const char* table_name,void* logger)
{
int table_id=0,ret=0;
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table %s register failed.\n",table_name);
}
else
{
ret=Maat_table_callback_register(feather, table_id,
Maat_read_entry_start_cb,
Maat_read_entry_cb,
Maat_read_entry_finish_cb,
logger);
if(ret<0)
{
printf("Maat callback register table %s error.\n",table_name);
}
}
return ret;
}
int test_expr_plus(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int ret=0;
int table_id=0;
struct Maat_rule_t result[4];
int found_pos[4];
const char* region_name="URL";
const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table %s register failed.\n",table_name);
return -1;
}
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
result,found_pos, 4,
mid, 0);
if(ret>0)
{
printf("Should not hit without set district.\n");
return -1;
}
ret=Maat_set_scan_status(feather, mid, MAAT_SET_SCAN_DISTRICT,region_name,strlen(region_name)+1);
if(ret<0)
{
printf("set MAAT_SET_SCAN_DISTRICT failed.\n");
return -1;
}
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
result,found_pos, 4,
mid, 0);
if(ret>0)
{
printf("Hit expr_puls rule %d.\n",result[0].config_id);
}
return ret;
}
int main(int argc,char* argv[])
{
Maat_feather_t feather=NULL;
int cb_table_id=-1,url_scan_table_id=-1,size_scan_table_id=-1,ip_scan_table_id=-1,digest_scan_table_id=-1;
int ret=-1;
int g_iThreadNum=4;
const char* table_info_path="./table_info.conf";
const char* json_path="./maat_json.json";
// const char* ful_cfg_dir="./maat_json.json_iris_tmp/index";
// const char* inc_cfg_dir="./rules/inc/index";
const char* log_file="./test.log";
const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
const char* digest_test_file="./digest_test.data";
const char* stat_file="./scan_staus.log";
struct stat digest_fstat;
unsigned long long read_size=0,scan_offset=0;
char digest_test_buff[4096]={0};
int scan_val=2015;
struct Maat_rule_t result[4];
int found_pos[4];
int scan_detail=0;
scan_status_t mid=NULL;
void *logger=MESA_create_runtime_log_handle(log_file,0);
@@ -80,178 +327,30 @@ int main(int argc,char* argv[])
printf("Maat initial error, see %s\n",log_file);
return -1;
}
test_plugin_table(feather, "QD_ENTRY_INFO",logger);
cb_table_id=Maat_table_register(feather,"QD_ENTRY_INFO");
if(cb_table_id==-1)
{
printf("Database table QD_ENTRY_INFO register failed.");
}
else
{
ret=Maat_table_callback_register(feather, cb_table_id,
Maat_read_entry_start_cb,
Maat_read_entry_cb,
Maat_read_entry_finish_cb,
logger);
if(ret<0)
{
printf("Maat callback register table QD_ENTRY_INFO error.");
}
}
url_scan_table_id=Maat_table_register(feather,"HTTP_URL");
if(url_scan_table_id==-1)
{
printf("Database table HTTP_URL register failed.");
}
else
{
ret=Maat_full_scan_string(feather, url_scan_table_id,CHARSET_GBK, scan_data, strlen(scan_data),
result,found_pos, 4,
&mid, 0);
switch(ret)
{
case -1:
printf("scan error.\n");
break;
case -2:
printf("hit current region,but not hit compile rule.\n");
break;
case 0:
printf("nothing hit\n");
break;
default://>0
printf("hit %d rules\n",ret);
break;
}
}
test_string_full_scan(feather, "HTTP_URL", &mid);
//not clean status here, to test_ipv4_scan make hit compile rule.
test_ipv4_scan(feather, "IP_CONFIG", &mid);
Maat_clean_status(&mid);
size_scan_table_id=Maat_table_register(feather,"CONTENT_SIZE");
if(size_scan_table_id==-1)
{
printf("Database table CONTENT_SIZE register failed.");
}
else
{
ret=Maat_scan_intval(feather, size_scan_table_id, scan_val, result,4,&mid, 0);
switch(ret)
{
case -1:
printf("scan error.\n");
break;
case -2:
printf("hit current region,but not hit compile rule.\n");
break;
case 0:
printf("nothing hit\n");
break;
default://>0
printf("hit %d rules\n",ret);
break;
}
}
sleep(2);
Maat_clean_status(&mid);
struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10);
stream_para_t sp=Maat_stream_scan_string_start(feather,url_scan_table_id,0);
int detail_ret=0;
if(sp==NULL)
{
printf("stream scan start failed.\n");
return -1;
}
ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,"www.cyberessays.com", strlen("www.cyberessays.com")
,result,4,hit_detail,10
,&detail_ret,&mid);
ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data, strlen(scan_data)
,result,4,hit_detail,10
,&detail_ret,&mid);
Maat_stream_scan_string_end(&sp);
struct ipaddr ipv4_addr,ipv6_addr;
struct stream_tuple4_v4 v4_addr;
struct stream_tuple4_v6 v6_addr;
ipv4_addr.addrtype=ADDR_TYPE_IPV4;
inet_pton(AF_INET,"10.0.6.205",&(v4_addr.saddr));
v4_addr.source=htons(50001);
inet_pton(AF_INET,"10.0.6.201",&(v4_addr.daddr));
v4_addr.dest=htons(80);
ipv4_addr.v4=&v4_addr;
ip_scan_table_id=Maat_table_register(feather,"IP_CONFIG");
if(ip_scan_table_id==-1)
{
printf("Database table IP_CONFIG register failed.");
}
else
{
ret=Maat_scan_proto_addr(feather,ip_scan_table_id,&ipv4_addr,6,result,4, &mid,0);
if(ret>0)
{
printf("ipv4 scan hit compile rule id %d.\n",result[0].config_id);
}
}
test_intval_scan(feather,"CONTENT_SIZE" , &mid);
Maat_clean_status(&mid);
ipv6_addr.addrtype=ADDR_TYPE_IPV6;
inet_pton(AF_INET6,"2001:da8:205:1::101",&(v6_addr.saddr));
v6_addr.source=htons(50001);
inet_pton(AF_INET6,"2001:da8:205:1::102",&(v6_addr.daddr));
v6_addr.dest=htons(80);
ipv6_addr.v6=&v6_addr;
if(ip_scan_table_id>=0)
{
ret=Maat_scan_proto_addr(feather,ip_scan_table_id,&ipv6_addr,6,result,4, &mid,0);
if(ret==-2)
{
printf("ipv6 scan hit region.\n");
}
else
{
printf("ipv6 scan result:%d ,shoulde be -2.\n",ret);
}
}
Maat_clean_status(&mid);
digest_scan_table_id=Maat_table_register(feather, "FILE_DIGEST");
if(digest_scan_table_id<0)
{
printf("registe table FILE_DIGEST error.\n");
return 0;
}
ret=stat(digest_test_file,&digest_fstat);
if(ret!=0)
{
printf("fstat %s error.\n",digest_test_file);
return 0;
}
FILE* fp=fopen(digest_test_file,"r");
if(fp!=NULL)
{
sp=Maat_stream_scan_digest_start(feather, digest_scan_table_id, digest_fstat.st_size, 0);
while(0==feof(fp))
{
read_size=fread(digest_test_buff,1,sizeof(digest_test_buff),fp);
ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,&mid);
scan_offset+=read_size;
if(ret>0)
{
printf("digest scan hit %d.\n",result[0].config_id);
}
}
fclose(fp);
}
else
{
printf("fopen %s error.\n",digest_test_file);
}
Maat_stream_scan_string_end(&sp);
test_ipv6_scan(feather, "IP_CONFIG", &mid);
Maat_clean_status(&mid);
test_digest_scan(feather,"FILE_DIGEST", &mid);
Maat_clean_status(&mid);
test_expr_plus(feather, "HTTP_REGION", &mid);
Maat_clean_status(&mid);
sleep(4);
Maat_burn_feather(feather);
free(hit_detail);
return 0;
}

3
test/table_info.conf
View File

@@ -1,7 +1,7 @@
#each collumn seperate with '\t'
#id (0~65535)
#name string
#type one of ip,expr,digest,intval,compile or plugin
#type one of ip,expr,expr_plus,digest,intval,compile or plugin
#src_charset one of GBK,BIG5,UNICODE,UTF8
#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/'
#do_merege yes or no
@@ -14,3 +14,4 @@
5 CONTENT_SIZE intval GBK GBK no 0
6 QD_ENTRY_INFO plugin GBK GBK no 0
7 FILE_DIGEST digest GBK GBK no 0
8 HTTP_REGION expr_plus GBK GBK no 0