gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature-IP-CIDR' into 'master'

Browse Source 
2.7正式版

Closes #13, #12, and #11

See merge request MESA_framework/maat!31
This commit is contained in:
郑超
2019-07-11 17:09:27 +08:00
parent 0cdf2224b6 85509f0988
commit eb7912a854
22 changed files with 3689 additions and 963 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
inc/Maat_command.h
View File

@@ -10,7 +10,11 @@ enum MAAT_OPERATION
MAAT_OP_ADD,
MAAT_OP_RENEW_TIMEOUT //Rule expire time is changed to now+cmd->expire_after
};
enum MAAT_GROUP_RELATION
{
PARENT_TYPE_COMPILE=0,
PARENT_TYPE_GROUP
};
enum MAAT_REGION_TYPE
{
REGION_EXPR,
@@ -104,8 +108,12 @@ struct Maat_region_t
};
struct Maat_group_t
{
int region_num;
const char* table_name;
int group_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number.
int parent_id;
int not_flag;
enum MAAT_GROUP_RELATION parent_type;
int region_num;
struct Maat_region_t *regions;
};
struct Maat_cmd_t
@@ -170,5 +178,14 @@ void Maat_cmd_key_free(struct Maat_cmd_key**keys, int number);
int Maat_cmd_key_select(Maat_feather_t feather, int label_id, struct Maat_cmd_key** keys);
int Maat_cmd_select(Maat_feather_t feather, int label_id, int * output_ids, unsigned int size);
int Maat_cmd_flushDB(Maat_feather_t feather);
int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num);
int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_region_t* region, int group_id);
int Maat_command_raw_set_group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_group_t* group);
int Maat_cmd_get_new_group_id(Maat_feather_t feather);
int Maat_cmd_get_new_region_id(Maat_feather_t feather);
#endif

8
inc/Maat_rule.h
View File

@@ -185,8 +185,8 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
enum MAAT_SCAN_OPT
{
MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*,SIZE= strlen(string).DEFAULT: no default.
MAAT_SET_SCAN_LAST_REGION //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan cobination.
MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*,SIZE= strlen(string). DEFAULT: no default.
MAAT_SET_SCAN_LAST_REGION //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination.
};
//return 0 if success, return -1 when failed;
int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT type,const void* value,int size);
@@ -209,7 +209,7 @@ int Maat_full_scan_string(Maat_feather_t feather,int table_id
,enum MAAT_CHARSET charset,const char* data,int data_len
,struct Maat_rule_t*result,int* found_pos,int rule_num
,scan_status_t* mid,int thread_num);
//hite_detail could be NULL if unconcern
//hit_detail could be NULL if not cared.
int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
,enum MAAT_CHARSET charset,const char* data,int data_len
,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num
@@ -220,7 +220,7 @@ int Maat_stream_scan_string(stream_para_t* stream_para
,enum MAAT_CHARSET charset,const char* data,int data_len
,struct Maat_rule_t*result,int* found_pos,int rule_num
,scan_status_t* mid);
//hited_detail could be NULL if unconcern
//hited_detail could be NULL if not cared.
int Maat_stream_scan_string_detail(stream_para_t* stream_para
,enum MAAT_CHARSET charset,const char* data,int data_len
,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num

4
src/CMakeLists.txt
View File

@@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.5)
set(MAAT_FRAME_MAJOR_VERSION 2)
set(MAAT_FRAME_MINOR_VERSION 6)
set(MAAT_FRAME_MINOR_VERSION 7)
set(MAAT_FRAME_PATCH_VERSION 0)
set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION})
@@ -20,6 +20,7 @@ set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
target_include_directories(maat_frame_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/)
#target_include_directories(maat_frame_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/hiredis)
target_link_libraries(maat_frame_static hiredis-vip-static)
target_link_libraries(maat_frame_static igraph-static)
# Shared Library Output
add_library(maat_frame_shared SHARED ${MAAT_SRC})
@@ -32,6 +33,7 @@ set_target_properties(maat_frame_shared PROPERTIES LINK_FLAGS "-Wl,--version-scr
target_include_directories(maat_frame_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/)
target_include_directories(maat_frame_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/hiredis)
target_link_libraries(maat_frame_shared hiredis-vip-static ${MAAT_DEPEND_DYN_LIB})
target_link_libraries(maat_frame_shared igraph-static ${MAAT_DEPEND_DYN_LIB})
install(FILES ${PROJECT_SOURCE_DIR}/inc/Maat_rule.h DESTINATION include/MESA/)
install(FILES ${PROJECT_SOURCE_DIR}/inc/Maat_command.h DESTINATION include/MESA/)

289
src/entry/Maat_api.cpp
View File

@@ -16,7 +16,7 @@
#include "rulescan.h"
#include "json2iris.h"
struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type)
struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int table_id, enum MAAT_TABLE_TYPE expect_type)
{
struct Maat_table_desc *p_table=NULL;
if(table_id>MAX_TABLE_NUM)
@@ -34,20 +34,20 @@ struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather,int tabl
}
if(p_table->table_type!=expect_type)
{
if(expect_type!=TABLE_TYPE_EXPR||
p_table->table_type!=TABLE_TYPE_EXPR_PLUS)
if((expect_type==TABLE_TYPE_EXPR && p_table->table_type!=TABLE_TYPE_EXPR_PLUS)||
(expect_type==TABLE_TYPE_IP && p_table->table_type!=TABLE_TYPE_IP_PLUS))
{
return NULL;
}
}
return p_table;
}
inline void INC_SCANNER_REF(Maat_scanner_t*scanner,int thread_num)
inline void INC_SCANNER_REF(Maat_scanner*scanner,int thread_num)
{
alignment_int64_array_add(scanner->ref_cnt, thread_num, 1);
return;
}
inline void DEC_SCANNER_REF(Maat_scanner_t*scanner,int thread_num)
inline void DEC_SCANNER_REF(Maat_scanner*scanner,int thread_num)
{
alignment_int64_array_add(scanner->ref_cnt, thread_num, -1);
@@ -64,7 +64,8 @@ void Maat_clean_status(scan_status_t* mid)
alignment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1);
if(_mid->inner!=NULL)
{
free(_mid->inner->hitted_group_id);
dynamic_array_destroy(_mid->inner->cur_hit_groups, NULL);
free(_mid->inner->all_hit_group_array);
free(_mid->inner);
alignment_int64_array_add(_mid->feather->inner_mid_cnt,_mid->thread_num,-1);
}
@@ -117,15 +118,17 @@ int insert_set_id(unsigned long long **set, size_t* size, size_t cnt, unsigned l
}
}
size_t pickup_hit_region_from_compile(struct bool_expr *compile_hit,
const unsigned long long* hitted_id, size_t hit_cnt, int* region_pos, size_t size)
struct dynamic_array_t* hitted_id, size_t hit_cnt, int* region_pos, size_t size)
{
size_t i=0, j=0;
size_t k=0;
unsigned long long group_id=0;
for(i=0;i<hit_cnt;i++)
{
group_id=(unsigned long long)dynamic_array_read(hitted_id, i);
for(j=0; j<compile_hit->item_num; j++)
{
if(hitted_id[i]==compile_hit->items[j].item_id)
{
if(group_id==compile_hit->items[j].item_id)
{
region_pos[k]=i;
k++;
@@ -135,114 +138,111 @@ size_t pickup_hit_region_from_compile(struct bool_expr *compile_hit,
}
return k;
}
void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len)
void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len)
{
memcpy(rule, rule_head, sizeof(struct _head_Maat_rule_t));
memcpy(rule, rule_head, sizeof(struct Maat_rule_head));
memcpy(rule->service_defined, srv_def, MIN(srv_def_len,MAX_SERVICE_DEFINE_LEN));
return;
}
static int compare_compile_id(const void *a, const void *b)
static int compare_compile_inner(const void *a, const void *b)
{
struct Maat_rule_t *ra=(struct Maat_rule_t *)a;
struct Maat_rule_t *rb=(struct Maat_rule_t *)b;
const struct Maat_compile_group_relation *ra=*(const struct Maat_compile_group_relation **)a;
const struct Maat_compile_group_relation *rb=*(const struct Maat_compile_group_relation **)b;
return (rb->config_id-ra->config_id);
if(ra->group_cnt!=rb->group_cnt)
{
return (ra->group_cnt-rb->group_cnt);
}
else
{
return (rb->compile_id-ra->compile_id);
}
}
int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num)
{
int scan_ret=0,result_cnt=0;
int ret=0,i=0;
int scan_ret=0, result_cnt=0;
int ret=0, i=0, j=0;
size_t r_in_c_cnt=0;
int shortcut_avilable_cnt=0;
unsigned char has_not_flag=0;
struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler;
struct Maat_group_inner_t* group_rule=NULL;
struct Maat_compile_inner_t* array_mi_rule[MAX_SCANNER_HIT_NUM];
struct Maat_compile_inner_t* _mi_rule=NULL;
struct Maat_group_inner* group_rule=NULL;
struct Maat_compile_group_relation* relation_array[MAX_SCANNER_HIT_NUM];
struct Maat_compile_group_relation* relation=NULL;
int region_pos[MAX_SCANNER_HIT_NUM];
_mid->cur_hit_cnt=0;
_mid->cur_hit_group_cnt=0;
for(i=0;i<region_hit_num;i++)
{
group_rule=*(struct Maat_group_inner_t**)((char*)region_hit+region_type_size*i+group_offset);
group_rule=*(struct Maat_group_inner**)((char*)region_hit+region_type_size*i+group_offset);
if(group_rule->group_id<0)
{
continue;
}
if(group_rule->compile_shortcut!=NULL&&group_rule->ref_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM)
{
array_mi_rule[shortcut_avilable_cnt]=(struct Maat_compile_inner_t*)(group_rule->compile_shortcut);
shortcut_avilable_cnt++;
assert(group_rule->ref_by_children_cnt>=0);
assert(group_rule->ref_by_parent_cnt>=0);
for(j=0; j<group_rule->top_group_cnt; j++)
{
if(_mid->cur_hit_group_cnt<MAX_SCANNER_HIT_NUM)
{
dynamic_array_write(_mid->cur_hit_groups, _mid->cur_hit_group_cnt, (void*)group_rule->top_groups[j]);
_mid->cur_hit_group_cnt++;
}
ret=insert_set_id(&(_mid->all_hit_group_array),
&(_mid->all_hit_group_array_sz),
_mid->all_hit_group_cnt,
group_rule->top_groups[j]);
_mid->all_hit_group_cnt+=ret;
}
_mid->cur_hit_id[_mid->cur_hit_cnt]=group_rule->group_id;
_mid->cur_hit_cnt++;
ret=insert_set_id(&(_mid->hitted_group_id),
&(_mid->hit_group_size),
_mid->hit_group_cnt,
group_rule->group_id);
_mid->hit_group_cnt+=ret;
}
if((region_hit_num>0 &&shortcut_avilable_cnt==region_hit_num) ||
shortcut_avilable_cnt==MAX_SCANNER_HIT_NUM)
scan_ret=bool_matcher_match(bm, thread_num,
_mid->all_hit_group_array, _mid->all_hit_group_cnt,
(void **)relation_array, MAX_SCANNER_HIT_NUM);
if(scan_ret>1)
{
//short cut for rules contains one group
scan_ret=shortcut_avilable_cnt;
alignment_int64_array_add(feather->orphan_group_saving, thread_num, 1);
}
else if(0&&shortcut_avilable_cnt==0&&region_hit_num==1&&_mid->hit_group_cnt==1&&is_last_region==1)
{
//This shortcut is NO longger valid after bool macher support NOT-logic.
//short cut for last scan and combination rules
//region_hit_num==1 : for current scan hitted rules, one and each other group may statisfy a compile rule.
//_mid->hit_group_cnt==1: With pre scan hitted group rules, one group may staisfy a compile rule
scan_ret=0;
}
else
{
scan_ret=bool_matcher_match(bm, thread_num,
_mid->hitted_group_id, _mid->hit_group_cnt,
(void **)array_mi_rule, MAX_SCANNER_HIT_NUM);
qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation**),
compare_compile_inner);
}
for(i=0;i<scan_ret&&result_cnt<size;i++)
{
_mi_rule=array_mi_rule[i];
if(_mi_rule==NULL)
relation=relation_array[i];
if(relation==NULL)
{
continue;
}
if(0==pthread_rwlock_tryrdlock(&(_mi_rule->rwlock)))
assert(relation->magic_num==COMPILE_RELATION_MAGIC);
if(0==pthread_rwlock_tryrdlock(&(relation->rwlock)))
{
if(_mi_rule->is_valid==1)
if(relation->compile)
{
if(_mi_rule->not_group_cnt>0 && !is_last_region)
if(relation->not_group_cnt>0 && !is_last_region)
{
_mid->not_grp_compile_hitted_flag=1;
}
else
{
make_group_set(_mi_rule, &(rs_result[result_cnt].group_set), &has_not_flag);
r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set), _mid->cur_hit_id, _mid->cur_hit_cnt,
make_group_set(relation, &(rs_result[result_cnt].group_set), &has_not_flag);
r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set), _mid->cur_hit_groups, _mid->cur_hit_group_cnt,
region_pos, MAX_SCANNER_HIT_NUM);
if(r_in_c_cnt>0 || //compile config hitted becasue of new reigon
_mid->cur_hit_cnt==0) //or ever hit a compile that refer a NOT-logic group
_mid->cur_hit_group_cnt==0) //or ever hit a compile that refer a NOT-logic group
{
fill_maat_rule(&(result[result_cnt]), &(_mi_rule->db_c_rule->m_rule_head),
_mi_rule->db_c_rule->service_defined ,_mi_rule->db_c_rule->m_rule_head.serv_def_len);
rs_result[result_cnt].compile_id=_mi_rule->compile_id;
fill_maat_rule(&(result[result_cnt]), &(relation->compile->head),
relation->compile->service_defined ,relation->compile->head.serv_def_len);
rs_result[result_cnt].compile_id=relation->compile_id;
result_cnt++;
}
}
}
pthread_rwlock_unlock(&(_mi_rule->rwlock));
pthread_rwlock_unlock(&(relation->rwlock));
}
}
if(result_cnt>0)
{
qsort(result, result_cnt, sizeof(struct Maat_rule_t),
compare_compile_id);
alignment_int64_array_add(feather->hit_cnt,thread_num,1);
alignment_int64_array_add(feather->hit_cnt, thread_num, 1);
}
if(region_hit_num==0&&result_cnt>0)
{
@@ -252,39 +252,34 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
return result_cnt;
}
int exprid2region_id(struct Maat_group_inner_t* group_rule,int expr_id,int* district_id)
int exprid2region_id(struct Maat_group_inner* group_rule,int expr_id,int* district_id, Maat_scanner* scanner)
{
int i=0,region_id=-1;
struct _Maat_region_inner_t* region_rule=NULL;
int region_id=-1;
struct Maat_region_inner* region_rule=NULL;
assert(group_rule->group_id>=0);
pthread_mutex_lock(&(group_rule->mutex));
for(i=0;i<group_rule->region_boundary;i++)
int array_idx=(int)(long)HASH_fetch_by_id(scanner->exprid_hash, expr_id);
pthread_mutex_lock(&(group_rule->mutex));
assert(array_idx<group_rule->region_boundary);
region_rule=(struct Maat_region_inner*)dynamic_array_read(group_rule->regions, array_idx);
if(region_rule)
{
region_rule=(struct _Maat_region_inner_t*)dynamic_array_read(group_rule->regions, i);
if(region_rule==NULL)
{
continue;
}
if(expr_id>=region_rule->expr_id_lb&&expr_id<=region_rule->expr_id_ub)
{
region_id=region_rule->region_id;
*district_id=region_rule->district_id;
break;
}
assert(expr_id>=region_rule->expr_id_lb&&expr_id<=region_rule->expr_id_ub);
region_id=region_rule->region_id;
*district_id=region_rule->district_id;
}
pthread_mutex_unlock(&(group_rule->mutex));
return region_id;
}
int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num)
int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num, Maat_scanner* scanner)
{
struct Maat_group_inner_t* group_rule=NULL;
struct Maat_group_inner* group_rule=NULL;
int i=0;
int district_id=-1,region_id=-1;
int ret_region_num=region_hit_num;
while(i<ret_region_num)
{
group_rule=(struct Maat_group_inner_t*)(region_hit[i].tag);
region_id=exprid2region_id(group_rule, region_hit[i].expr_id,&district_id);
group_rule=(struct Maat_group_inner*)(region_hit[i].tag);
region_id=exprid2region_id(group_rule, region_hit[i].expr_id,&district_id, scanner);
if(region_id>0&&district_id!=_mid->district_id)
{
ret_region_num--;
@@ -368,7 +363,8 @@ int hit_pos_RS2Maat(struct sub_item_pos_t* maat_sub_item,int size,rule_result_t*
int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mid,
scan_result_t *region_hit,int region_cnt,
_compile_result_t *compile_hit,int compile_cnt,
struct Maat_hit_detail_t *hit_detail,int detail_num)
struct Maat_hit_detail_t *hit_detail,int detail_num,
struct Maat_scanner* scanner)
{
int i=0,j=0,k=0;
char r_in_c_flag[region_cnt];
@@ -379,20 +375,20 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi
memset(r_in_c_flag,0,sizeof(r_in_c_flag));
memset(region_pos,0,sizeof(region_pos));
struct Maat_group_inner_t* group_rule=NULL;
struct Maat_group_inner* group_rule=NULL;
//for each hitted compile cfg,find its region_ids
for(i=0;i<compile_cnt&&i<detail_num;i++)
{
hit_detail[i].config_id=compile_hit[i].compile_id;
r_in_c_cnt=pickup_hit_region_from_compile(&(compile_hit[i].group_set), _mid->cur_hit_id, _mid->cur_hit_cnt,
r_in_c_cnt=pickup_hit_region_from_compile(&(compile_hit[i].group_set), _mid->cur_hit_groups, _mid->cur_hit_group_cnt,
region_pos, MAX_SCANNER_HIT_NUM);
assert(r_in_c_cnt>0);//previous hitted compile was elimited in region_compile
for(j=0,k=0;j<r_in_c_cnt&&k<MAAT_MAX_HIT_RULE_NUM;j++)
{
pos=region_pos[j];
r_in_c_flag[pos]=1;
group_rule=(struct Maat_group_inner_t*)(region_hit[pos].tag);
region_id=exprid2region_id(group_rule,region_hit[pos].expr_id,&district_id);
group_rule=(struct Maat_group_inner*)(region_hit[pos].tag);
region_id=exprid2region_id(group_rule,region_hit[pos].expr_id, &district_id, scanner);
if(region_id<0)
{
continue;
@@ -411,10 +407,10 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi
{
if(r_in_c_flag[k]==0)
{
group_rule=(struct Maat_group_inner_t*)(region_hit[k].tag);
group_rule=(struct Maat_group_inner*)(region_hit[k].tag);
hit_detail[j].config_id=-2;
hit_detail[j].hit_region_cnt=1;
hit_detail[j].region_pos[0].region_id=exprid2region_id(group_rule,region_hit[k].expr_id,&district_id);
hit_detail[j].region_pos[0].region_id=exprid2region_id(group_rule,region_hit[k].expr_id,&district_id, scanner);
hit_detail[j].region_pos[0].sub_item_num=region_hit[k].rnum;
hit_pos_RS2Maat(hit_detail[j].region_pos[0].sub_item_pos,MAAT_MAX_EXPR_ITEM_NUM,
region_hit[k].result,region_hit[k].rnum,scan_buff);
@@ -427,10 +423,11 @@ struct _INNER_scan_status_t* _make_inner_status(void)
{
struct _INNER_scan_status_t* inner_mid=NULL;
inner_mid=ALLOC(struct _INNER_scan_status_t, 1);
inner_mid->cur_hit_cnt=0;
inner_mid->hit_group_cnt=0;
inner_mid->hit_group_size=4;
inner_mid->hitted_group_id= ALLOC(unsigned long long, inner_mid->hit_group_size);
inner_mid->cur_hit_group_cnt=0;
inner_mid->cur_hit_groups=dynamic_array_create(32, 32);
inner_mid->all_hit_group_cnt=0;
inner_mid->all_hit_group_array_sz=4;
inner_mid->all_hit_group_array= ALLOC(unsigned long long, inner_mid->all_hit_group_array_sz);
return inner_mid;
}
struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather, int thread_num)
@@ -503,7 +500,12 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
return NULL;
}
_Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1);
feather->table_cnt=read_table_description(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger);
feather->table_cnt=read_table_description(feather->p_table_info, MAX_TABLE_NUM, table_info_path, max_thread_num, logger);
if(feather->table_cnt==0)
{
free(feather);
return NULL;
}
feather->map_tablename2id=map_create();
int i=0,j=0,ret=0;
for(i=0;i<MAX_TABLE_NUM;i++)
@@ -543,7 +545,6 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
feather->outer_mid_cnt=alignment_int64_array_alloc(max_thread_num);
feather->inner_mid_cnt=alignment_int64_array_alloc(max_thread_num);
feather->hit_cnt=alignment_int64_array_alloc(max_thread_num);
feather->orphan_group_saving=alignment_int64_array_alloc(max_thread_num);
feather->not_grp_hit_cnt=alignment_int64_array_alloc(max_thread_num);
feather->maat_version=0;
feather->last_full_version=0;
@@ -636,12 +637,12 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
break;
case MAAT_OPT_JSON_FILE_PATH:
assert(_feather->input_mode==SOURCE_NONE);
ret=json2iris((const char*)value
,_feather->compile_tn,_feather->group_tn
,NULL
,_feather->json_ctx.iris_file
,sizeof(_feather->json_ctx.iris_file)
,_feather->logger);
ret=json2iris((const char*)value,
_feather->compile_tn,_feather->group_tn,
NULL,
_feather->json_ctx.iris_file,
sizeof(_feather->json_ctx.iris_file),
_feather->logger);
if(ret<0)
{
return -1;
@@ -1042,19 +1043,21 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user)
{
struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user;
struct Maat_compile_inner_t *compile_inner=(struct Maat_compile_inner_t *)data;
struct Maat_compile_group_relation *relation=(struct Maat_compile_group_relation *)data;
struct Maat_compile_rule* compile_rule=relation->compile;
MAAT_RULE_EX_DATA ad=NULL;
if(compile_inner->ref_table==NULL || compile_inner->ref_table->table_id!=ex_desc->table_id || compile_inner->db_c_rule==NULL)
if(compile_rule==NULL|| compile_rule->ref_table->table_id!=ex_desc->table_id)
{
return;
}
pthread_rwlock_wrlock(&(compile_inner->rwlock));
ad=rule_ex_data_new(&(compile_inner->db_c_rule->m_rule_head),
compile_inner->db_c_rule->service_defined,
pthread_rwlock_wrlock(&(relation->rwlock));
ad=rule_ex_data_new(&(relation->compile->head),
relation->compile->service_defined,
ex_desc);
compile_inner->ads[ex_desc->idx]=ad;
pthread_rwlock_unlock(&(compile_inner->rwlock));
compile_rule->ads[ex_desc->idx]=ad;
pthread_rwlock_unlock(&(relation->rwlock));
return;
}
int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name,
@@ -1104,22 +1107,24 @@ failed:
MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx)
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct Maat_compile_inner_t *compile_inner=NULL;
struct Maat_compile_group_relation *relation=NULL;
struct Maat_compile_rule* compile=NULL;
const struct compile_table_desc* compile_desc=NULL;
const struct compile_ex_data_idx* ex_desc=NULL;
MAAT_RULE_EX_DATA ad=NULL;
compile_inner=(struct Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(compile_inner==NULL)
relation=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(relation==NULL||relation->compile==NULL)
{
return NULL;
}
pthread_rwlock_rdlock(&(compile_inner->rwlock));
compile_desc=&(compile_inner->ref_table->compile);
pthread_rwlock_rdlock(&(relation->rwlock));
compile=relation->compile;
compile_desc=&(compile->ref_table->compile);
assert(idx<compile_desc->ex_data_num);
ex_desc=compile_desc->ex_desc+idx;
ex_desc->dup_func(ex_desc->idx, &ad, compile_inner->ads+idx, ex_desc->argl,ex_desc->argp);
pthread_rwlock_unlock(&(compile_inner->rwlock));
ex_desc->dup_func(ex_desc->idx, &ad, compile->ads+idx, ex_desc->argl,ex_desc->argp);
pthread_rwlock_unlock(&(relation->rwlock));
return ad;
}
struct wrap_plugin_EX_data
@@ -1152,7 +1157,7 @@ MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_
MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs));
hargs.thread_safe=0;
hargs.thread_safe=8;
hargs.hash_slot_size = slot_size;
hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
@@ -1310,7 +1315,6 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id
return NULL;
}
table_rt= _feather->scanner->table_rt[table_id];
pthread_rwlock_rdlock(&table_rt->plugin.rwlock);
wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash,
(const unsigned char*)key, strlen(key));
if(wrap_data!=NULL)
@@ -1318,7 +1322,6 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id
plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata),
plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
}
pthread_rwlock_unlock(&table_rt->plugin.rwlock);
return exdata;
}
@@ -1337,7 +1340,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
struct Maat_table_desc *p_table=NULL;
struct expr_table_desc* expr_desc=NULL;
struct timespec start,end;
Maat_scanner_t* my_scanner=NULL;
Maat_scanner* my_scanner=NULL;
if(data==NULL||data_len<=0)
{
return 0;
@@ -1410,7 +1413,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
}
if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS)
{
hit_region_cnt=match_district(_mid,region_result,hit_region_cnt);
hit_region_cnt=match_district(_mid,region_result,hit_region_cnt, my_scanner);
}
if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
{
@@ -1435,7 +1438,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
*detail_ret=fill_region_hit_detail(data,_mid->inner,
region_result,hit_region_cnt,
compile_result,compile_ret,
hit_detail,detail_num);
hit_detail,detail_num, my_scanner);
}
}
if(_feather->perf_on==1)
@@ -1477,7 +1480,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_scanner_t* my_scanner=NULL;
struct Maat_scanner* my_scanner=NULL;
intval_scan_data.rule_type=RULETYPE_INT;
intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0);
intval_scan_data.int_data=intval;
@@ -1567,7 +1570,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
Maat_table_desc* p_table=NULL;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_scanner_t* my_scanner=NULL;
struct Maat_scanner* my_scanner=NULL;
struct timespec start,end;
if(_feather->perf_on==1)
{
@@ -1688,7 +1691,7 @@ int Maat_scan_addr(Maat_feather_t feather,int table_id
stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,int thread_num)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_scanner_t* scanner=NULL;
struct Maat_scanner* scanner=NULL;
struct Maat_table_desc *p_table=NULL;
assert(thread_num<_feather->scan_thread_num);
@@ -1746,7 +1749,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
,int* detail_ret,scan_status_t* mid)
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct Maat_scanner_t* scanner=sp->feather->scanner;
struct Maat_scanner* scanner=sp->feather->scanner;
int sub_type=0;
int region_ret=0,hit_region_cnt=0,compile_ret=0;
@@ -1855,7 +1858,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
}
if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS)
{
hit_region_cnt=match_district(_mid,region_result,hit_region_cnt);
hit_region_cnt=match_district(_mid, region_result, hit_region_cnt, scanner);
}
if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
{
@@ -1882,14 +1885,14 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
*detail_ret=fill_region_hit_detail(sp->scan_buff,_mid->inner,
region_result,hit_region_cnt,
compile_result,compile_ret,
hit_detail,detail_num);
hit_detail,detail_num, scanner);
}
else
{
*detail_ret=fill_region_hit_detail(data,_mid->inner,
region_result,hit_region_cnt,
compile_result,compile_ret,
hit_detail,detail_num);
hit_detail,detail_num, scanner);
}
}
}
@@ -1929,7 +1932,7 @@ int Maat_stream_scan_string(stream_para_t* stream_para
void Maat_stream_scan_string_end(stream_para_t* stream_para)
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct Maat_scanner_t* scanner=sp->feather->scanner;
struct Maat_scanner* scanner=sp->feather->scanner;
struct Maat_table_runtime* table_rt=NULL;
if(scanner!=NULL)
{
@@ -1971,7 +1974,7 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para)
stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,unsigned long long total_len,int thread_num)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_scanner_t* scanner=NULL;
struct Maat_scanner* scanner=NULL;
sfh_instance_t * tmp_fuzzy_handle=NULL;
struct Maat_table_desc *p_table=NULL;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST);
@@ -2130,7 +2133,7 @@ fast_out:
void Maat_stream_scan_digest_end(stream_para_t* stream_para)
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct Maat_scanner_t* scanner=sp->feather->scanner;
struct Maat_scanner* scanner=sp->feather->scanner;
struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id];
alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1);
if(scanner!=NULL)
@@ -2153,11 +2156,11 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum
{
int ret=0;
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct Maat_compile_inner_t *compile_inner=NULL;
struct Maat_compile_group_relation *compile_inner=NULL;
switch(type)
{
case MAAT_RULE_SERV_DEFINE:
compile_inner=(struct Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
compile_inner=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(compile_inner==NULL)
{
ret=0;
@@ -2165,8 +2168,8 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum
else
{
pthread_rwlock_rdlock(&(compile_inner->rwlock));
ret=MIN(size,compile_inner->db_c_rule->m_rule_head.serv_def_len);
memcpy(value,compile_inner->db_c_rule->service_defined,ret);
ret=MIN(size,compile_inner->compile->head.serv_def_len);
memcpy(value,compile_inner->compile->service_defined,ret);
pthread_rwlock_unlock(&(compile_inner->rwlock));
}
break;
@@ -2227,7 +2230,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
GIE_result_t region_result[MAX_SCANNER_HIT_NUM];
_compile_result_t compile_result[rule_num];
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_scanner_t* my_scanner=NULL;
struct Maat_scanner* my_scanner=NULL;
Maat_table_desc* p_table=NULL;
struct timespec start,end;
if(_feather->perf_on==1)

179
src/entry/Maat_command.cpp
View File

@@ -192,6 +192,9 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_
case TABLE_TYPE_IP:
column_seq=14;
break;
case TABLE_TYPE_IP_PLUS:
column_seq=18;
break;
case TABLE_TYPE_COMPILE:
column_seq=8;
break;
@@ -239,13 +242,39 @@ int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq)
line[i]='0';
return 0;
}
void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int size)
void serialize_group(const struct Maat_group_t* p_group, enum MAAT_OPERATION op, char* buff, size_t sz)
{
UNUSED int ret=0;
if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD;
snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d", p_group->group_id,
p_group->parent_id,
op,
p_group->not_flag,
p_group->parent_type);
return;
}
void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int group_num, enum MAAT_OPERATION op, char* buff, size_t sz)
{
if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD;
const char* service_define=huge_service_defined?huge_service_defined:(strlen(p_m_rule->service_defined)?p_m_rule->service_defined:"null");
snprintf(buff, sz,"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t%d\t%d",
p_m_rule->config_id,
p_m_rule->service_id,
p_m_rule->action,
p_m_rule->do_blacklist,
p_m_rule->do_log,
service_define,
op,
group_num);
return;
}
void serialize_region(const struct Maat_region_t* p, int group_id, char* buff, size_t sz)
{
UNUSED size_t ret=0;
switch(p->region_type)
{
case REGION_IP:
ret=snprintf(buff,size,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1"
ret=snprintf(buff,sz,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1"
,p->region_id
,group_id
,p->ip_rule.addr_type
@@ -263,7 +292,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
case REGION_EXPR:
if(p->expr_rule.district==NULL)
{
ret=snprintf(buff,size,"%d\t%d\t%s\t%d\t%d\t%d\t1"
ret=snprintf(buff,sz,"%d\t%d\t%s\t%d\t%d\t%d\t1"
,p->region_id
,group_id
,p->expr_rule.keywords
@@ -273,7 +302,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
}
else //expr_plus
{
ret=snprintf(buff,size,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1"
ret=snprintf(buff,sz,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1"
,p->region_id
,group_id
,p->expr_rule.keywords
@@ -284,14 +313,14 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
}
break;
case REGION_INTERVAL:
ret=snprintf(buff,size,"%d\t%d\t%u\t%u\t1"
ret=snprintf(buff,sz,"%d\t%d\t%u\t%u\t1"
,p->region_id
,group_id
,p->interval_rule.low_boundary
,p->interval_rule.up_boundary);
break;
case REGION_DIGEST:
ret=snprintf(buff,size,"%d\t%d\t%llu\t%s\t%hd\t1"
ret=snprintf(buff,sz,"%d\t%d\t%llu\t%s\t%hd\t1"
,p->region_id
,group_id
,p->digest_rule.orgin_len
@@ -299,7 +328,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
,p->digest_rule.confidence_degree);
break;
case REGION_SIMILARITY:
ret=snprintf(buff,size,"%d\t%d\t%s\t%hd\t1"
ret=snprintf(buff,sz,"%d\t%d\t%s\t%hd\t1"
,p->region_id
,group_id
,p->similarity_rule.target
@@ -308,7 +337,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
default:
assert(0);
}
assert(ret<size);
assert(ret<sz);
return;
}
void empty_serial_rules(struct serial_rule_t* rule)
@@ -329,15 +358,15 @@ void empty_serial_rules(struct serial_rule_t* rule)
memset(rule,0,sizeof(struct serial_rule_t));
return;
}
void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout)
void set_serial_rule(struct serial_rule_t* rule, enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout)
{
memset(rule, 0, sizeof(struct serial_rule_t));
rule->op=op;
rule->rule_id=rule_id;
rule->label_id=label_id;
rule->timeout=timeout;
assert(strlen(table_name)<sizeof(rule->table_name));
memset(rule->table_name, 0, sizeof(rule->table_name));
memcpy(rule->table_name,table_name,strlen(table_name));
strncpy(rule->table_name, table_name, sizeof(rule->table_name));
if(line!=NULL)
{
rule->table_line=_maat_strdup(line);
@@ -836,35 +865,35 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
struct Maat_group_t* group_cmd=NULL;
struct Maat_region_t* region_cmd=NULL;
struct Maat_compile_inner_t *compile_inner=NULL;
struct Maat_group_inner_t* group_inner=NULL;
struct _Maat_region_inner_t* region_inner=NULL;
struct Maat_compile_group_relation *relation=NULL;
struct Maat_group_inner* group_inner=NULL;
struct Maat_region_inner* region_inner=NULL;
void* logger=feather->logger;
int config_id=cmd->compile.config_id;
if(feather->scanner==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_command
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_command
,"MAAT not ready.");
return -1;
}
compile_inner=(struct Maat_compile_inner_t *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id);
//Operation on compile_inner is thread safe, no immediate memory free when delete a compile rule or a scanner.
//In another words, if the compile_inner is accessable from compile means, its was valid in at least 10 seconds (garbage bury).
if(compile_inner==NULL)
relation=(struct Maat_compile_group_relation *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id);
//Operation on relation is thread safe, no immediate memory free when delete a compile rule or a scanner.
//In another words, if the relation is accessable from compile means, its was valid in at least 10 seconds (garbage bury).
if(relation==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_command
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_command
,"config %d not exist."
,config_id);
return -1;
}
pthread_rwlock_rdlock(&(compile_inner->rwlock));
cmd->group_num=compile_inner->group_cnt;
pthread_rwlock_rdlock(&(relation->rwlock));
cmd->group_num=relation->group_cnt;
assert(cmd->groups==NULL);
cmd->groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),cmd->group_num);
for(i=0;i<compile_inner->group_boundary;i++)
cmd->groups=ALLOC(struct Maat_group_t, cmd->group_num);
for(i=0;i<relation->group_boundary;i++)
{
group_inner=(struct Maat_group_inner_t*)dynamic_array_read(compile_inner->groups,i);
group_inner=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i);
if(group_inner==NULL)
{
continue;
@@ -872,15 +901,15 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
group_cmd=&(cmd->groups[grp_idx]);
group_cmd->group_id=group_inner->group_id;
if(group_inner->ref_cnt>1)
if(group_inner->ref_by_parent_cnt>1)
{
continue;
}
group_cmd->region_num=group_inner->region_cnt;
group_cmd->regions=(struct Maat_region_t*)calloc(sizeof(struct Maat_region_t),group_cmd->region_num);
group_cmd->regions=ALLOC(struct Maat_region_t, group_cmd->region_num);
for(j=0;j<group_inner->region_boundary;j++)
{
region_inner=(struct _Maat_region_inner_t*)dynamic_array_read(group_inner->regions,j);
region_inner=(struct Maat_region_inner*)dynamic_array_read(group_inner->regions,j);
if(region_inner==NULL)
{
continue;
@@ -893,11 +922,11 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
}
grp_idx++;
}
pthread_rwlock_unlock(&(compile_inner->rwlock));
pthread_rwlock_unlock(&(relation->rwlock));
return 0;
}
int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,struct serial_rule_t* list, int size)
int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd, struct serial_rule_t* list, int size)
{
struct Maat_group_t* p_group=NULL;
struct Maat_region_t* p_region=NULL;
@@ -915,19 +944,12 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st
}
if(op==MAAT_OP_ADD)
{
snprintf(line,sizeof(line),"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t1\t%d",p_m_rule->config_id
,p_m_rule->service_id
,p_m_rule->action
,p_m_rule->do_blacklist
,p_m_rule->do_log
,(_cmd->huge_service_defined!=NULL)?(_cmd->huge_service_defined):(p_m_rule->service_defined)
,cmd->group_num);
set_serial_rule(list+rule_num,MAAT_OP_ADD,cmd->compile.config_id,cmd->label_id,feather->compile_tn,line,timeout);
serialize_compile(p_m_rule, _cmd->huge_service_defined, cmd->group_num, MAAT_OP_ADD, line, sizeof(line));
set_serial_rule(list+rule_num, MAAT_OP_ADD, cmd->compile.config_id, cmd->label_id, feather->compile_tn, line, timeout);
}
else
{
set_serial_rule(list+rule_num,op,cmd->compile.config_id,cmd->label_id,feather->compile_tn,NULL,timeout);
set_serial_rule(list+rule_num, op, cmd->compile.config_id, cmd->label_id, feather->compile_tn, NULL, timeout);
}
rule_num++;
for(i=0;i<cmd->group_num;i++)
@@ -940,9 +962,10 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st
p_group->group_id=feather->base_grp_seq;
feather->base_grp_seq++;
}
snprintf(line,sizeof(line),"%d\t%d\t1",p_group->group_id
,p_m_rule->config_id);
set_serial_rule(list+rule_num,MAAT_OP_ADD,p_group->group_id,0,feather->group_tn,line,timeout);
p_group->parent_id=p_m_rule->config_id;
p_group->parent_type=PARENT_TYPE_COMPILE;
serialize_group(p_group, MAAT_OP_ADD, line, sizeof(line));
set_serial_rule(list+rule_num, MAAT_OP_ADD, p_group->group_id, 0, feather->group_tn, line, timeout);
}
else
{
@@ -1358,10 +1381,10 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
struct Maat_region_t* p_region=NULL;
enum MAAT_TABLE_TYPE table_type;
struct Maat_compile_inner_t *compile_rule=NULL;
struct Maat_compile_group_relation *compile_rule=NULL;
if(feather->scanner!=NULL)
{
compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id);
compile_rule=(struct Maat_compile_group_relation*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id);
if(compile_rule!=NULL)
{
MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module
@@ -2056,8 +2079,8 @@ int Maat_cmd_set_group(Maat_feather_t feather,int group_id, const struct Maat_re
if(_feather->AUTO_NUMBERING_ON==1)
{
return -1;
}
//struct Maat_group_inner_t* group_inner=NULL;
}
//struct Maat_group_inner* group_inner=NULL;
//group_inner=(struct Maat_group_inner*)HASH_fetch_by_id(_feather->scanner->group_hash, group_id);
//NOT implemented yet.
assert(0);
@@ -2371,7 +2394,7 @@ int Maat_cmd_commit(Maat_feather_t feather)
_feather->server_time=redis_server_time(write_ctx);
if(_feather->AUTO_NUMBERING_ON==1)
{
{
data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_region_id_var, new_region_num);
if(data_reply->type!=REDIS_REPLY_INTEGER)
{
@@ -2381,7 +2404,7 @@ int Maat_cmd_commit(Maat_feather_t feather)
}
_feather->base_rgn_seq=data_reply->integer-new_region_num;
freeReplyObject(data_reply);
data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_group_id_var, new_group_num);
if(data_reply->type!=REDIS_REPLY_INTEGER)
{
@@ -2445,17 +2468,30 @@ long long Maat_cmd_incrby(Maat_feather_t feather,const char* key, int increment)
}
freeReplyObject(data_reply);
return result;
}
int Maat_cmd_get_new_group_id(Maat_feather_t feather)
{
int group_id=0;
group_id=(int) Maat_cmd_incrby(feather, mr_group_id_var, 1);
return group_id;
}
int Maat_cmd_get_new_region_id(Maat_feather_t feather)
{
int region_id=0;
region_id=(int) Maat_cmd_incrby(feather, mr_region_id_var, 1);
return region_id;
}
void Maat_cmd_key_free(struct Maat_cmd_key**keys, int size)
{
int i=0;
struct Maat_cmd_key* p=*keys;
struct Maat_cmd_key* p=*keys;
for(i=0; i<size; i++, p++)
{
free(p->table_name);
p->table_name=NULL;
p->rule_id=0;
}
}
free(*keys);
*keys=NULL;
return;
@@ -2547,9 +2583,9 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger)
redisAppendCommand(ctx,"SET MAAT_VERSION %lld",maat_redis_version);
append_cmd_cnt++;
redisAppendCommand(ctx,"SET MAAT_PRE_VER %lld",maat_redis_version);
append_cmd_cnt++;
append_cmd_cnt++;
redisAppendCommand(ctx,"SET %s 1", mr_region_id_var);
append_cmd_cnt++;
append_cmd_cnt++;
redisAppendCommand(ctx,"SET %s 1", mr_group_id_var);
append_cmd_cnt++;
redisAppendCommand(ctx,"EXEC");
@@ -2571,6 +2607,41 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger)
);
}
return redis_transaction_success;
}
static int _Maat_command_set_one_line(struct _Maat_feather_t* _feather, enum MAAT_OPERATION op, int id, const char* table_name, const char* line)
{
redisContext* write_ctx=get_redis_ctx_for_write(_feather);
_feather->server_time=redis_server_time(write_ctx);
struct serial_rule_t s_rule;
set_serial_rule(&s_rule, op, id, 0, table_name, line, 0);
int transaction_success=0;
transaction_success=exec_serial_rule(write_ctx, &s_rule, 1, _feather->server_time, _feather->logger);
empty_serial_rules(&s_rule);
return transaction_success;
}
int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num)
{
struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather;
char line[MAX_TABLE_LINE_SIZE];
serialize_compile(compile, huge_service_defined, group_num, op, line, sizeof(line));
int ret=_Maat_command_set_one_line(_feather, op, compile->config_id, table_name, line);
return ret;
}
int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_region_t* region, int group_id)
{
struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather;
char line[MAX_TABLE_LINE_SIZE];
serialize_region(region, group_id, line, sizeof(line));
int ret=_Maat_command_set_one_line(_feather, op, region->region_id, region->table_name, line);
return ret;
}
int Maat_command_raw_set_group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_group_t* group)
{
struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather;
char line[MAX_TABLE_LINE_SIZE];
serialize_group(group, op, line, sizeof(line));
int ret=_Maat_command_set_one_line(_feather, op, group->group_id*1024*1204+group->parent_id, group->table_name, line);
return ret;
}
int Maat_cmd_flushDB(Maat_feather_t feather)
{

1563
src/entry/Maat_rule.cpp
View File

File diff suppressed because it is too large Load Diff

7
src/entry/Maat_stat.cpp
View File

@@ -21,7 +21,6 @@ enum MAAT_FS_STATUS{
STATUS_ICONV_ERR_CNT,
STATUS_SCAN_ERR_CNT,
STATUS_ZOMBIE_RS_STREAM,
STATUS_ORPHAN_GROUP_SAVING,
STATUS_NOT_GROUP_HIT,
STATUS_CMD_NUM,
STATUS_CMD_Q_SIZE,
@@ -73,7 +72,6 @@ void maat_stat_init(struct _Maat_feather_t* feather)
feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid");
feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"z_stream");
feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"shortcut_sv");
feather->fs_status_id[STATUS_NOT_GROUP_HIT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"nt_grp_hit");
feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes");
@@ -168,7 +166,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0;
long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
long long outer_mid_cnt=0,inner_mid_cnt=0;
long long orphan_group_saving=0, not_grp_hit_cnt=0;
long long not_grp_hit_cnt=0;
long long total_update_error=0,total_iconv_error=0;
long long compile_rule_num=0, group_rule_num=0, not_group_rule_num=0, plugin_cache_num=0, plugin_acc_num=0;
int i=0;
@@ -183,7 +181,6 @@ void maat_stat_output(struct _Maat_feather_t* feather)
active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num);
inner_mid_cnt=alignment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num);
orphan_group_saving=alignment_int64_array_sum(feather->orphan_group_saving,feather->scan_thread_num);
not_grp_hit_cnt=alignment_int64_array_sum(feather->not_grp_hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);
@@ -191,7 +188,6 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING], 0,FS_OP_SET,orphan_group_saving);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0,FS_OP_SET,not_grp_hit_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_NUM], 0,FS_OP_SET,feather->cmd_acc_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_Q_SIZE], 0,FS_OP_SET,feather->cmd_q_cnt);
@@ -234,6 +230,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
total_iconv_error=p_table->expr.iconv_err_cnt;
break;
case TABLE_TYPE_IP:
case TABLE_TYPE_IP_PLUS:
table_regex_ipv6_num=table_rt->ip.ipv6_rule_cnt;
break;
default:

328
src/entry/json2iris.cpp
View File

@@ -83,7 +83,7 @@ static int get_group_seq(struct iris_description_t* iris_cfg)
}
else
{
data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_GROUP 1");
data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY %s 1", mr_group_id_var);
sequence=(int)data_reply->integer-1;
freeReplyObject(data_reply);
}
@@ -100,7 +100,7 @@ static int get_region_seq(struct iris_description_t* iris_cfg)
}
else
{
data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_REGION 1");
data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY %s 1", mr_region_id_var);
sequence=(int)data_reply->integer-1;
freeReplyObject(data_reply);
}
@@ -141,6 +141,7 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,
map_register(iris_cfg->str2int_map, "no",0);
map_register(iris_cfg->str2int_map, "ip",TABLE_TYPE_IP);
map_register(iris_cfg->str2int_map, "ip_plus",TABLE_TYPE_IP_PLUS);
map_register(iris_cfg->str2int_map, "string",TABLE_TYPE_EXPR);
map_register(iris_cfg->str2int_map, "expr",TABLE_TYPE_EXPR);
map_register(iris_cfg->str2int_map, "expr_plus",TABLE_TYPE_EXPR_PLUS);
@@ -321,7 +322,7 @@ error_out:
}
return -1;
}
int write_ip_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger)
int write_ip_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger)
{
struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
int cmd_cnt=0;
@@ -407,7 +408,119 @@ int write_ip_rule(cJSON *region_json,struct iris_description_t *p_iris,const cha
return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
}
int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,enum MAAT_TABLE_TYPE table_type,void * logger)
int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger)
{
struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
int cmd_cnt=0;
memset(json_cmd,0,sizeof(json_cmd));
json_cmd[cmd_cnt].json_string="region_id";
json_cmd[cmd_cnt].json_type=cJSON_Number;
cmd_cnt++;
json_cmd[cmd_cnt].json_string="group_id";
json_cmd[cmd_cnt].json_type=cJSON_Number;
cmd_cnt++;
json_cmd[cmd_cnt].json_string="addr_type";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].str2int_flag=1;
cmd_cnt++;
json_cmd[cmd_cnt].json_string="saddr_format";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_string="mask";
cmd_cnt++;
json_cmd[cmd_cnt].json_string="src_ip1";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_string="0.0.0.0";
cmd_cnt++;
json_cmd[cmd_cnt].json_string="src_ip2";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_string="255.255.255.255";
cmd_cnt++;
json_cmd[cmd_cnt].json_string="sport_format";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_string="mask";
cmd_cnt++;
json_cmd[cmd_cnt].json_string="src_port1";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_string="0";
cmd_cnt++;
json_cmd[cmd_cnt].json_string="src_port2";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_string="65535";
cmd_cnt++;
json_cmd[cmd_cnt].json_string="daddr_format";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_string="mask";
cmd_cnt++;
json_cmd[cmd_cnt].json_string="dst_ip1";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_string="0.0.0.0";
cmd_cnt++;
json_cmd[cmd_cnt].json_string="dst_ip2";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_string="255.255.255.255";
cmd_cnt++;
json_cmd[cmd_cnt].json_string="dport_format";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_string="mask";
cmd_cnt++;
json_cmd[cmd_cnt].json_string="dst_port1";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_string="0";
cmd_cnt++;
json_cmd[cmd_cnt].json_string="dst_port2";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_string="65535";
cmd_cnt++;
json_cmd[cmd_cnt].json_string="protocol";
json_cmd[cmd_cnt].json_type=cJSON_Number;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_int=0;
cmd_cnt++;
json_cmd[cmd_cnt].json_string="direction";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].str2int_flag=1;
json_cmd[cmd_cnt].empty_allowed=1;
json_cmd[cmd_cnt].default_string="double";
cmd_cnt++;
json_cmd[cmd_cnt].json_string="is_valid";
json_cmd[cmd_cnt].json_type=cJSON_Number;
cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
}
int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,enum MAAT_TABLE_TYPE table_type,void * logger)
{
struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
int cmd_cnt=0;
@@ -454,7 +567,7 @@ int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const c
return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
}
int write_intval_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger)
int write_intval_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger)
{
struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
int cmd_cnt=0;
@@ -483,7 +596,7 @@ int write_intval_rule(cJSON *region_json,struct iris_description_t *p_iris,const
return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
}
int write_digest_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger)
int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger)
{
struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
int cmd_cnt=0;
@@ -516,7 +629,7 @@ int write_digest_rule(cJSON *region_json,struct iris_description_t *p_iris,const
return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
}
int write_similar_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger)
int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger)
{
struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
int cmd_cnt=0;
@@ -546,7 +659,7 @@ int write_similar_rule(cJSON *region_json,struct iris_description_t *p_iris,cons
}
int write_plugin_table(cJSON* plug_table_json,int sequence,iris_description_t* p_iris,void* logger)
int write_plugin_line(cJSON* plug_table_json,int sequence,iris_description_t* p_iris,void* logger)
{
cJSON* item=NULL,*table_content=NULL,*each_line=NULL;
struct iris_table_t* table_info=NULL;
@@ -665,19 +778,22 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri
{
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
ret=write_expr_rule(table_content, p_iris, table_info->table_path,table_type, logger);
ret=write_expr_line(table_content, p_iris, table_info->table_path,table_type, logger);
break;
case TABLE_TYPE_IP:
ret=write_ip_rule(table_content, p_iris, table_info->table_path, logger);
ret=write_ip_line(table_content, p_iris, table_info->table_path, logger);
break;
case TABLE_TYPE_IP_PLUS:
write_ip_plus_line(table_content, p_iris, table_info->table_path, logger);
break;
case TABLE_TYPE_INTERVAL:
ret=write_intval_rule(table_content, p_iris, table_info->table_path, logger);
ret=write_intval_line(table_content, p_iris, table_info->table_path, logger);
break;
case TABLE_TYPE_DIGEST:
ret=write_digest_rule(table_content, p_iris, table_info->table_path, logger);
ret=write_digest_line(table_content, p_iris, table_info->table_path, logger);
break;
case TABLE_TYPE_SIMILARITY:
write_similar_rule(table_content, p_iris,table_info->table_path, logger);
write_similar_line(table_content, p_iris,table_info->table_path, logger);
break;
default:
assert(0);
@@ -691,7 +807,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri
return ret;
}
int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * logger)
int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void * logger)
{
int compile_id=-1,cmd_cnt=0,ret=-1;
cJSON* item=NULL;
@@ -772,7 +888,7 @@ int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * l
set_file_rulenum(table_info->table_path,table_info->line_count,logger);
return compile_id;
}
int write_group_rule(int compile_id ,int group_id, int group_not_flag, struct iris_description_t *p_iris,void * logger)
int write_group_line(int group_id, int parent_id, int group_not_flag, int parent_type, struct iris_description_t *p_iris, void * logger)
{
FILE*fp=NULL;
int ret=0;
@@ -792,7 +908,7 @@ int write_group_rule(int compile_id ,int group_id, int group_not_flag, struct ir
"fopen %s error %s.",p_iris->group_table->table_path,strerror(errno));
return -1;
}
fprintf(fp,"%d\t%d\t1\t%d\n",group_id, compile_id, group_not_flag);
fprintf(fp,"%d\t%d\t1\t%d\t%d\n",group_id, parent_id, group_not_flag, parent_type);
fclose(fp);
p_iris->group_table->line_count++;
ret=set_file_rulenum(p_iris->group_table->table_path,p_iris->group_table->line_count,logger);
@@ -818,17 +934,104 @@ int write_index_file(struct iris_description_t *p_iris,void* logger)
fclose(fp);
return 0;
}
int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger)
{
int i=0,j=0,k=0;
int compile_id=-1,compile_cnt=0,group_cnt=0,region_cnt=0,plug_table_cnt=0;
const char* _str_parent_type[2]={"compile", "group"};
int i=0;
int sub_group_cnt=0, region_cnt=0;
int ret=0;
int group_not_flag=0;
cJSON *c_rules=NULL,*g_rules=NULL,*r_rules=NULL,*item=NULL,*plug_tables=NULL;
cJSON *compile_rule=NULL,*group_rule=NULL,*region_rule=NULL,*each_plug_table=NULL;
cJSON *region_json=NULL, *item=NULL;
cJSON *sub_groups=NULL, *region_rule=NULL;
const char* group_name=NULL;
struct group_info_t *group_info=NULL;
struct group_info_t untitled_group;
item=cJSON_GetObjectItem(group_json, "group_name");
if(item==NULL||item->type!=cJSON_String)
{
group_name=untitled_group_name;
}
else
{
group_name=item->valuestring;
}
item=cJSON_GetObjectItem(group_json,"not_flag");
if(item==NULL||item->type!=cJSON_Number)
{
group_not_flag=0;
}
else
{
group_not_flag=item->valueint;
}
if(parent_type==PARENT_TYPE_GROUP)
{
group_not_flag=0;
}
group_info=(struct group_info_t*)MESA_htable_search(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name));
if(group_info==NULL)//exist group name, region already read
{
if(0==strncasecmp(group_name, untitled_group_name, strlen(untitled_group_name)))
{
group_info=&untitled_group;
group_info->group_id=get_group_seq(p_iris);
}
else
{
group_info=ALLOC(struct group_info_t, 1);
group_info->group_id=get_group_seq(p_iris);
MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info);
}
}
ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, p_iris, logger);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"%s rule %d write group error.", _str_parent_type[parent_type], parent_id);
return -1;
}
region_json=cJSON_GetObjectItem(group_json,"regions");
if(region_json!=NULL)
{
region_cnt=cJSON_GetArraySize(region_json);
for(i=0; i<region_cnt; i++)
{
region_rule=cJSON_GetArrayItem(region_json,i);
ret=write_region_rule(region_rule, tracking_compile_id, group_info->group_id, p_iris, logger);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"compile rule %d write region error.", tracking_compile_id);
return -1;
}
}
}
sub_groups=cJSON_GetObjectItem(group_json,"sub_groups");
if(sub_groups!=NULL)
{
//recursively
sub_group_cnt=cJSON_GetArraySize(sub_groups);
for(i=0; i<sub_group_cnt; i++)
{
item=cJSON_GetArrayItem(sub_groups, i);
ret=write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, p_iris, logger);
if(ret<0)
{
return -1;
}
}
}
return 0;
}
int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
{
int i=0,j=0;
int compile_id=-1, compile_cnt=0, group_cnt=0, plug_table_cnt=0;
int ret=0;
cJSON *c_rules=NULL, *g_rules=NULL, *plug_tables=NULL;
cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL;
plug_tables=cJSON_GetObjectItem(json,"plugin_table");
if(NULL!=plug_tables)
{
@@ -836,7 +1039,7 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
for(i=0;i<plug_table_cnt;i++)
{
each_plug_table=cJSON_GetArrayItem(plug_tables,i);
write_plugin_table(each_plug_table, i, p_iris, logger);
write_plugin_line(each_plug_table, i, p_iris, logger);
}
}
c_rules=cJSON_GetObjectItem(json,"rules");
@@ -857,7 +1060,7 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
for(i=0;i<compile_cnt;i++)
{
compile_rule=cJSON_GetArrayItem(c_rules,i);
compile_id=write_compile_rule(compile_rule,p_iris, logger);
compile_id=write_compile_line(compile_rule,p_iris, logger);
if(compile_id<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
@@ -881,87 +1084,11 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
for(j=0;j<group_cnt;j++)
{
group_rule=cJSON_GetArrayItem(g_rules,j);
item=cJSON_GetObjectItem(group_rule,"not_flag");
if(item==NULL||item->type!=cJSON_Number)
{
group_not_flag=0;
}
else
{
group_not_flag=item->valueint;
}
item=cJSON_GetObjectItem(group_rule,"group_name");
if(item==NULL||item->type!=cJSON_String)
{
group_name=untitled_group_name;
}
else
{
group_name=item->valuestring;
}
group_info=(struct group_info_t*)MESA_htable_search(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name));
if(group_info!=NULL)//exist group name ,region already read
{
ret=write_group_rule(compile_id, group_info->group_id, group_not_flag, p_iris, logger);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"compile rule %d write group error.",compile_id);
return -1;
}
r_rules=cJSON_GetObjectItem(group_rule,"regions");
if(r_rules!=NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_json,
"compile rule %d's %s declared in previous compile rule, regions NOT take effect."
,compile_id,group_name);
}
continue;
}
if(0==strncasecmp(group_name,untitled_group_name,strlen(untitled_group_name)))
{
group_info=&untitled_group;
group_info->group_id=get_group_seq(p_iris);
}
else
{
group_info=(struct group_info_t*)malloc(sizeof(struct group_info_t));
group_info->group_id=get_group_seq(p_iris);
MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info);
}
r_rules=cJSON_GetObjectItem(group_rule,"regions");
if(r_rules==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"compile rule %d's %s has no region.",compile_id,group_name);
return -1;
}
region_cnt=cJSON_GetArraySize(r_rules);
if(region_cnt<=0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"compile rule %d's %s has no region.",compile_id,group_name);
return -1;
}
for(k=0;k<region_cnt;k++)
{
region_rule=cJSON_GetArrayItem(r_rules,k);
ret=write_region_rule(region_rule, compile_id, group_info->group_id, p_iris, logger);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"compile rule %d write region error.",compile_id);
return -1;
}
}
ret=write_group_rule(compile_id, group_info->group_id, group_not_flag, p_iris, logger);
ret=write_group_rule(group_rule, compile_id, PARENT_TYPE_COMPILE, compile_id, p_iris, logger);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"compile rule %d write group error.",compile_id);
return -1;
}
}
}
ret=write_index_file(p_iris,logger);
@@ -971,6 +1098,7 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
}
return 0;
}
// redis_write_ctx is used by maat_redis_tool to write json to redis.
int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger)
{
FILE* json_fp=NULL;

123
src/inc_internal/Maat_rule_internal.h
View File

@@ -12,7 +12,7 @@
#include "dynamic_array.h"
#include "bool_matcher.h"
#include "hiredis.h"
#include "igraph/igraph.h"
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
#include "alignment_int64.h"
@@ -23,6 +23,8 @@
extern const char *maat_module;
#define mr_region_id_var "SEQUENCE_REGION"
#define mr_group_id_var "SEQUENCE_GROUP"
#define MAX_TABLE_NUM 256
@@ -69,14 +71,14 @@ struct db_ip_rule_t
};
int is_valid;
};
struct db_intval_rule_t
struct db_intval_rule
{
int region_id;
int group_id;
interval_rule_t intval;
int is_valid;
};
struct db_digest_rule_t
struct db_digest_rule
{
int region_id;
int group_id;
@@ -85,7 +87,7 @@ struct db_digest_rule_t
short confidence_degree;
int is_valid;
};
struct _head_Maat_rule_t
struct Maat_rule_head
{
int config_id;
int service_id;
@@ -95,20 +97,23 @@ struct _head_Maat_rule_t
char resevered;
int serv_def_len;
};
struct db_compile_rule_t
struct Maat_compile_rule
{
struct _head_Maat_rule_t m_rule_head;// fix len of Maat_rule_t
struct Maat_rule_head head;// fix len of Maat_rule_t
char* service_defined;
long long effective_range;
int is_valid;
int declare_grp_num;
int declared_grp_num;
const struct Maat_table_desc* ref_table;
MAAT_RULE_EX_DATA* ads;
};
struct db_group_rule_t
{
int group_id;
int compile_id;
int parent_id;
int is_valid;
int not_flag;
int parent_type; //PARENT_TYPE_**, 0:compile, 1: group.
};
struct op_expr_t
{
@@ -120,7 +125,7 @@ struct op_expr_t
int rule_type;
};
struct _Maat_region_inner_t
struct Maat_region_inner
{
int region_id;
int district_id;
@@ -130,32 +135,38 @@ struct _Maat_region_inner_t
int expr_id_ub;
enum MAAT_TABLE_TYPE table_type;
};
struct Maat_group_inner_t
#define COMPILE_RELATION_MAGIC 0x1a2b3c4d
struct Maat_compile_group_relation
{
long long magic_num;
struct Maat_compile_rule *compile;
dynamic_array_t *groups; //element is struct Maat_group_inner*
char not_flag[MAX_ITEMS_PER_BOOL_EXPR];
int compile_id;//equal to compile->m_rule.config_id
int group_boundary;
int group_cnt;
int not_group_cnt;
pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read
};
struct Maat_group_inner
{
int group_id;
int table_id;
int region_boundary;
int region_cnt;
int ref_cnt;
int ref_by_parent_cnt;
int ref_by_children_cnt;
char* group_name;
int has_compile_neighbors;
int vertex_id;
int top_group_cnt;
long long* top_groups;
dynamic_array_t *regions;
void* compile_shortcut;
pthread_mutex_t mutex;
};
struct Maat_compile_inner_t
{
struct db_compile_rule_t *db_c_rule;
dynamic_array_t *groups; //element is struct Maat_group_inner_t*
char not_flag[MAX_ITEMS_PER_BOOL_EXPR];
char is_valid;
int compile_id;//equal to db_c_rule->m_rule.config_id
const struct Maat_table_desc* ref_table;
int group_boundary;
int group_cnt;
int not_group_cnt;
MAAT_RULE_EX_DATA* ads;
pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read
};
struct _compile_result_t
{
int compile_id;
@@ -163,11 +174,11 @@ struct _compile_result_t
};
struct _INNER_scan_status_t
{
size_t cur_hit_cnt;
size_t hit_group_cnt;
size_t hit_group_size;
unsigned long long cur_hit_id[MAX_SCANNER_HIT_NUM];
unsigned long long *hitted_group_id;
size_t cur_hit_group_cnt;
size_t all_hit_group_cnt;
size_t all_hit_group_array_sz;
struct dynamic_array_t* cur_hit_groups;
unsigned long long *all_hit_group_array;
char not_grp_compile_hitted_flag;
};
struct _OUTER_scan_status_t
@@ -184,6 +195,7 @@ enum maat_garbage_type
GARBAGE_SCANNER=0,
GARBAGE_GROUP_RULE,
GARBAGE_COMPILE_RULE,
GARBAGE_COMPILE_GOURP_RELATION,
GARBAGE_BOOL_MATCHER,
GARBAGE_MAP_STR2INT,
GARBAGE_FOREIGN_FILE
@@ -218,13 +230,13 @@ struct similar_runtime
GIE_handle_t* gie_handle;
MESA_lqueue_head update_q;
};
struct plugin_runtime
{
dynamic_array_t *cache_lines;
long long cache_line_num;
long long acc_line_num;
long long cache_size;
pthread_rwlock_t rwlock;
MESA_htable_handle key2ex_hash;
};
struct expr_runtime
@@ -252,7 +264,7 @@ struct Maat_table_runtime
struct plugin_runtime plugin;
struct expr_runtime expr;
struct ip_runtime ip;
struct group_runtime group;
struct group_runtime group;
void * other;
};
mcore_long_t scan_cnt;
@@ -266,21 +278,33 @@ struct rule_tag
char* tag_name;
char* tag_val;
};
struct Maat_scanner_t
struct Maat_scanner
{
long long version;
time_t last_update_time;
mcore_long_t ref_cnt;
rule_scanner_t region;
long gie_total_q_size;
size_t gie_update_q_size;
size_t to_update_group_cnt;
size_t to_update_compile_cnt;
struct Maat_table_runtime* table_rt[MAX_TABLE_NUM];
MESA_htable_handle region_hash;
MESA_htable_handle group_hash;
MESA_htable_handle compile_hash;
MESA_htable_handle region_hash; //key: region_id, value: struct region_group_relation*
MESA_htable_handle exprid_hash; //key: expr_id, value: int array_idx of Maat_group_inner->regions;
MESA_htable_handle group_hash; //key: group_id, value: struct Maat_group_inner*
MESA_htable_handle compile_hash;//key: compile_id, value: struct Maat_compile_group_relation *
MESA_htable_handle district_map;
MESA_htable_handle tmp_district_map;
MESA_htable_handle vertex_id2group;
igraph_t group_graph;
int grp_vertex_id_generator;
int most_popular_sub_group;
long long max_presented_top_group_cnt;
unsigned int district_num;
unsigned int cfg_num;
unsigned int exprid_generator;
@@ -288,6 +312,7 @@ struct Maat_scanner_t
MESA_lqueue_head region_update_q;
struct bool_matcher * bool_matcher_expr_compiler;
scan_result_t *region_rslt_buff;
void* logger_ref;
MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q
int max_thread_num;
@@ -323,8 +348,8 @@ struct source_redis_ctx
};
struct _Maat_feather_t
{
struct Maat_scanner_t *scanner;
struct Maat_scanner_t *update_tmp_scanner;
struct Maat_scanner *scanner;
struct Maat_scanner *update_tmp_scanner;
MESA_lqueue_head garbage_q;
int table_cnt;
int DEFERRED_LOAD_ON;
@@ -390,7 +415,6 @@ struct _Maat_feather_t
mcore_long_t inner_mid_cnt;
mcore_long_t hit_cnt;
mcore_long_t thread_call_cnt;//size indicate by scan_thread_num,
mcore_long_t orphan_group_saving;
mcore_long_t not_grp_hit_cnt;
long long total_scan_bytes;
long long total_scan_cnt;
@@ -411,9 +435,10 @@ struct _maat_garbage_t
int ok_times;
union
{
struct Maat_scanner_t* scanner;
struct Maat_group_inner_t* group_rule;
struct Maat_compile_inner_t* compile_rule;
struct Maat_scanner* scanner;
struct Maat_group_inner* group_rule;
struct Maat_compile_rule* compile_rule;
struct Maat_compile_group_relation * compile_group_relation;
struct bool_matcher* bool_matcher;
void * raw;
MESA_htable_handle str2int_map;
@@ -443,7 +468,7 @@ int parse_accept_tag(const char* value, struct rule_tag** result, void* logger);
void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(struct Maat_compile_inner_t* compile_rule, struct bool_expr* a_set, unsigned char *has_not);
void make_group_set(struct Maat_compile_group_relation* compile_rule, struct bool_expr* a_set, unsigned char *has_not);
int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
void maat_start_cb(long long new_version,int update_type,void*u_para);
int maat_update_cb(const char* table_name,const char* line,void *u_para);
@@ -466,9 +491,9 @@ int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_lis
void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, int print_fn, void *logger);
void rewrite_table_line_with_foreign(struct serial_rule_t*p);
void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len);
MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc);
void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc);
void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len);
MAAT_RULE_EX_DATA rule_ex_data_new(const struct Maat_rule_head * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc);
void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc);
MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index);
int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger);
int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger);

1
src/inc_internal/Maat_table_description.h
View File

@@ -18,6 +18,7 @@ enum MAAT_TABLE_TYPE
{
TABLE_TYPE_EXPR=0,
TABLE_TYPE_IP,
TABLE_TYPE_IP_PLUS,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_DIGEST,
TABLE_TYPE_EXPR_PLUS,

52
src/inc_internal/view_only/rulescan.h
View File

@@ -28,7 +28,9 @@ extern "C"
{
RULESCAN_DETAIL_RESULT=1, /* <20><><EFBFBD><EFBFBD>־λ<D6BE><CEBB>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ϸ<EFBFBD><CFB8><EFBFBD><EFBFBD>λ<EFBFBD>õ<EFBFBD><C3B5><EFBFBD>Ϣ, optval<61><6C>ΪNULL<4C><4C>optlen<65><6E>Ϊ0<CEAA><30>Ĭ<EFBFBD>ϲ<EFBFBD><CFB2><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ϸ<EFBFBD><CFB8>Ϣ*/
RULESCAN_REGEX_GROUP =2, /* <20><><EFBFBD><EFBFBD>־λ<D6BE><CEBB>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽƥ<CABD><C6A5><EFBFBD>ķ<EFBFBD><C4B7><EFBFBD><EFBFBD><EFBFBD>Ϣ<EFBFBD><CFA2><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֶΣ<D6B6><CEA3><EFBFBD>Ҫ<EFBFBD><D2AA><EFBFBD><EFBFBD><EFBFBD><EFBFBD>RULESCAN_DETAIL_RESULT<4C><54>־λ,optval<61><6C>ΪNULL<4C><4C>optlen<65><6E>Ϊ0<CEAA><30>Ĭ<EFBFBD>ϲ<EFBFBD><CFB2><EFBFBD><EFBFBD>ط<EFBFBD><D8B7><EFBFBD><EFBFBD><EFBFBD>Ϣ */
RULESCAN_QUICK_SCAN /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ҫ<EFBFBD><EFBFBD><EFBFBD><EFBFBD>ɨ<EFBFBD><EFBFBD>ģʽ<EFBFBD><EFBFBD>sub_type, <20><><EFBFBD>û<EFBFBD><EFBFBD>Լ<EFBFBD><EFBFBD><EFBFBD><EFBFBD>壬optval<EFBFBD><EFBFBD>ֵΪ0-4096<EFBFBD><EFBFBD>optlen<EFBFBD><EFBFBD>Ϊ4<EFBFBD><EFBFBD>Ĭ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͨɨ<EFBFBD><EFBFBD>ģʽ<EFBFBD><EFBFBD>*/
RULEACAN_ERRLOG_CLOSE, /* <20><><EFBFBD><EFBFBD>־λ<EFBFBD><EFBFBD>ʾ<EFBFBD><EFBFBD><EFBFBD>ر<EFBFBD>Rulescan<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>־<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>optval<EFBFBD><EFBFBD>ΪNULL<EFBFBD><EFBFBD>optlen<EFBFBD><EFBFBD>Ϊ0<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>õĻ<EFBFBD>Ĭ<EFBFBD>ϴ<EFBFBD><EFBFBD><EFBFBD>Rulescan<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>־<EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
RULESCAN_ERRLOG_FILE_PATH, /* <20><><EFBFBD><EFBFBD>Rulescan<61><6E><EFBFBD><EFBFBD><EFBFBD><EFBFBD>־<EFBFBD><D6BE>·<EFBFBD><C2B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>û<EFBFBD><C3BB><EFBFBD><EFBFBD>룬optval<61><6C>ֵΪ<D6B5><CEAA><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>־·<D6BE><C2B7><EFBFBD><EFBFBD>optlenΪ·<CEAA><C2B7><EFBFBD><EFBFBD><EFBFBD>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD><EFBFBD>û<EFBFBD><C3BB><EFBFBD>趨<EFBFBD><E8B6A8>
<09><><EFBFBD><EFBFBD>־Ĭ<D6BE>ϴ洢<CFB4>ڿ<EFBFBD>ִ<EFBFBD>г<EFBFBD><D0B3><EFBFBD><EFBFBD><EFBFBD>ǰĿ¼<C4BF>µ<EFBFBD>rulescan_tmp<6D><70> */
};
#define MAX_REGEX_GROUP_NUM 5 /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD><EFBFBD>֧<EFBFBD>ֵ<EFBFBD><D6B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ĸ<EFBFBD><C4B8><EFBFBD> */
@@ -45,7 +47,7 @@ extern "C"
const unsigned int RULETYPE_IPv6 = 4; /* IPv6<76><36><EFBFBD><EFBFBD> */
const unsigned int MAX_RULETYPE = 5; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
const unsigned int MAX_SUB_RULETYPE = 4096; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
const unsigned int MAX_SUB_RULETYPE = 4096; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
/* <20>ַ<EFBFBD><D6B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD>͹<EFBFBD><CDB9>򣨿ɱ<F2A3A8BF>ʾ<EFBFBD>ı<EFBFBD><C4B1>ַ<EFBFBD><D6B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ַ<EFBFBD><D6B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD> */
typedef struct _string_rule_t
@@ -65,34 +67,34 @@ extern "C"
unsigned int ub; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>½磨<C2BD><E7A3A8><EFBFBD><EFBFBD>ub<75><62><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ĭ<EFBFBD><C4AC>Ϊ0 */
}interval_rule_t;
/* <EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>IPv4<EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
/* IPv4<76><34><EFBFBD><EFBFBD> */
typedef struct _ipv4_rule_t
{
unsigned int saddr; /* ԴIP<EFBFBD><EFBFBD>ַ<EFBFBD><EFBFBD>0<EFBFBD><EFBFBD>ʾ<EFBFBD><EFBFBD><EFBFBD>Ա<EFBFBD><EFBFBD>ֶ<EFBFBD> */
unsigned int smask; /* ԴIP<EFBFBD><EFBFBD>ַ<EFBFBD><EFBFBD><EFBFBD>룻0<EFBFBD><EFBFBD>ʾ<EFBFBD>̶<EFBFBD>IP=saddr */
unsigned int daddr; /* Ŀ<><EFBFBD>IP<EFBFBD><EFBFBD>ַ<EFBFBD><EFBFBD>0<EFBFBD><EFBFBD>ʾ<EFBFBD><EFBFBD><EFBFBD>Ա<EFBFBD><EFBFBD>ֶ<EFBFBD> */
unsigned int dmask; /* Ŀ<><EFBFBD>IP<EFBFBD><EFBFBD>ַ<EFBFBD><EFBFBD><EFBFBD>룻0<EFBFBD><EFBFBD>ʾ<EFBFBD>̶<EFBFBD>IP=daddr */
unsigned short int min_sport; /* Դ<>˿ڷ<CBBF>Χ<EFBFBD>½磻0<E7A3BB><30>ʾ<EFBFBD><CABE><EFBFBD>Ա<EFBFBD><D4B1>ֶ<EFBFBD> */
unsigned short int max_sport; /* Դ<>˿ڷ<CBBF>Χ<EFBFBD>Ͻ磻0<E7A3BB><30>ʾ<EFBFBD>̶<EFBFBD><CCB6>˿<EFBFBD>=min_sport */
unsigned short int min_dport; /* Ŀ<>Ķ˿ڷ<CBBF>Χ<EFBFBD>½磻0<E7A3BB><30>ʾ<EFBFBD><CABE><EFBFBD>Ա<EFBFBD><D4B1>ֶ<EFBFBD> */
unsigned short int max_dport; /* Ŀ<>Ķ˿ڷ<CBBF>Χ<EFBFBD>Ͻ磻0<E7A3BB><30>ʾ<EFBFBD>̶<EFBFBD><CCB6>˿<EFBFBD>=min_dport */
unsigned short int proto; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Э<EFBFBD>飬6<E9A3AC><36>ʾTCP<43><50>17<31><37>ʾUDP<44><50>0<EFBFBD><30>ʾ<EFBFBD><CABE><EFBFBD>Ա<EFBFBD><D4B1>ֶ<EFBFBD> */
unsigned short int direction; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>0<EFBFBD><30>ʾ˫<CABE><CBAB><EFBFBD><EFBFBD>1<EFBFBD><31>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD> */
unsigned int min_saddr; /* Դ<><D4B4>ַ<EFBFBD>½磻0<EFBFBD><EFBFBD>ʾ<EFBFBD><EFBFBD><EFBFBD>Ա<EFBFBD><EFBFBD>ֶ<EFBFBD> */
unsigned int max_saddr; /* Դ<><D4B4>ַ<EFBFBD>Ͻ磻0<EFBFBD><EFBFBD>ʾ<EFBFBD>̶<EFBFBD>IP=min_saddr */
unsigned int min_daddr; /* Ŀ<>ĵ<EFBFBD>ַ<EFBFBD>½磻0<EFBFBD><EFBFBD>ʾ<EFBFBD><EFBFBD><EFBFBD>Ա<EFBFBD><EFBFBD>ֶ<EFBFBD> */
unsigned int max_daddr; /* Ŀ<>ĵ<EFBFBD>ַ<EFBFBD>Ͻ磻0<EFBFBD><EFBFBD>ʾ<EFBFBD>̶<EFBFBD>IP=min_daddr */
unsigned short min_sport; /* Դ<>˿ڷ<CBBF>Χ<EFBFBD>½磻0<E7A3BB><30>ʾ<EFBFBD><CABE><EFBFBD>Ա<EFBFBD><D4B1>ֶ<EFBFBD> */
unsigned short max_sport; /* Դ<>˿ڷ<CBBF>Χ<EFBFBD>Ͻ磻0<E7A3BB><30>ʾ<EFBFBD>̶<EFBFBD><CCB6>˿<EFBFBD>=min_sport */
unsigned short min_dport; /* Ŀ<>Ķ˿ڷ<CBBF>Χ<EFBFBD>½磻0<E7A3BB><30>ʾ<EFBFBD><CABE><EFBFBD>Ա<EFBFBD><D4B1>ֶ<EFBFBD> */
unsigned short max_dport; /* Ŀ<>Ķ˿ڷ<CBBF>Χ<EFBFBD>Ͻ磻0<E7A3BB><30>ʾ<EFBFBD>̶<EFBFBD><CCB6>˿<EFBFBD>=min_dport */
unsigned short proto; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Э<EFBFBD>飬6<E9A3AC><36>ʾTCP<43><50>17<31><37>ʾUDP<44><50>0<EFBFBD><30>ʾ<EFBFBD><CABE><EFBFBD>Ա<EFBFBD><D4B1>ֶ<EFBFBD> */
unsigned short direction; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>0<EFBFBD><30>ʾ˫<CABE><CBAB><EFBFBD><EFBFBD>1<EFBFBD><31>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD> */
}ipv4_rule_t;
/* <EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>IPv6<EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
/* IPv6<76><36><EFBFBD><EFBFBD> */
typedef struct _ipv6_rule_t
{
unsigned int saddr[4]; /* ԴIP<EFBFBD><EFBFBD>ַ<EFBFBD><EFBFBD>0<EFBFBD><EFBFBD>ʾ<EFBFBD><EFBFBD><EFBFBD>Ա<EFBFBD><EFBFBD>ֶ<EFBFBD> */
unsigned int smask[4]; /* ԴIP<EFBFBD><EFBFBD>ַ<EFBFBD><EFBFBD><EFBFBD>룻0<EFBFBD><EFBFBD>ʾ<EFBFBD>̶<EFBFBD>IP=saddr */
unsigned int daddr[4]; /* Ŀ<><EFBFBD>IP<EFBFBD><EFBFBD>ַ<EFBFBD><EFBFBD>0<EFBFBD><EFBFBD>ʾ<EFBFBD><EFBFBD><EFBFBD>Ա<EFBFBD><EFBFBD>ֶ<EFBFBD> */
unsigned int dmask[4]; /* Ŀ<><EFBFBD>IP<EFBFBD><EFBFBD>ַ<EFBFBD><EFBFBD><EFBFBD>룻0<EFBFBD><EFBFBD>ʾ<EFBFBD>̶<EFBFBD>IP=daddr */
unsigned short int min_sport; /* Դ<>˿ڷ<CBBF>Χ<EFBFBD>½磻0<E7A3BB><30>ʾ<EFBFBD><CABE><EFBFBD>Ա<EFBFBD><D4B1>ֶ<EFBFBD> */
unsigned short int max_sport; /* Դ<>˿ڷ<CBBF>Χ<EFBFBD>Ͻ磻0<E7A3BB><30>ʾ<EFBFBD>̶<EFBFBD><CCB6>˿<EFBFBD>=min_sport */
unsigned short int min_dport; /* Ŀ<>Ķ˿ڷ<CBBF>Χ<EFBFBD>½磻0<E7A3BB><30>ʾ<EFBFBD><CABE><EFBFBD>Ա<EFBFBD><D4B1>ֶ<EFBFBD> */
unsigned short int max_dport; /* Ŀ<>Ķ˿ڷ<CBBF>Χ<EFBFBD>Ͻ磻0<E7A3BB><30>ʾ<EFBFBD>̶<EFBFBD><CCB6>˿<EFBFBD>=min_dport */
unsigned short int proto; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Э<EFBFBD>飬6<E9A3AC><36>ʾTCP<43><50>17<31><37>ʾUDP<44><50><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ĭ<EFBFBD><C4AC>Ϊ0 */
unsigned short int direction; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>0<EFBFBD><30>ʾ˫<CABE><CBAB><EFBFBD><EFBFBD>1<EFBFBD><31>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD> */
unsigned int min_saddr[4]; /* Դ<><D4B4>ַ<EFBFBD>½磻ȫ0<EFBFBD><EFBFBD>ʾ<EFBFBD><EFBFBD><EFBFBD>Ա<EFBFBD><EFBFBD>ֶ<EFBFBD> */
unsigned int max_saddr[4]; /* Դ<><D4B4>ַ<EFBFBD>Ͻ磻ȫ0<EFBFBD><EFBFBD>ʾ<EFBFBD>̶<EFBFBD>IP=min_saddr */
unsigned int min_daddr[4]; /* Ŀ<>ĵ<EFBFBD>ַ<EFBFBD>½磻ȫ0<EFBFBD><EFBFBD>ʾ<EFBFBD><EFBFBD><EFBFBD>Ա<EFBFBD><EFBFBD>ֶ<EFBFBD> */
unsigned int max_daddr[4]; /* Ŀ<>ĵ<EFBFBD>ַ<EFBFBD>Ͻ磻ȫ0<EFBFBD><EFBFBD>ʾ<EFBFBD>̶<EFBFBD>IP=min_daddr */
unsigned short min_sport; /* Դ<>˿ڷ<CBBF>Χ<EFBFBD>½磻0<E7A3BB><30>ʾ<EFBFBD><CABE><EFBFBD>Ա<EFBFBD><D4B1>ֶ<EFBFBD> */
unsigned short max_sport; /* Դ<>˿ڷ<CBBF>Χ<EFBFBD>Ͻ磻0<E7A3BB><30>ʾ<EFBFBD>̶<EFBFBD><CCB6>˿<EFBFBD>=min_sport */
unsigned short min_dport; /* Ŀ<>Ķ˿ڷ<CBBF>Χ<EFBFBD>½磻0<E7A3BB><30>ʾ<EFBFBD><CABE><EFBFBD>Ա<EFBFBD><D4B1>ֶ<EFBFBD> */
unsigned short max_dport; /* Ŀ<>Ķ˿ڷ<CBBF>Χ<EFBFBD>Ͻ磻0<E7A3BB><30>ʾ<EFBFBD>̶<EFBFBD><CCB6>˿<EFBFBD>=min_dport */
unsigned short proto; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Э<EFBFBD>飬6<E9A3AC><36>ʾTCP<43><50>17<31><37>ʾUDP<44><50><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ĭ<EFBFBD><C4AC>Ϊ0 */
unsigned short direction; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>0<EFBFBD><30>ʾ˫<CABE><CBAB><EFBFBD><EFBFBD>1<EFBFBD><31>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD> */
}ipv6_rule_t;
/* ͨ<>õĹ<C3B5><C4B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
@@ -188,7 +190,7 @@ extern "C"
unsigned int length[MAX_MATCH_POS_NUM]; /* <20>ù<EFBFBD><C3B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>н<EFBFBD><D0BD><EFBFBD><EFBFBD>ij<EFBFBD><C4B3>ȣ<EFBFBD><C8A3><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>н<EFBFBD><D0BD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ǰ<EFBFBD><C7B0><EFBFBD><EFBFBD><EFBFBD>ݰ<EFBFBD><DDB0><EFBFBD><EFBFBD><EFBFBD><EFBFBD>еģ<D0B5><C4A3><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ö<EFBFBD>Ӧ<EFBFBD><D3A6>length=0, <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>IP<49>࣬<EFBFBD><E0A3AC><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ϊ0*/
}rule_result_t;
/* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͣ<EFBFBD>quickģʽ<EFBFBD><EFBFBD>ֻ<EFBFBD><EFBFBD>expr_id<EFBFBD>Լ<EFBFBD>tag<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ч */
/* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
typedef struct _scan_result_t
{
unsigned int expr_id; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD>ID */

1
src/version.map
View File

@@ -1,6 +1,7 @@
VERS_2.4{
global:
extern "C++" {
*MAAT_FRAME_VERSION_*;
*Maat_*;
*SFH_*;
*GIE_*;

3
test/CMakeLists.txt
View File

@@ -1,6 +1,9 @@
add_executable(maat_demo maat_demo.cpp)
target_link_libraries(maat_demo maat_frame_shared)
add_executable(test_igraph test_igraph.cpp)
target_link_libraries(test_igraph igraph-static)
add_executable(test_maatframe test_maatframe.cpp)
target_link_libraries(test_maatframe maat_frame_shared gtest)

292
test/maat_json.json
View File

@@ -12,7 +12,7 @@
"is_valid": "yes",
"groups": [
{
"group_name": "IP_group",
"group_name": "123_IP_group",
"regions": [
{
"table_name": "IP_CONFIG",
@@ -77,7 +77,7 @@
"is_valid": "yes",
"groups": [
{
"group_name": "IP_group"
"group_name": "123_IP_group"
},
{
"group_name": "Untitled",
@@ -144,7 +144,7 @@
]
},
{
"group_name": "Untitled",
"group_name": "126_interval_group",
"regions": [
{
"table_name": "CONTENT_SIZE",
@@ -555,7 +555,7 @@
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "anything",
"user_region": "StringScan.UTF8EncodedURL",
"is_valid": "yes",
"groups": [
{
@@ -580,7 +580,7 @@
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "NOT-logic-test1",
"user_region": "NOTLogic.OneRegion",
"is_valid": "yes",
"groups": [
{
@@ -595,7 +595,7 @@
"match_method": "sub",
"format": "uncase plain"
}
}
}
]
},
{
@@ -621,7 +621,7 @@
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "NOT-logic-test2",
"user_region": "NOTLogic.ScanNotAtLast",
"is_valid": "yes",
"groups": [
{
@@ -662,7 +662,7 @@
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "NOT-logic-test3",
"user_region": "NOTLogic.ScanNotIP",
"is_valid": "yes",
"groups": [
{
@@ -682,7 +682,7 @@
},
{
"not_flag":1,
"group_name": "IP_group"
"group_name": "123_IP_group"
}
]
},
@@ -692,7 +692,7 @@
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "anything",
"user_region": "StringScan.Regex",
"is_valid": "yes",
"groups": [
{
@@ -717,7 +717,7 @@
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "anything",
"user_region": "StringScan.UTF8EncodedURL",
"is_valid": "yes",
"groups": [
{
@@ -744,7 +744,7 @@
"compile_id": 148,
"service": 0,
"do_blacklist": 0,
"user_region": "APP_ID=100001;BEHAV_ID=100002",
"user_region": "StringScan.ExprPlusWithOffset",
"groups": [
{
"regions": [
@@ -770,7 +770,7 @@
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "anything",
"user_region": "StringScan.Regex",
"is_valid": "yes",
"groups": [
{
@@ -796,7 +796,7 @@
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "0",
"user_region": "StringScan.BugReport20190325",
"is_valid": "yes",
"groups": [
{
@@ -838,7 +838,7 @@
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "suffix_hit",
"user_region": "StringScan.PrefixAndSuffix",
"is_valid": "yes",
"groups": [
{
@@ -865,7 +865,7 @@
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "prefix_hit",
"user_region": "StringScan.PrefixAndSuffix",
"is_valid": "yes",
"groups": [
{
@@ -879,9 +879,267 @@
"format": "uncase plain",
"match_method": "prefix"
}
},
{
"table_type": "expr",
"table_name": "MAIL_ADDR",
"table_content": {
"keywords": "ceshi6@mailhost.cn",
"expr_type": "none",
"format": "uncase plain",
"match_method": "prefix"
}
}
],
"group_name": "Untitled"
"group_name": "152_mail_addr"
},
{
"group_name": "interval_group_refered",
"sub_groups": [
{"group_name": "126_interval_group"}
]
}
]
},
{
"compile_id": 153,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "Policy.SubGroup",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "expr",
"table_name": "MAIL_ADDR",
"table_content": {
"keywords": "ceshi4@mailhost.cn",
"expr_type": "none",
"format": "uncase plain",
"match_method": "prefix"
}
}
],
"group_name": "Untitled",
"sub_groups": [
{"group_name": "152_mail_addr"}
],
"not_flag" : 0
},
{
"group_name": "IP_group_refered",
"sub_groups": [
{"group_name": "123_IP_group"}
]
}
]
},
{
"compile_id": 154,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "ipv4_plus",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "ip_plus",
"table_name": "IP_PLUS_CONFIG",
"table_content": {
"addr_type": "ipv4",
"saddr_format": "range",
"src_ip1": "10.0.7.100",
"src_ip2": "10.0.7.106",
"sport_format": "range",
"src_port1": "5000",
"src_port2": "5001",
"daddr_format": "mask",
"dst_ip1": "123.56.104.218",
"dst_ip2": "255.255.255.0",
"dport_format": "range",
"dst_port1": "7400",
"dst_port2": "7400",
"protocol": 6,
"direction": "double"
}
}
],
"not_flag" : 0
}
]
},
{
"compile_id": 155,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "ipv6_plus",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "ip_plus",
"table_name": "IP_PLUS_CONFIG",
"table_content": {
"addr_type": "ipv6",
"saddr_format": "range",
"src_ip1": "1001:da8:205:1::101",
"src_ip2": "1001:da8:205:1::201",
"sport_format": "mask",
"src_port1": "5210",
"src_port2": "65520",
"daddr_format": "mask",
"dst_ip1": "3001:da8:205:1::401",
"dst_ip2": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:0000",
"dport_format": "mask",
"dst_port1": "0",
"dst_port2": "65535",
"protocol": 6,
"direction": "double"
}
}
],
"not_flag" : 0
}
]
},
{
"compile_id": 156,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "ExprPlusWithHex",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_REGION",
"table_type": "expr_plus",
"table_content": {
"district": "Content-Type",
"keywords": "2f68746d6c",
"expr_type": "none",
"match_method": "sub",
"format": "hexbin"
}
}
]
}
]
},
{
"compile_id": 157,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "StringScan.StreamScanUTF8",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "expr",
"table_name": "TROJAN_PAYLOAD",
"table_content": {
"keywords": "我的订单",
"expr_type": "none",
"format": "none",
"match_method": "sub"
}
}
]
}
]
},
{
"compile_id": 158,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "IPScan.IPv4_CIDR",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "ip_plus",
"table_name": "IP_PLUS_CONFIG",
"table_content": {
"addr_type": "ipv4",
"saddr_format": "CIDR",
"src_ip1": "192.168.0.1",
"src_ip2": "32",
"sport_format": "range",
"src_port1": "5210",
"src_port2": "65520",
"daddr_format": "CIDR",
"dst_ip1": "10.0.6.1",
"dst_ip2": "24",
"dport_format": "mask",
"dst_port1": "0",
"dst_port2": "65535",
"protocol": 6,
"direction": "double"
}
}
],
"not_flag" : 0
}
]
},
{
"compile_id": 159,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "IPScan.IPv6_CIDR",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "ip_plus",
"table_name": "IP_PLUS_CONFIG",
"table_content": {
"addr_type": "ipv6",
"saddr_format": "CIDR",
"src_ip1": "2001:db8::",
"src_ip2": "120",
"sport_format": "mask",
"src_port1": "5210",
"src_port2": "65520",
"daddr_format": "CIDR",
"dst_ip1": "2001:4860:4860::8888",
"dst_ip2": "65",
"dport_format": "mask",
"dst_port1": "0",
"dst_port2": "65535",
"protocol": 6,
"direction": "double"
}
}
],
"not_flag" : 0
}
]
}

7
test/table_info.conf
View File

@@ -1,7 +1,7 @@
#each collumn seperate with '\t'
#id (0~65535)
#name string
#type one of ip,expr,expr_plus,digest,intval,compile or plugin
#type one of ip, ip_plus, expr, expr_plus, digest, intval, compile or plugin
#src_charset one of GBK,BIG5,UNICODE,UTF8
#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/'
#do_merege [yes/no]
@@ -24,7 +24,7 @@
5 CONTENT_SIZE intval --
6 QD_ENTRY_INFO plugin 4 --
7 FILE_DIGEST digest --
8 HTTP_REGION expr_plus GBK GBK no 0
8 HTTP_REGION expr_plus GBK GBK yes 0
9 SIM_URL similar --
10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff
11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} --
@@ -34,4 +34,5 @@
15 IR_INTERCEPT_IP plugin {"valid":14,"tag":18}
16 APP_PAYLOAD expr_plus UTF8 UTF8 yes 0 quickoff
17 TROJAN_PAYLOAD expr UTF8 UTF8 yes 0 quickoff
18 MAIL_ADDR expr UTF8 UTF8 yes 0 quickoff
18 MAIL_ADDR expr UTF8 UTF8 yes 0 quickoff
19 IP_PLUS_CONFIG ip_plus --

62
test/test_igraph.cpp Normal file
View File

@@ -0,0 +1,62 @@
#include "igraph/igraph.h"
#include <assert.h>
void print_vector(igraph_vector_t *v, FILE *f) {
long int i;
for (i=0; i<igraph_vector_size(v); i++) {
fprintf(f, " %li", (long int) VECTOR(*v)[i]);
}
fprintf(f, "\n");
}
int main() {
igraph_t g;
int ret;
ret=igraph_empty(&g, 0, IGRAPH_DIRECTED);
igraph_es_t es;
igraph_integer_t edge_num_before=0, edge_num_after=0;
int v[10];
int i=0;
for(i=0; i<sizeof(v)/sizeof(int); i++)
{
v[i]=i;
igraph_add_vertices(&g, 1, NULL); //Add 1 vertice.
}
igraph_add_edge(&g, v[0], v[1]);
igraph_add_edge(&g, v[0], v[1]);
igraph_add_edge(&g, v[2], v[3]);
int edge_id=0;
ret=igraph_get_eid(&g, &edge_id, v[2], v[3], IGRAPH_DIRECTED, 0);
assert(edge_id>0);
ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, v[0], v[1], -1);
assert(ret==IGRAPH_SUCCESS);
edge_num_before=igraph_ecount(&g);
ret=igraph_delete_edges(&g, es);
edge_num_after=igraph_ecount(&g);
assert(edge_num_before-edge_num_after==1);
assert(ret==IGRAPH_SUCCESS);
igraph_es_destroy(&es);
ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, v[3], v[4], -1);
assert(ret==IGRAPH_SUCCESS);
edge_num_before=igraph_ecount(&g);
/* error test, no such edge to delete */
igraph_set_error_handler(igraph_error_handler_ignore);
ret=igraph_delete_edges(&g, es);
edge_num_after=igraph_ecount(&g);
assert(edge_num_before=edge_num_after);
assert(ret!=IGRAPH_SUCCESS);
igraph_es_destroy(&es);
igraph_destroy(&g);
return 0;
}

549
test/test_maatframe.cpp
View File

@@ -42,14 +42,16 @@ void wait_for_cmd_effective(Maat_feather_t feather, long long version_before)
{
long long version_after=version_before;
int is_updating=1;
long long wating_us=0, sleep_us=1000*100;
while(is_updating||version_before==version_after)
{
Maat_read_state(feather,MAAT_STATE_IN_UPDATING, &is_updating, sizeof(is_updating));
Maat_read_state(feather,MAAT_STATE_VERSION, &version_after, sizeof(version_after));
usleep(1000*100);//waiting for commands go into effect
usleep(sleep_us);//waiting for commands go into effect
wating_us+=sleep_us;
}
// printf("wait for %lld ms\n", wating_us/1000);
}
void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old)
@@ -305,41 +307,48 @@ TEST(StringScan, BugReport20190325)
EXPECT_EQ(result[0].config_id, 150);
}
#define PrefixAndSuffixTest
TEST(StringScan, PrefixAndSuffix)
{
int ret=0;
int table_id=0;
int mail_addr_table=0, cont_sz_table=0;
struct Maat_rule_t result[4];
int found_pos[4];
const char* table_name="MAIL_ADDR";
scan_status_t mid=NULL;
const char* hit_twice="ceshi3@mailhost.cn";
const char* hit_suffix="11111111111ceshi3@mailhost.cn";
const char* hit_prefix="ceshi3@mailhost.cn11111111111";
table_id=Maat_table_register(g_feather,table_name);
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, hit_twice, strlen(hit_twice),
cont_sz_table=Maat_table_register(g_feather, "CONTENT_SIZE");
ret=Maat_scan_intval(g_feather, cont_sz_table, 2015,
result, 4, &mid, 0);
mail_addr_table=Maat_table_register(g_feather, "MAIL_ADDR");
ASSERT_GT(mail_addr_table, 0);
ret=Maat_full_scan_string(g_feather, mail_addr_table,CHARSET_GBK, hit_twice, strlen(hit_twice),
result,found_pos, 4, &mid, 0);
EXPECT_EQ(ret, 2);
EXPECT_EQ(result[0].config_id, 152);
EXPECT_EQ(result[1].config_id, 151);
EXPECT_EQ(result[0].config_id, 151);//compile has more groups is priority
EXPECT_EQ(result[1].config_id, 152);
Maat_clean_status(&mid);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, hit_suffix, strlen(hit_suffix),
ret=Maat_full_scan_string(g_feather, mail_addr_table,CHARSET_GBK, hit_suffix, strlen(hit_suffix),
result,found_pos, 4, &mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 151);
Maat_clean_status(&mid);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, hit_prefix, strlen(hit_prefix),
ret=Maat_scan_intval(g_feather, cont_sz_table, 2015,
result, 4, &mid, 0);
ret=Maat_full_scan_string(g_feather, mail_addr_table,CHARSET_GBK, hit_prefix, strlen(hit_prefix),
result,found_pos, 4, &mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 152);
Maat_clean_status(&mid);
}
#define ExprPlusTest 1
TEST(StringScan, ExprPlus)
{
int ret=0;
@@ -363,6 +372,8 @@ TEST(StringScan, ExprPlus)
&mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 128);
Maat_clean_status(&mid);
return;
}
@@ -414,9 +425,35 @@ TEST(StringScan, ExprPlusWithOffset)
&mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 148);
Maat_clean_status(&mid);
return;
}
TEST(IPScan, IPv4)
TEST(StringScan, ExprPlusWithHex)
{
int table_id=0,ret=0;
struct Maat_rule_t result[4];
const char* scan_data="text/html; charset=UTF-8";
const char* region_name="Content-Type";
int found_pos[4];
table_id=Maat_table_register(g_feather, "HTTP_REGION");
ASSERT_GT(table_id, 0);
scan_status_t mid=NULL;
ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, region_name, strlen(region_name));
ASSERT_EQ(ret, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
result, found_pos, 4,
&mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 156);
Maat_clean_status(&mid);
return;
}
#define TEST_IPSCAN
TEST(IPScan, IPv4_mask)
{
int table_id=0,ret=0;
const char* table_name="IP_CONFIG";
@@ -449,7 +486,7 @@ TEST(IPScan, IPv4)
Maat_clean_status(&mid);
return;
}
TEST(IPScan, IPv6)
TEST(IPScan, IPv6_mask)
{
int table_id=0,ret=0;
struct Maat_rule_t result[4];
@@ -474,6 +511,119 @@ TEST(IPScan, IPv6)
Maat_clean_status(&mid);
return;
}
TEST(IPScan, IPv4_range)
{
int table_id=0,ret=0;
const char* table_name="IP_PLUS_CONFIG";
struct Maat_rule_t result[4];
scan_status_t mid=NULL;
struct ipaddr ipv4_addr;
struct stream_tuple4_v4 v4_addr;
ipv4_addr.addrtype=ADDR_TYPE_IPV4;
inet_pton(AF_INET, "10.0.7.106", &(v4_addr.saddr));
v4_addr.source=htons(5000);
inet_pton(AF_INET, "123.56.104.254", &(v4_addr.daddr));
v4_addr.dest=htons(7400);
ipv4_addr.v4=&v4_addr;
table_id=Maat_table_register(g_feather, table_name);
EXPECT_GT(table_id, 0);
ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 154);
Maat_clean_status(&mid);
return;
}
TEST(IPScan, IPv6_range)
{
int table_id=0,ret=0;
struct Maat_rule_t result[4];
struct ipaddr ipv6_addr;
struct stream_tuple4_v6 v6_addr;
scan_status_t mid=NULL;
ipv6_addr.addrtype=ADDR_TYPE_IPV6;
inet_pton(AF_INET6,"1001:da8:205:1::151",&(v6_addr.saddr));
v6_addr.source=htons(5204);//5200~5299?
inet_pton(AF_INET6,"3001:da8:205:1::901",&(v6_addr.daddr));
v6_addr.dest=htons(80);//any
ipv6_addr.v6=&v6_addr;
const char* table_name="IP_PLUS_CONFIG";
table_id=Maat_table_register(g_feather,table_name);
EXPECT_GT(table_id, 0);
//for improving performance.
Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION,NULL, 0);
ret=Maat_scan_proto_addr(g_feather, table_id, &ipv6_addr, 6, result,4, &mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 155);
Maat_clean_status(&mid);
return;
}
TEST(IPScan, IPv4_CIDR)
{
int table_id=0,ret=0;
const char* table_name="IP_PLUS_CONFIG";
struct Maat_rule_t result[4];
scan_status_t mid=NULL;
struct ipaddr ipv4_addr;
struct stream_tuple4_v4 v4_addr;
ipv4_addr.addrtype=ADDR_TYPE_IPV4;
inet_pton(AF_INET, "192.168.0.1", &(v4_addr.saddr));
v4_addr.source=htons(5210);
inet_pton(AF_INET, "10.0.6.210", &(v4_addr.daddr));
v4_addr.dest=htons(7400);
ipv4_addr.v4=&v4_addr;
table_id=Maat_table_register(g_feather, table_name);
EXPECT_GT(table_id, 0);
ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 158);
Maat_clean_status(&mid);
return;
}
TEST(IPScan, IPv6_CIDR)
{
int table_id=0,ret=0;
struct Maat_rule_t result[4];
struct ipaddr ipv6_addr;
struct stream_tuple4_v6 v6_addr;
scan_status_t mid=NULL;
ipv6_addr.addrtype=ADDR_TYPE_IPV6;
inet_pton(AF_INET6,"2001:db8::00fe",&(v6_addr.saddr));
v6_addr.source=htons(5204);//5200~5299?
inet_pton(AF_INET6,"2001:4860:4860::8888",&(v6_addr.daddr));
v6_addr.dest=htons(80);//any
ipv6_addr.v6=&v6_addr;
const char* table_name="IP_PLUS_CONFIG";
table_id=Maat_table_register(g_feather,table_name);
EXPECT_GT(table_id, 0);
//for improving performance.
Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION,NULL, 0);
ret=Maat_scan_proto_addr(g_feather, table_id, &ipv6_addr, 6, result,4, &mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 159);
Maat_clean_status(&mid);
return;
}
#define TEST_NOTLogic 1
TEST(NOTLogic, OneRegion)
{
const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-143.";
@@ -489,15 +639,15 @@ TEST(NOTLogic, OneRegion)
ASSERT_GT(table_id, 0);
Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_hit, strlen(string_should_hit),
result,found_pos, 4, &mid, 0);
ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, string_should_hit, strlen(string_should_hit),
result, found_pos, 4, &mid, 0);
EXPECT_GE(ret, 1);
Maat_clean_status(&mid);
Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_not_hit, strlen(string_should_not_hit),
result,found_pos, 4, &mid, 0);
ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, string_should_not_hit, strlen(string_should_not_hit),
result, found_pos, 4, &mid, 0);
EXPECT_EQ(ret, -2);
Maat_clean_status(&mid);
@@ -713,7 +863,7 @@ TEST(StringScan, UTF8EncodedURL)
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url_utf8_qs, strlen(url_utf8_qs),
result,found_pos, 4,
&mid, 0);
EXPECT_GT(ret, 1);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 142);
Maat_clean_status(&mid);
@@ -843,6 +993,8 @@ TEST(StringScan, StreamInput)
Maat_clean_status(&mid);
return;
}
TEST(SimilarScan, Pure)
{
int ret=0;
@@ -946,6 +1098,51 @@ TEST(StringScan, OffsetChunk1460)
test_offset_str_scan_with_chunk(1460);
return;
}
#define StringScan_StreamScanUTF8 1
TEST(StringScan, StreamScanUTF8)
{
//Bug report: https://mantis.mesalab.cn/view.php?id=712
int table_id=0,ret=0;
int read_size=0,pass_flag=0;
struct Maat_rule_t result[4];
scan_status_t mid=NULL;
const char* table_name="TROJAN_PAYLOAD";
const char* fn="./testdata/jd.com.html";
FILE* fp=fopen(fn,"r");
ASSERT_FALSE(fp==NULL);
char scan_data[2048];
table_id=Maat_table_register(g_feather,table_name);
ASSERT_GT(table_id, 0);
struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10);
stream_para_t sp=Maat_stream_scan_string_start(g_feather,table_id,0);
int detail_ret=0;
ASSERT_FALSE(sp==NULL);
while(0==feof(fp))
{
read_size=fread(scan_data,1,sizeof(scan_data),fp);
ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data,read_size
,result,4,hit_detail,10
,&detail_ret,&mid);
if(ret>0)
{
pass_flag=1;
break;
}
}
EXPECT_EQ(pass_flag, 1);
EXPECT_EQ(result[0].config_id, 157);
Maat_stream_scan_string_end(&sp);
free(hit_detail);
fclose(fp);
Maat_clean_status(&mid);
return;
}
void accept_tags_entry_cb(int table_id,const char* table_line,void* u_para)
{
@@ -959,7 +1156,7 @@ void accept_tags_entry_cb(int table_id,const char* table_line,void* u_para)
return;
}
TEST(RuleTags, Plugin1)
TEST(Policy, PluginRuleTags1)
{
#define RuleTags_Plugin
int ret=0;
@@ -985,7 +1182,7 @@ void accept_tags_entry2_cb(int table_id,const char* table_line,void* u_para)
return;
}
TEST(RuleTags, Plugin2)
TEST(Policy, PluginRuleTags2)
{
#define RuleTags_Plugin2
int ret=0;
@@ -1005,7 +1202,7 @@ TEST(RuleTags, Plugin2)
return;
}
TEST(RuleTags, Compile)
TEST(Policy, CompileRuleTags)
{
#define RuleTags_Compile
int ret=0;
@@ -1086,7 +1283,7 @@ void compile_ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *fro
TEST(EX_DATA_INDEX, MaatRuleEXData)
TEST(Policy, CompileEXData)
{
#define rule_EX_data_index
@@ -1120,6 +1317,42 @@ TEST(EX_DATA_INDEX, MaatRuleEXData)
Maat_clean_status(&mid);
return;
}
TEST(Policy, SubGroup)
{
#define TestSubGroup
int ret=0, table_id=0;
const char* scan_string="ceshi6@mailhost.cn";
struct Maat_rule_t result[4];
memset(result, 0, sizeof(result));
scan_status_t mid=NULL;
struct ipaddr ipv4_addr;
struct stream_tuple4_v4 v4_addr;
ipv4_addr.addrtype=ADDR_TYPE_IPV4;
inet_pton(AF_INET,"10.0.6.205",&(v4_addr.saddr));
v4_addr.source=htons(50001);
inet_pton(AF_INET,"10.0.6.201",&(v4_addr.daddr));
v4_addr.dest=htons(80);
ipv4_addr.v4=&v4_addr;
table_id=Maat_table_register(g_feather,"MAIL_ADDR");
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, scan_string, strlen(scan_string),
result,NULL, 4, &mid, 0);
EXPECT_EQ(ret, -2);
table_id=Maat_table_register(g_feather, "IP_CONFIG");
ASSERT_GT(table_id, 0);
ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid,0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 153);
Maat_clean_status(&mid);
return;
}
TEST(StreamFuzzyHash, Pure)
@@ -1374,6 +1607,7 @@ protected:
static int linger_timeout;
};
Maat_feather_t MaatCmdTest::_shared_feather;
void* MaatCmdTest::logger;
int MaatCmdTest::linger_timeout;
@@ -1475,7 +1709,7 @@ TEST_F(MaatCmdTest, SetIP)
EXPECT_GE(ret, 0);
Maat_free_cmd(cmd);
cmd=NULL;
wait_for_cmd_effective(feather, version_before);
struct ipaddr ipv4_addr;
struct stream_tuple4_v4 v4_addr;
@@ -1503,10 +1737,11 @@ TEST_F(MaatCmdTest, SetIP)
//reset timeout.
cmd=Maat_create_cmd(&rule, 0);
cmd->expire_after=10;
cmd->expire_after=timeout+5;
ret=Maat_cmd(feather, cmd, MAAT_OP_RENEW_TIMEOUT);
EXPECT_EQ(ret ,1);
// RENEW_TIMEOUT doesn't change MAAT_VERSION, thus we cannot use wait_for_cmd_effective.
// wait_for_cmd_effective(feather, version_before);
Maat_free_cmd(cmd);
@@ -1608,7 +1843,7 @@ TEST_F(MaatCmdTest, RuleIDRecycle)
struct Maat_rule_t result;
scan_status_t mid=NULL;
memset(&result, 0, sizeof(result));
int table_id=0;
table_id=Maat_table_register(feather,table_name);
ASSERT_GT(table_id, 0);
@@ -1658,7 +1893,7 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder)
int table_id=0;
table_id=Maat_table_register(feather,table_name);
table_id=Maat_table_register(feather, table_name);
ASSERT_GT(table_id, 0);
int i=0, repeat_times=4;
@@ -1677,6 +1912,7 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder)
ret=Maat_cmd_commit(feather);
EXPECT_TRUE(ret>=0);
usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
memset(&result, 0, sizeof(result));
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
result, NULL, 8,
&mid, 0);
@@ -1689,7 +1925,256 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder)
}
return;
}
#define MaatCmdTest_SubGroup 0
TEST_F(MaatCmdTest, SubGroup)
{
const char* table_name="HTTP_URL";
const char* group_table_name="GROUP";
const char* compile_table_name="COMPILE";
const char* scan_data1="www.v2ex.com/t/573028#程序员的核心竞争力是什么";
const char* keyword1="程序员&核心竞争力";
const char* scan_data2="https://ask.leju.com/bj/detail/12189672562229248/?bi=tg&type=sina-pc&pos=index-dbtlwzl&wt_campaign=M_5CE750003F393&wt_source=PDPS_514ACACFD9E770";
// const char* keyword2="ask.leju.com/bj/detail/12189672562229248/?bi=tg&type=sina-pc&pos=index-dbtlwzl&wt_campaign=M_5CE750003F393&wt_source=PDPS_514ACACFD9E770";
const char* keyword2="ask.leju.com/b&/detail/12189672562229248/?&?bi=tg\\&type=sina-pc\\&&\\&pos=index-dbtlwzl\\&&\\&type=sina-pc\\&pos=index-dbtlwzl\\&";
Maat_feather_t feather=MaatCmdTest::_shared_feather;
int table_id=0;
table_id=Maat_table_register(feather, table_name);
struct Maat_rule_t compile1, compile2;
struct Maat_group_t group1, group2, group3;
struct Maat_region_t region1,region2;
memset(&compile1, 0, sizeof(compile1));
compile1.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1);
memset(&compile2, 0, sizeof(compile2));
compile2.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile2, compile_table_name, NULL, 1);
//group1->compile1
memset(&group1, 0, sizeof(group1));
group1.group_id=Maat_cmd_get_new_group_id(feather);
group1.table_name=group_table_name;
group1.parent_id=compile1.config_id;
group1.parent_type=PARENT_TYPE_COMPILE;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
//group1->compile2
group1.parent_id=compile2.config_id;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
/*group2->group1->compile1
\
\__compile2
*/
memset(&group2, 0, sizeof(group2));
group2.group_id=Maat_cmd_get_new_group_id(feather);
group2.table_name=group_table_name;
group2.parent_id=group1.group_id;
group2.parent_type=PARENT_TYPE_GROUP;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2);
/*region1->group2->group1->compile1
\
\_compile2
*/
memset(&region1, 0, sizeof(region1));
region1.region_id=Maat_cmd_get_new_region_id(feather);
region1.region_type=REGION_EXPR;
region1.table_name=table_name;
region1.expr_rule.keywords=keyword1;
region1.expr_rule.expr_type=EXPR_TYPE_AND;
Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region1, group2.group_id);
sleep(1);
struct Maat_rule_t result[4];
memset(&result, 0, sizeof(result));
scan_status_t mid=NULL;
int ret=0;
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1),
result, NULL, 4,
&mid, 0);
EXPECT_EQ(ret, 2);
EXPECT_EQ(result[0].config_id, compile2.config_id);
EXPECT_EQ(result[1].config_id, compile1.config_id);
Maat_clean_status(&mid);
/*region1->group2->group1->compile1
\
\--X--compile2
*/
Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1);
sleep(1);
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1),
result, NULL, 4,
&mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, compile1.config_id);
Maat_clean_status(&mid);
/*region1->group2->group1--X--
\
\->compile2
*/
group1.parent_id=compile1.config_id;
Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1);
Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile1, compile_table_name, NULL, 1);
group2.parent_type=PARENT_TYPE_COMPILE;
group2.parent_id=compile2.config_id;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2);
sleep(1);
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1),
result, NULL, 4,
&mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, compile2.config_id);
Maat_clean_status(&mid);
/*region1->group2->group1--X--
\
\->compile2
region2->group3
*/
memset(&group3, 0, sizeof(group3));
group3.group_id=Maat_cmd_get_new_group_id(feather);
group3.parent_id=group1.group_id;
group3.parent_type=PARENT_TYPE_GROUP;
group3.table_name=group_table_name;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group3);
memset(&region2, 0, sizeof(region2));
region2.region_id=Maat_cmd_get_new_region_id(feather);
region2.region_type=REGION_EXPR;
region2.table_name=table_name;
region2.expr_rule.keywords=keyword2;
region2.expr_rule.expr_type=EXPR_TYPE_AND;
Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region2, group3.group_id);
sleep(1);
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data2, strlen(scan_data2),
result, NULL, 4,
&mid, 0);
EXPECT_EQ(ret, -2);
Maat_clean_status(&mid);
/*region1->group2->group1--X--
\
\->compile2
region2->group3
*/
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1);
group1.parent_id=compile1.config_id;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile1, compile_table_name, NULL, 1);
group1.parent_id=compile1.config_id;
Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1);
sleep(1);
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1),
result, NULL, 4,
&mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, compile2.config_id);
Maat_clean_status(&mid);
}
#define MaatCmdTest_RefGroup 0
TEST_F(MaatCmdTest, RefGroup)
{
const char* table_name="HTTP_URL";
const char* group_table_name="GROUP";
const char* compile_table_name="COMPILE";
const char* scan_data1="m.facebook.com/help/2297503110373101?helpref=hc_nav&refid=69";
const char* keyword1="something-should-not-hit";
const char* keyword2="facebook.com/help/2297503110373101";
Maat_feather_t feather=MaatCmdTest::_shared_feather;
int table_id=0;
table_id=Maat_table_register(feather, table_name);
struct Maat_rule_t compile1;
struct Maat_group_t group1, group2;
struct Maat_region_t region1, region2;
int value=0;
Maat_set_feather_opt(feather, MAAT_OPT_ENABLE_UPDATE, &value, sizeof(value));
memset(&compile1, 0, sizeof(compile1));
compile1.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1);
//group1->compile1
memset(&group1, 0, sizeof(group1));
group1.group_id=Maat_cmd_get_new_group_id(feather);
group1.table_name=group_table_name;
group1.parent_id=compile1.config_id;
group1.parent_type=PARENT_TYPE_COMPILE;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
/*region1->group1->compile1
*/
memset(&region1, 0, sizeof(region1));
region1.region_id=Maat_cmd_get_new_region_id(feather);
region1.region_type=REGION_EXPR;
region1.table_name=table_name;
region1.expr_rule.keywords=keyword1;
region1.expr_rule.expr_type=EXPR_TYPE_AND;
Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region1, group1.group_id);
value=1;
Maat_set_feather_opt(feather, MAAT_OPT_ENABLE_UPDATE, &value, sizeof(value));
sleep(1);
value=0;
Maat_set_feather_opt(feather, MAAT_OPT_ENABLE_UPDATE, &value, sizeof(value));
//region1->group1-X-compile1
//region2->group2->compile1
memset(&group2, 0, sizeof(group2));
group2.group_id=Maat_cmd_get_new_group_id(feather);
group2.table_name=group_table_name;
group2.parent_id=compile1.config_id;
group2.parent_type=PARENT_TYPE_COMPILE;
memset(&region2, 0, sizeof(region2));
region2.region_id=Maat_cmd_get_new_region_id(feather);
region2.region_type=REGION_EXPR;
region2.table_name=table_name;
region2.expr_rule.keywords=keyword2;
region2.expr_rule.expr_type=EXPR_TYPE_AND;
Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region2, group2.group_id);
Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile1, compile_table_name, NULL, 1);
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1);
Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1);
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2);
value=1;
Maat_set_feather_opt(feather, MAAT_OPT_ENABLE_UPDATE, &value, sizeof(value));
sleep(1);
struct Maat_rule_t result[4];
memset(&result, 0, sizeof(result));
scan_status_t mid=NULL;
int ret=0;
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1),
result, NULL, 4,
&mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, compile1.config_id);
Maat_clean_status(&mid);
}
TEST_F(MaatCmdTest, SetLines)
{
@@ -2105,7 +2590,7 @@ int main(int argc, char ** argv)
const char* decrypt_key="mesa2017wy";
const char* accept_tags="{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
// const char* subsitute_acc_tags="{\"tags\":[{\"tag\":\"location\",\"value\":\"Astana\"}]}";
int scan_detail=0;
int scan_detail=0, ret=0;
::testing::InitGoogleTest(&argc, argv);
g_logger=MESA_create_runtime_log_handle(log_file, 0);
@@ -2113,8 +2598,8 @@ int main(int argc, char ** argv)
g_feather=Maat_feather(g_iThreadNum, table_info_path, g_logger);
Maat_set_feather_opt(g_feather, MAAT_OPT_INSTANCE_NAME, "demo", strlen("demo")+1);
Maat_set_feather_opt(g_feather, MAAT_OPT_DECRYPT_KEY, decrypt_key, strlen(decrypt_key)+1);
Maat_set_feather_opt(g_feather, MAAT_OPT_JSON_FILE_PATH, json_path, strlen(json_path)+1);
ret=Maat_set_feather_opt(g_feather, MAAT_OPT_JSON_FILE_PATH, json_path, strlen(json_path)+1);
assert(ret==0);
Maat_set_feather_opt(g_feather, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms));
//Set a short intevral for testing.
Maat_set_feather_opt(g_feather, MAAT_OPT_EFFECT_INVERVAL_MS, &effective_interval_ms, sizeof(effective_interval_ms));
@@ -2127,7 +2612,7 @@ int main(int argc, char ** argv)
Maat_initiate_feather(g_feather);
printf("Maat initiating, see %s\n",log_file);
int ret=RUN_ALL_TESTS();
ret=RUN_ALL_TESTS();
Maat_burn_feather(g_feather);
MESA_destroy_runtime_log_handle(g_logger);
return ret;

968
test/testdata/jd.com.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

5
tools/CMakeLists.txt
View File

@@ -8,4 +8,9 @@ target_link_libraries(maat_redis_tool maat_frame_static ${MAAT_DEPEND_DYN_LIB})
target_include_directories(maat_redis_tool PRIVATE ${PROJECT_SOURCE_DIR}/src/inc_internal/)
target_include_directories(maat_redis_tool PRIVATE hiredis-vip-static)
add_executable(maat_debug_tool maat_debug_tool.cpp)
add_dependencies(maat_debug_tool maat_frame_shared)
target_link_libraries(maat_debug_tool maat_frame_static ${MAAT_DEPEND_DYN_LIB})
target_include_directories(maat_debug_tool PRIVATE ${PROJECT_SOURCE_DIR}/src/inc_internal/)
install(TARGETS maat_redis_tool DESTINATION /usr/local/bin/)

174
tools/maat_debug_tool.cpp Normal file
View File

@@ -0,0 +1,174 @@
#include "Maat_rule.h"
#include "Maat_command.h"
#include <MESA/MESA_handle_logger.h>
#include <stdio.h> /* for printf */
#include <stdlib.h> /* for exit */
#include <getopt.h>
#include <sys/stat.h>
int debug_maat_str_scan(Maat_feather_t feather, const char* table_name, const char* district, char* data, size_t sz)
{
int table_id=0,ret=0;
int i=0;
int scan_result_max=64;
struct Maat_rule_t result[scan_result_max];
scan_status_t mid=NULL;
table_id=Maat_table_register(feather, table_name);
if(table_id<0)
{
printf("Unkown table %s\n", table_name);
}
struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10);
enum MAAT_CHARSET maat_charset=CHARSET_UTF8;
if(strlen(district)>0)
{
ret=Maat_set_scan_status(feather, &mid, MAAT_SET_SCAN_DISTRICT, district, strlen(district));
}
ret=Maat_full_scan_string(feather, table_id, maat_charset, data, sz,
result, NULL, scan_result_max,
&mid, 0);
printf("Scan table %s ", table_name);
if(ret==-1)
{
printf("error.\n");
}
else if(ret==-2)
{
printf("hits group, but not compile.\n");
}
else if(ret==0)
{
printf("not hit.");
}
else
{
printf("hits ");
for(i=0; i<ret; i++)
{
printf("%d, ", result[i].config_id);
}
}
printf("\n");
free(hit_detail);
Maat_clean_status(&mid);
return 0;
}
enum tool_arg_type{
ARG_TABLE_INFO=0,
ARG_TABLE_NAME,
ARG_SCAN_FILE,
ARG_SCAN_STRING,
ARG_SCAN_DISTRICT,
ARG_SCAN_CHARSET,
ARG_INPUT_JSON,
ARG_DECRYPT_KEY,
ARG_ACCEPT_TAGS,
__ARG_MAX
};
int main(int argc, char ** argv)
{
FILE* fp=NULL;
char* file_buff=NULL;
size_t file_size=0;
struct stat file_info;
char arg_value[__ARG_MAX][1024];
memset(arg_value, 0, sizeof(arg_value));
if(argc<2)
{
return 0;
}
int c=0;
while (1) {
int option_index = 0;
static struct option long_options[]={
{"table-info", required_argument, 0, 0},
{"table-name", required_argument, 0, 0},
{"scan-file", optional_argument, 0, 0},
{"scan-string", optional_argument, 0, 0},
{"scan-district", optional_argument, 0, 0},
{"scan-charset", optional_argument, 0, 0},
{"maat-json", required_argument, 0, 0},
{"decrypt-key", optional_argument, 0, 0},
{"accept-tags", optional_argument, 0, 0},
{0, 0, 0, 0}
};
c = getopt_long(argc, argv, "ab", long_options, &option_index);
if (c == -1)
break;
if(c!=0)
{
printf("Invalid parameter, long options only.\n");
}
else
{
strcpy(arg_value[option_index], optarg);
}
}
const char* log_file="./test.log";
int scan_detail=0, ret=0;
Maat_feather_t feather=NULL;
void *g_logger=NULL;
g_logger=MESA_create_runtime_log_handle(log_file, 0);
feather=Maat_feather(4, arg_value[ARG_TABLE_INFO], g_logger);
Maat_set_feather_opt(feather, MAAT_OPT_INSTANCE_NAME, "debugtool", strlen("debugtool")+1);
if(strlen(arg_value[ARG_DECRYPT_KEY])>0)
{
Maat_set_feather_opt(feather, MAAT_OPT_DECRYPT_KEY, arg_value[ARG_DECRYPT_KEY], strlen(arg_value[ARG_DECRYPT_KEY])+1);
}
if(strlen(arg_value[ARG_ACCEPT_TAGS])>0)
{
Maat_set_feather_opt(feather, MAAT_OPT_ACCEPT_TAGS, arg_value[ARG_ACCEPT_TAGS], strlen(arg_value[ARG_ACCEPT_TAGS])+1);
}
ret=Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, arg_value[ARG_INPUT_JSON], strlen(arg_value[ARG_INPUT_JSON])+1);
if(ret!=0)
{
printf("Read %s failed, invalid maat json.\n", arg_value[ARG_INPUT_JSON]);
ret=-1;
goto clean_up;
}
Maat_set_feather_opt(feather, MAAT_OPT_SCAN_DETAIL, &scan_detail, sizeof(scan_detail));
Maat_initiate_feather(feather);
if(strlen(arg_value[ARG_SCAN_FILE])>0)
{
if(stat(arg_value[ARG_SCAN_FILE], &file_info)<0)
{
printf("Stat %s failed.\n", arg_value[ARG_SCAN_FILE]);
goto clean_up;
}
file_size=file_info.st_size;
fp=fopen(arg_value[ARG_SCAN_FILE], "r");
if(fp==NULL)
{
printf("Open %s failed.\n", arg_value[ARG_SCAN_FILE]);
goto clean_up;
}
file_buff=(char*)malloc(file_size);
fread(file_buff,1,file_size,fp);
fclose(fp);
debug_maat_str_scan(feather, arg_value[ARG_TABLE_NAME], arg_value[ARG_SCAN_DISTRICT], file_buff, file_size);
free(file_buff);
}
clean_up:
Maat_burn_feather(feather);
MESA_destroy_runtime_log_handle(g_logger);
return ret;
}

16
vendor/CMakeLists.txt vendored
View File

@@ -30,3 +30,19 @@ file(MAKE_DIRECTORY ${HIREDIS_INCLUDE_DIRECTORIES})
add_library(hiredis-vip-static STATIC IMPORTED GLOBAL)
set_property(TARGET hiredis-vip-static PROPERTY IMPORTED_LOCATION ${SOURCE_DIR}/libhiredis_vip.a)
set_property(TARGET hiredis-vip-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${SOURCE_DIR})
ExternalProject_Add(igraph PREFIX igraph
URL ${CMAKE_CURRENT_SOURCE_DIR}/igraph-0.7.1.tar.gz
URL_MD5 4f6e7c16b45fce8ed423516a9786e4e8
CONFIGURE_COMMAND ./configure --prefix=<INSTALL_DIR> --disable-shared --with-pic
BUILD_COMMAND make
INSTALL_COMMAND make install
BUILD_IN_SOURCE 1)
ExternalProject_Get_Property(igraph INSTALL_DIR)
file(MAKE_DIRECTORY ${INSTALL_DIR}/include)
add_library(igraph-static STATIC IMPORTED GLOBAL)
add_dependencies(igraph-static igraph)
set_property(TARGET igraph-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libigraph.a)
set_property(TARGET igraph-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include)

BIN
vendor/igraph-0.7.1.tar.gz vendored Normal file
View File

Binary file not shown.