gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

为支持命中路径功能，重构scan_status相关函数，用TAILQ替代dynamic_array。

Browse Source
This commit is contained in:
zhengchao
2020-02-11 00:14:56 +08:00
parent 2cd54ae7cf
commit 278a6b9203
6 changed files with 500 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
inc/Maat_rule.h
View File

@@ -183,14 +183,41 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
Maat_finish_callback_t *finish,//u_para Maat_finish_callback_t *finish,//u_para
void* u_para); void* u_para);
#define MAX_HIT_REGION_NUM_PER_GROUP 128
struct Maat_hit_group_id_t
{
int compile_id;
int group_id;
size_t n_hit_region;
int region_id[MAX_HIT_REGION_NUM_PER_GROUP];
};
struct Maat_hit_path_t
{
int Nth_scan;//Maat_stream_scan_**_start is a individual scan.
int region_id;
int sub_group_id;
int top_group_id;
int virtual_table_id;
int compile_id;
};
struct Maat_hit_compile_id_t
{
int config_id;
size_t n_hit_group;
struct Maat_hit_group_id_t *group_ids;
};
enum MAAT_SCAN_OPT enum MAAT_SCAN_OPT
{ {
MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*,SIZE= strlen(string). DEFAULT: no default. MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*, SIZE= strlen(string). DEFAULT: no default.
MAAT_SET_SCAN_LAST_REGION //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination. MAAT_SET_SCAN_LAST_REGION, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination.
MAAT_GET_SCAN_HIT_PATH //VALUE is struct Maat_hit_path_t*, an array of struct Maat_hit_path_t, SIZE= sizeof(struct Maat_hit_path_t)*N,
//Maat_get_scan_status returns actual filled number.
}; };
//return 0 if success, return -1 when failed; //return 0 if success, return -1 when failed;
int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT type,const void* value,int size); int Maat_set_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_SCAN_OPT type, const void* value, int size);
//return >=0 if success, return -1 when failed;
int Maat_get_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_SCAN_OPT type, void* value, int size);
//Return hit rule number, return -1 when error occurs,return -2 when hit current region //Return hit rule number, return -1 when error occurs,return -2 when hit current region
//mid MUST set to NULL before fist call //mid MUST set to NULL before fist call

673
src/entry/Maat_api.cpp
View File

@@ -28,27 +28,7 @@ inline void DEC_SCANNER_REF(Maat_scanner*scanner,int thread_num)
alignment_int64_array_add(scanner->ref_cnt, thread_num, -1); alignment_int64_array_add(scanner->ref_cnt, thread_num, -1);
return; return;
} }
void Maat_clean_status(scan_status_t* mid)
{
struct _OUTER_scan_status_t* _mid=NULL;
if(*mid==NULL)
{
return;
}
_mid=(struct _OUTER_scan_status_t*)(*mid);
alignment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1);
if(_mid->inner!=NULL)
{
dynamic_array_destroy(_mid->inner->cur_hit_groups, NULL);
free(_mid->inner->all_hit_group_array);
free(_mid->inner);
alignment_int64_array_add(_mid->feather->inner_mid_cnt,_mid->thread_num,-1);
}
_mid->feather=NULL;
free(_mid);
*mid=NULL;
return;
}
inline int scan_status_should_compile_NOT(struct _OUTER_scan_status_t* _mid) inline int scan_status_should_compile_NOT(struct _OUTER_scan_status_t* _mid)
{ {
if( _mid && if( _mid &&
@@ -63,6 +43,24 @@ inline int scan_status_should_compile_NOT(struct _OUTER_scan_status_t* _mid)
return 0; return 0;
} }
} }
int exprid2region_id(struct Maat_group_inner* group_rule, int expr_id, int* district_id, struct Maat_scanner* scanner)
{
int region_id=-1;
struct Maat_region_inner* region_rule=NULL;
assert(group_rule->group_id>=0);
int array_idx=(int)(long)HASH_fetch_by_id(scanner->exprid_hash, expr_id);
pthread_mutex_lock(&(group_rule->mutex));
assert(array_idx<group_rule->region_boundary);
region_rule=(struct Maat_region_inner*)dynamic_array_read(group_rule->regions, array_idx);
if(region_rule)
{
assert(expr_id>=region_rule->expr_id_lb&&expr_id<=region_rule->expr_id_ub);
region_id=region_rule->region_id;
*district_id=region_rule->district_id;
}
pthread_mutex_unlock(&(group_rule->mutex));
return region_id;
}
//return 1 if insert a unique id //return 1 if insert a unique id
//return 0 if id is duplicated //return 0 if id is duplicated
@@ -92,27 +90,7 @@ int insert_set_id(unsigned long long **set, size_t* size, size_t cnt, unsigned l
return 0; return 0;
} }
} }
size_t pickup_hit_region_from_compile(struct bool_expr *compile_hit,
struct dynamic_array_t* hitted_id, size_t hit_cnt, int* region_pos, size_t size)
{
size_t i=0, j=0;
size_t k=0;
unsigned long long group_id=0;
for(i=0;i<hit_cnt;i++)
{
group_id=(unsigned long long)dynamic_array_read(hitted_id, i);
for(j=0; j<compile_hit->item_num; j++)
{
if(group_id==compile_hit->items[j].item_id)
{
region_pos[k]=i;
k++;
assert(k<size);
}
}
}
return k;
}
void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len) void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len)
{ {
memcpy(rule, rule_head, sizeof(struct Maat_rule_head)); memcpy(rule, rule_head, sizeof(struct Maat_rule_head));
@@ -207,56 +185,177 @@ size_t Maat_rule_sort_by_evaluation_order(Maat_feather_t feather, struct Maat_ru
} }
struct scan_region_hit_wraper struct scan_region_hit_wraper
{ {
int Nth_scan;
void* elem_array; void* elem_array;
size_t elem_size; size_t elem_size;
size_t n_elem; size_t n_elem;
size_t group_offset; size_t group_offset;
size_t expr_id_offset;
int virtual_table_id; int virtual_table_id;
int is_last_region; int is_last_region;
}; };
void scan_region_hit_wraper_build_with_rulescan(struct scan_region_hit_wraper* region_hit, scan_result_t* rulescan_rslt, size_t n_rslt, int is_last_region, int virual_table_id) void scan_region_hit_wraper_build_with_rulescan(struct scan_region_hit_wraper* region_hit, scan_result_t* rulescan_rslt, size_t n_rslt, int is_last_region, int virual_table_id, int Nth_scan)
{ {
memset(region_hit, 0, sizeof(struct scan_region_hit_wraper)); memset(region_hit, 0, sizeof(struct scan_region_hit_wraper));
region_hit->elem_array=rulescan_rslt; region_hit->elem_array=rulescan_rslt;
region_hit->n_elem=n_rslt; region_hit->n_elem=n_rslt;
region_hit->elem_size=sizeof(scan_result_t); region_hit->elem_size=sizeof(scan_result_t);
region_hit->expr_id_offset=offsetof(scan_result_t, expr_id);
region_hit->group_offset=offsetof(scan_result_t, tag); region_hit->group_offset=offsetof(scan_result_t, tag);
region_hit->is_last_region=is_last_region; region_hit->is_last_region=is_last_region;
region_hit->virtual_table_id=virual_table_id; region_hit->virtual_table_id=virual_table_id;
region_hit->Nth_scan=Nth_scan;
return; return;
} }
void scan_region_hit_wraper_build_with_GIE(struct scan_region_hit_wraper* region_hit, GIE_result_t* GIE_rslt, size_t n_rslt, int is_last_region, int virual_table_id) void scan_region_hit_wraper_build_with_GIE(struct scan_region_hit_wraper* region_hit, GIE_result_t* GIE_rslt, size_t n_rslt, int is_last_region, int virual_table_id, int Nth_scan)
{ {
memset(region_hit, 0, sizeof(struct scan_region_hit_wraper)); memset(region_hit, 0, sizeof(struct scan_region_hit_wraper));
region_hit->elem_array=GIE_rslt; region_hit->elem_array=GIE_rslt;
region_hit->n_elem=n_rslt; region_hit->n_elem=n_rslt;
region_hit->elem_size=sizeof(GIE_result_t); region_hit->elem_size=sizeof(GIE_result_t);
region_hit->expr_id_offset=offsetof(GIE_result_t, id);
region_hit->group_offset=offsetof(GIE_result_t, tag); region_hit->group_offset=offsetof(GIE_result_t, tag);
region_hit->is_last_region=is_last_region; region_hit->is_last_region=is_last_region;
region_hit->virtual_table_id=virual_table_id; region_hit->virtual_table_id=virual_table_id;
region_hit->Nth_scan=Nth_scan;
return; return;
} }
struct scan_hit_status* scan_hit_status_new(void)
{
struct scan_hit_status* inner_mid=NULL;
inner_mid=ALLOC(struct scan_hit_status, 1);
TAILQ_INIT(&inner_mid->hit_path_qhead);
return inner_mid;
}
void scan_hit_status_free(struct scan_hit_status* p)
{
struct Maat_hit_path_inner * tmp = TAILQ_FIRST(&p->hit_path_qhead);
while(tmp != NULL)
{
TAILQ_REMOVE(&p->hit_path_qhead, tmp, entries);
free(tmp);
p->hit_path_cnt--;
tmp = TAILQ_FIRST(&p->hit_path_qhead);
}
assert(p->hit_path_cnt==0);
free(p->all_hit_group_array);
p->all_hit_group_array=NULL;
free(p);
}
int region_compile(_Maat_feather_t*feather, struct _INNER_scan_status_t *_mid, const struct scan_region_hit_wraper* region_hit_wraper, struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num) void scan_hit_status_update_by_group(struct scan_hit_status* hit_status, struct Maat_group_inner* group_rule, int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result)
{
size_t i=0;
int ret=0;
struct Maat_hit_path_inner* hit_path=NULL;
for(i=0; i<group_rule->top_group_cnt; i++)
{
hit_path=ALLOC(struct Maat_hit_path_inner, 1);
hit_path->Nth_hit_region=Nth_region_result;
hit_path->path.Nth_scan=Nth_scan;
hit_path->path.compile_id=-1;
hit_path->path.region_id=region_id;
hit_path->path.sub_group_id=group_rule->group_id;
hit_path->path.top_group_id=group_rule->top_groups[i];
hit_path->path.virtual_table_id=virtual_table_id;
TAILQ_INSERT_TAIL(&hit_status->hit_path_qhead, hit_path, entries);
hit_status->hit_path_cnt++;
ret=insert_set_id(&(hit_status->all_hit_group_array),
&(hit_status->all_hit_group_array_sz),
hit_status->all_hit_group_cnt,
TO_RELATION_ID(virtual_table_id, group_rule->top_groups[i]));
hit_status->all_hit_group_cnt+=ret;
}
return;
}
size_t scan_hit_status_update_by_compile(struct scan_hit_status* hit_status, struct Maat_compile_group_relation* compile_rule, int Nth_scan)
{
size_t i=0, j=0;
struct Maat_hit_path_inner* p=NULL, *q=NULL;
struct bool_expr a_set;
unsigned char has_not=0;
make_group_set(compile_rule, &a_set, &has_not);
TAILQ_FOREACH(p, &hit_status->hit_path_qhead, entries)
{
for(i=0; i<a_set.item_num; i++)
{
if(TO_RELATION_ID(p->path.virtual_table_id, p->path.top_group_id)==a_set.items[i].item_id
&& p->path.compile_id!=compile_rule->compile_id)
{
if(p->path.compile_id<0)
{
p->path.compile_id=compile_rule->compile_id;
}
else //current path already have a compile as endpoint, so we duplicate a new path.
{
q=ALLOC(struct Maat_hit_path_inner, 1);
*q=*p;
q->path.compile_id=compile_rule->compile_id;
TAILQ_INSERT_HEAD(&hit_status->hit_path_qhead, q, entries);
hit_status->hit_path_cnt++;
}
if(p->path.Nth_scan==Nth_scan)//Compile was satisfied by new region hits.
{
j++;
}
}
}
}
return j;
}
size_t scan_hit_status_read_group_ids(const struct scan_hit_status* hit_status, const unsigned long long** const grp_ids)
{
*grp_ids=hit_status->all_hit_group_array;
return hit_status->all_hit_group_cnt;
}
//Maybe select?
size_t scan_hit_status_read_hit_path_inner(const struct scan_hit_status* hit_status, int Nth_scan,int compile_id,
struct Maat_hit_path_inner* hit_paths, size_t n_path)
{
struct Maat_hit_path_inner* p=NULL;
size_t i=0;
TAILQ_FOREACH(p, &hit_status->hit_path_qhead, entries)
{
if(p->path.Nth_scan!=Nth_scan||p->path.compile_id!=compile_id)
{
continue;
}
if(i<n_path)
{
hit_paths[i]=*p;
i++;
}
}
return i;
}
int region_compile(_Maat_feather_t*feather, struct scan_hit_status *_mid, const struct scan_region_hit_wraper* region_hit_wraper, struct Maat_rule_t* result, int size,int thread_num)
{ {
int is_last_region=region_hit_wraper->is_last_region; int is_last_region=region_hit_wraper->is_last_region;
void* region_hit=region_hit_wraper->elem_array; void* region_hit=region_hit_wraper->elem_array;
size_t region_type_size=region_hit_wraper->elem_size; size_t region_type_size=region_hit_wraper->elem_size;
size_t expr_id_offset=region_hit_wraper->expr_id_offset;
size_t group_offset=region_hit_wraper->group_offset; size_t group_offset=region_hit_wraper->group_offset;
size_t region_hit_num=region_hit_wraper->n_elem; size_t region_hit_num=region_hit_wraper->n_elem;
unsigned int expr_id=0;
int region_id=0;
int scan_ret=0, result_cnt=0; int scan_ret=0, result_cnt=0;
int ret=0; int tmp=0;
size_t i=0, j=0; size_t i=0;
size_t r_in_c_cnt=0; size_t r_in_c_cnt=0;
unsigned char has_not_flag=0;
struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler; struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler;
struct Maat_group_inner* group_rule=NULL; struct Maat_group_inner* group_rule=NULL;
struct Maat_compile_group_relation* relation_array[MAX_SCANNER_HIT_NUM]; struct Maat_compile_group_relation* relation_array[MAX_SCANNER_HIT_NUM];
struct Maat_compile_group_relation* relation=NULL; struct Maat_compile_group_relation* relation=NULL;
int region_pos[MAX_SCANNER_HIT_NUM]; const unsigned long long* hit_group_ids=NULL;
_mid->cur_hit_group_cnt=0; size_t hit_group_id_cnt=0;
for(i=0;i<region_hit_num;i++) for(i=0;i<region_hit_num;i++)
{ {
group_rule=*(struct Maat_group_inner**)((char*)region_hit+region_type_size*i+group_offset); group_rule=*(struct Maat_group_inner**)((char*)region_hit+region_type_size*i+group_offset);
@@ -266,25 +365,17 @@ int region_compile(_Maat_feather_t*feather, struct _INNER_scan_status_t *_mid, c
} }
assert(group_rule->ref_by_children_cnt>=0); assert(group_rule->ref_by_children_cnt>=0);
assert(group_rule->ref_by_parent_cnt>=0); assert(group_rule->ref_by_parent_cnt>=0);
for(j=0; j<group_rule->top_group_cnt; j++) expr_id=*(unsigned int*)((char*)region_hit+region_type_size*i+expr_id_offset);
{ region_id=exprid2region_id(group_rule, expr_id, &tmp, feather->scanner);
if(_mid->cur_hit_group_cnt<MAX_SCANNER_HIT_NUM) scan_hit_status_update_by_group(_mid, group_rule, region_id,
{ region_hit_wraper->virtual_table_id, region_hit_wraper->Nth_scan, i);
dynamic_array_write(_mid->cur_hit_groups, _mid->cur_hit_group_cnt, (void*)TO_RELATION_ID(region_hit_wraper->virtual_table_id,group_rule->top_groups[j]));
_mid->cur_hit_group_cnt++;
}
ret=insert_set_id(&(_mid->all_hit_group_array),
&(_mid->all_hit_group_array_sz),
_mid->all_hit_group_cnt,
TO_RELATION_ID(region_hit_wraper->virtual_table_id, group_rule->top_groups[j]));
_mid->all_hit_group_cnt+=ret;
}
} }
if(bm) if(bm)
{ {
hit_group_id_cnt=scan_hit_status_read_group_ids(_mid, &hit_group_ids);
scan_ret=bool_matcher_match(bm, thread_num, scan_ret=bool_matcher_match(bm, thread_num,
_mid->all_hit_group_array, _mid->all_hit_group_cnt, hit_group_ids, hit_group_id_cnt,
(void **)relation_array, MAX_SCANNER_HIT_NUM); (void **)relation_array, MAX_SCANNER_HIT_NUM);
} }
else else
@@ -314,15 +405,12 @@ int region_compile(_Maat_feather_t*feather, struct _INNER_scan_status_t *_mid, c
} }
else else
{ {
make_group_set(relation, &(rs_result[result_cnt].group_set), &has_not_flag); r_in_c_cnt=scan_hit_status_update_by_compile(_mid, relation, region_hit_wraper->Nth_scan);
r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set), _mid->cur_hit_groups, _mid->cur_hit_group_cnt, if(r_in_c_cnt>0 || //compile hitted becasue of new reigon
region_pos, MAX_SCANNER_HIT_NUM); region_hit_num==0) //or hit a compile that refer a NOT-logic group in previous scan.
if(r_in_c_cnt>0 || //compile config hitted becasue of new reigon
_mid->cur_hit_group_cnt==0) //or ever hit a compile that refer a NOT-logic group
{ {
fill_maat_rule(&(result[result_cnt]), &(relation->compile->head), fill_maat_rule(&(result[result_cnt]), &(relation->compile->head),
relation->compile->service_defined ,relation->compile->head.serv_def_len); relation->compile->service_defined, relation->compile->head.serv_def_len);
rs_result[result_cnt].compile_id=relation->compile_id;
result_cnt++; result_cnt++;
} }
} }
@@ -330,6 +418,7 @@ int region_compile(_Maat_feather_t*feather, struct _INNER_scan_status_t *_mid, c
pthread_rwlock_unlock(&(relation->rwlock)); pthread_rwlock_unlock(&(relation->rwlock));
} }
} }
if(result_cnt>0) if(result_cnt>0)
{ {
alignment_int64_array_add(feather->hit_cnt, thread_num, 1); alignment_int64_array_add(feather->hit_cnt, thread_num, 1);
@@ -342,24 +431,7 @@ int region_compile(_Maat_feather_t*feather, struct _INNER_scan_status_t *_mid, c
return result_cnt; return result_cnt;
} }
int exprid2region_id(struct Maat_group_inner* group_rule,int expr_id,int* district_id, Maat_scanner* scanner)
{
int region_id=-1;
struct Maat_region_inner* region_rule=NULL;
assert(group_rule->group_id>=0);
int array_idx=(int)(long)HASH_fetch_by_id(scanner->exprid_hash, expr_id);
pthread_mutex_lock(&(group_rule->mutex));
assert(array_idx<group_rule->region_boundary);
region_rule=(struct Maat_region_inner*)dynamic_array_read(group_rule->regions, array_idx);
if(region_rule)
{
assert(expr_id>=region_rule->expr_id_lb&&expr_id<=region_rule->expr_id_ub);
region_id=region_rule->region_id;
*district_id=region_rule->district_id;
}
pthread_mutex_unlock(&(group_rule->mutex));
return region_id;
}
int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num, Maat_scanner* scanner) int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num, Maat_scanner* scanner)
{ {
struct Maat_group_inner* group_rule=NULL; struct Maat_group_inner* group_rule=NULL;
@@ -450,76 +522,60 @@ int hit_pos_RS2Maat(struct sub_item_pos_t* maat_sub_item,int size,rule_result_t*
} }
return k; return k;
} }
int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mid, int fill_region_hit_detail(const char* scan_buff, const scan_hit_status* _mid,
scan_result_t *region_hit,int region_cnt, scan_result_t *region_hit, int region_cnt,
_compile_result_t *compile_hit,int compile_cnt, struct Maat_rule_t* compile, int compile_cnt,
struct Maat_hit_detail_t *hit_detail,int detail_num, int Nth_scan,
struct Maat_hit_detail_t *hit_detail, int detail_num,
struct Maat_scanner* scanner) struct Maat_scanner* scanner)
{ {
int i=0,j=0,k=0; int i=0;
size_t j=0;
char r_in_c_flag[region_cnt]; char r_in_c_flag[region_cnt];
int region_pos[MAX_SCANNER_HIT_NUM],pos=0; int district_id=-1; //Indifference
int r_in_c_cnt=0; memset(r_in_c_flag, 0, sizeof(r_in_c_flag));
int region_id=-1;
int district_id=-1; //Indifferenc
memset(r_in_c_flag,0,sizeof(r_in_c_flag));
memset(region_pos,0,sizeof(region_pos));
struct Maat_group_inner* group_rule=NULL; struct Maat_group_inner* group_rule=NULL;
//for each hitted compile cfg,find its region_ids
for(i=0;i<compile_cnt&&i<detail_num;i++)
{
hit_detail[i].config_id=compile_hit[i].compile_id;
r_in_c_cnt=pickup_hit_region_from_compile(&(compile_hit[i].group_set), _mid->cur_hit_groups, _mid->cur_hit_group_cnt,
region_pos, MAX_SCANNER_HIT_NUM);
assert(r_in_c_cnt>0);//previous hitted compile was elimited in region_compile
for(j=0,k=0;j<r_in_c_cnt&&k<MAAT_MAX_HIT_RULE_NUM;j++)
{
pos=region_pos[j];
r_in_c_flag[pos]=1;
group_rule=(struct Maat_group_inner*)(region_hit[pos].tag);
region_id=exprid2region_id(group_rule,region_hit[pos].expr_id, &district_id, scanner);
if(region_id<0)
{
continue;
}
hit_detail[i].region_pos[k].region_id=region_id;
hit_detail[i].region_pos[k].sub_item_num=region_hit[pos].rnum;
hit_pos_RS2Maat(hit_detail[i].region_pos[k].sub_item_pos,MAAT_MAX_EXPR_ITEM_NUM,
region_hit[pos].result, region_hit[pos].rnum,scan_buff);
k++;
}
hit_detail[i].hit_region_cnt=k;
struct Maat_hit_path_inner hit_paths[MAX_SCANNER_HIT_NUM];
size_t hit_path_cnt=0;
int Nth_hit_region=0;
//for each hitted compile cfg,find its region_ids
for(i=0; i<compile_cnt&&i<detail_num; i++)
{
hit_detail[i].config_id=compile[i].config_id;
hit_path_cnt=scan_hit_status_read_hit_path_inner(_mid, Nth_scan, compile[i].config_id, hit_paths, MAX_SCANNER_HIT_NUM);
for(j=0; j<hit_path_cnt && j<MAAT_MAX_HIT_RULE_NUM; j++)
{
Nth_hit_region=hit_paths[j].Nth_hit_region;
hit_detail[i].region_pos[j].region_id=hit_paths[j].path.region_id;
hit_detail[i].region_pos[j].sub_item_num=region_hit[Nth_hit_region].rnum;
hit_pos_RS2Maat(hit_detail[i].region_pos[j].sub_item_pos, MAAT_MAX_EXPR_ITEM_NUM,
region_hit[Nth_hit_region].result, region_hit[Nth_hit_region].rnum, scan_buff);
r_in_c_flag[Nth_hit_region]=1;
}
hit_detail[i].hit_region_cnt=j;
} }
//for each region_ids, if belongs to no compile cfg, fill hit_detail as a half hit cfg //for each region_ids, if belongs to no compile cfg, fill hit_detail as a half hit cfg
for(k=0,j=i;k<region_cnt&&j<region_cnt&&j<detail_num;k++) for(j=0; j<(size_t)region_cnt && i<detail_num; j++)
{ {
if(r_in_c_flag[k]==0) if(r_in_c_flag[j]==0)
{ {
group_rule=(struct Maat_group_inner*)(region_hit[k].tag); group_rule=(struct Maat_group_inner*)(region_hit[j].tag);
hit_detail[j].config_id=-2; hit_detail[i].config_id=-2;
hit_detail[j].hit_region_cnt=1; hit_detail[i].hit_region_cnt=1;
hit_detail[j].region_pos[0].region_id=exprid2region_id(group_rule,region_hit[k].expr_id,&district_id, scanner); hit_detail[i].region_pos[0].region_id=exprid2region_id(group_rule, region_hit[j].expr_id, &district_id, scanner);
hit_detail[j].region_pos[0].sub_item_num=region_hit[k].rnum; hit_detail[i].region_pos[0].sub_item_num=region_hit[j].rnum;
hit_pos_RS2Maat(hit_detail[j].region_pos[0].sub_item_pos,MAAT_MAX_EXPR_ITEM_NUM, hit_pos_RS2Maat(hit_detail[i].region_pos[0].sub_item_pos,MAAT_MAX_EXPR_ITEM_NUM,
region_hit[k].result,region_hit[k].rnum,scan_buff); region_hit[j].result, region_hit[j].rnum, scan_buff);
j++; i++;
} }
} }
return j; return i;
}
struct _INNER_scan_status_t* _make_inner_status(void)
{
struct _INNER_scan_status_t* inner_mid=NULL;
inner_mid=ALLOC(struct _INNER_scan_status_t, 1);
inner_mid->cur_hit_group_cnt=0;
inner_mid->cur_hit_groups=dynamic_array_create(32, 32);
inner_mid->all_hit_group_cnt=0;
inner_mid->all_hit_group_array_sz=4;
inner_mid->all_hit_group_array= ALLOC(unsigned long long, inner_mid->all_hit_group_array_sz);
return inner_mid;
} }
struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather, int thread_num) struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather, int thread_num)
{ {
struct _OUTER_scan_status_t* outer_mid=NULL; struct _OUTER_scan_status_t* outer_mid=NULL;
@@ -532,21 +588,18 @@ struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather, int th
} }
struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* feather,int thread_num,int is_hit_region) struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* feather,int thread_num,int is_hit_region)
{ {
struct _OUTER_scan_status_t* _mid=NULL; struct _OUTER_scan_status_t* _mid=(struct _OUTER_scan_status_t*)(*raw_mid);;
if(*raw_mid!=NULL) if(_mid==NULL)
{ {
_mid=(struct _OUTER_scan_status_t*)(*raw_mid); _mid=_make_outer_status(feather, thread_num);
*raw_mid=_mid;
} }
if(is_hit_region==1) if(is_hit_region==1)
{ {
if(_mid==NULL)
{
_mid=_make_outer_status(feather,thread_num);
*raw_mid=_mid;
}
if(_mid->inner==NULL) if(_mid->inner==NULL)
{ {
_mid->inner=_make_inner_status(); _mid->inner=scan_hit_status_new();
alignment_int64_array_add(feather->inner_mid_cnt,thread_num,1); alignment_int64_array_add(feather->inner_mid_cnt,thread_num,1);
} }
} }
@@ -1268,25 +1321,27 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
struct _OUTER_scan_status_t* _mid=(struct _OUTER_scan_status_t*)(*mid); struct _OUTER_scan_status_t* _mid=(struct _OUTER_scan_status_t*)(*mid);
scan_result_t *region_result=NULL; scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];//dynamic array
struct Maat_table_desc *p_table=NULL; struct Maat_table_desc *p_table=NULL;
struct expr_table_desc* expr_desc=NULL; struct expr_table_desc* expr_desc=NULL;
struct timespec start,end; struct timespec start,end;
Maat_scanner* my_scanner=NULL; Maat_scanner* my_scanner=NULL;
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
_mid=grab_mid(mid,_feather, thread_num, 0);
_mid->scan_cnt++;
if(data==NULL||data_len<=0) if(data==NULL||data_len<=0)
{ {
return 0; return -1;
} }
my_scanner=_feather->scanner; my_scanner=_feather->scanner;
if(my_scanner==NULL) if(my_scanner==NULL)
{ {
return 0; return 0;
} }
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
_mid=grab_mid(mid,_feather, thread_num, 0);
p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id); p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id);
if(p_table==NULL) if(p_table==NULL)
{ {
@@ -1355,10 +1410,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
} }
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
struct scan_region_hit_wraper region_hit_wraper; struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, virtual_table_id); scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, hit_region_cnt,
_mid->is_last_region, virtual_table_id, _mid->scan_cnt);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
&region_hit_wraper, &region_hit_wraper,
result,compile_result,rule_num, result, rule_num,
thread_num); thread_num);
assert(_mid->is_last_region<2); assert(_mid->is_last_region<2);
if(_mid->is_last_region==1) if(_mid->is_last_region==1)
@@ -1367,10 +1423,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
} }
if(hit_region_cnt>0&&hit_detail!=NULL&&_feather->rule_scan_type!=0) if(hit_region_cnt>0&&hit_detail!=NULL&&_feather->rule_scan_type!=0)
{ {
*detail_ret=fill_region_hit_detail(data,_mid->inner, *detail_ret=fill_region_hit_detail(data, _mid->inner,
region_result,hit_region_cnt, region_result, hit_region_cnt,
compile_result,compile_ret, result, compile_ret,
hit_detail,detail_num, my_scanner); _mid->scan_cnt,
hit_detail, detail_num, my_scanner);
} }
} }
if(_feather->perf_on==1) if(_feather->perf_on==1)
@@ -1410,7 +1467,6 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
struct _OUTER_scan_status_t* _mid=NULL; struct _OUTER_scan_status_t* _mid=NULL;
scan_data_t intval_scan_data; scan_data_t intval_scan_data;
scan_result_t *region_result=NULL; scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_scanner* my_scanner=NULL; struct Maat_scanner* my_scanner=NULL;
@@ -1420,6 +1476,9 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
{ {
clock_gettime(CLOCK_MONOTONIC,&start); clock_gettime(CLOCK_MONOTONIC,&start);
} }
_mid=grab_mid(mid, _feather, thread_num, 0);
_mid->scan_cnt++;
int virutal_table_id=0; int virutal_table_id=0;
p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_INTERVAL, &virutal_table_id); p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_INTERVAL, &virutal_table_id);
if(p_table==NULL) if(p_table==NULL)
@@ -1461,10 +1520,11 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
} }
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
struct scan_region_hit_wraper region_hit_wraper; struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, region_ret, _mid->is_last_region, virutal_table_id); scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, region_ret,
_mid->is_last_region, virutal_table_id, _mid->scan_cnt);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
&region_hit_wraper, &region_hit_wraper,
result,compile_result,rule_num, result, rule_num,
thread_num); thread_num);
assert(_mid->is_last_region<2); assert(_mid->is_last_region<2);
if(_mid->is_last_region==1) if(_mid->is_last_region==1)
@@ -1492,6 +1552,90 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
} }
int Maat_similar_scan_string(Maat_feather_t feather,int table_id
,const char* data,int data_len
,struct Maat_rule_t*result,int rule_num
,scan_status_t* mid,int thread_num)
{
int hit_region_cnt=0,compile_ret=0;
struct _OUTER_scan_status_t* _mid=NULL;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_scanner* my_scanner=NULL;
Maat_table_desc* p_table=NULL;
struct timespec start,end;
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
_mid=grab_mid(mid, _feather, thread_num, 0);
_mid->scan_cnt++;
int virtual_table_id=0;
p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
return -1;
}
my_scanner=_feather->scanner;
if(my_scanner==NULL)
{
return 0;
}
GIE_result_t* region_result=my_scanner->gie_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id);
if(table_rt->origin_rule_num==0)
{
return 0;
}
GIE_handle_t* gie_handle=table_rt->similar.gie_handle;
INC_SCANNER_REF(my_scanner,thread_num);
alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
hit_region_cnt=GIE_query(gie_handle, data, data_len, region_result, MAX_SCANNER_HIT_NUM);
if(hit_region_cnt<0)
{
DEC_SCANNER_REF(my_scanner, thread_num);
_feather->scan_err_cnt++;
return -1;
}
else if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
{
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_GIE(&region_hit_wraper, region_result, hit_region_cnt,
_mid->is_last_region, virtual_table_id, _mid->scan_cnt);
compile_ret=region_compile(_feather,_mid->inner,
&region_hit_wraper,
result, rule_num,
thread_num);
assert(_mid->is_last_region<2);
if(_mid->is_last_region==1)
{
_mid->is_last_region=2;
}
}
DEC_SCANNER_REF(my_scanner,thread_num);
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_rt,0,&start, &end,thread_num);
}
else
{
maat_stat_table(table_rt,0,NULL, NULL,thread_num);
}
if(compile_ret==0&&hit_region_cnt>0)
{
return -2;
}
return compile_ret;
}
int Maat_scan_proto_addr(Maat_feather_t feather,int table_id int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
,struct ipaddr* addr,unsigned short int proto ,struct ipaddr* addr,unsigned short int proto
,struct Maat_rule_t*result,int rule_num ,struct Maat_rule_t*result,int rule_num
@@ -1501,7 +1645,6 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
struct _OUTER_scan_status_t* _mid=NULL; struct _OUTER_scan_status_t* _mid=NULL;
scan_data_t ip_scan_data; scan_data_t ip_scan_data;
scan_result_t *region_result=NULL; scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];
Maat_table_desc* p_table=NULL; Maat_table_desc* p_table=NULL;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
@@ -1511,6 +1654,8 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
{ {
clock_gettime(CLOCK_MONOTONIC,&start); clock_gettime(CLOCK_MONOTONIC,&start);
} }
_mid=grab_mid(mid, _feather, thread_num, 0);
_mid->scan_cnt++;
int virtual_table_id=0; int virtual_table_id=0;
p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_IP, &virtual_table_id); p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_IP, &virtual_table_id);
if(p_table==NULL) if(p_table==NULL)
@@ -1584,10 +1729,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
} }
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
struct scan_region_hit_wraper region_hit_wraper; struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, region_ret, _mid->is_last_region, virtual_table_id); scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, region_ret,
_mid->is_last_region, virtual_table_id, _mid->scan_cnt);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
&region_hit_wraper, &region_hit_wraper,
result,compile_result,rule_num, result, rule_num,
thread_num); thread_num);
assert(_mid->is_last_region<2); assert(_mid->is_last_region<2);
if(_mid->is_last_region==1) if(_mid->is_last_region==1)
@@ -1693,19 +1839,24 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
int region_ret=0,hit_region_cnt=0,compile_ret=0; int region_ret=0,hit_region_cnt=0,compile_ret=0;
struct _OUTER_scan_status_t* _mid=NULL; struct _OUTER_scan_status_t* _mid=NULL;
scan_result_t *region_result; scan_result_t *region_result;
_compile_result_t compile_result[rule_num];//dynamic array
scan_data_t region_scan_data; scan_data_t region_scan_data;
struct timespec start,end; struct timespec start,end;
if(sp->feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC, &start);
}
_mid=grab_mid(mid, sp->feather, sp->thread_num, 0);
if(sp->Nth_scan==0)
{
_mid->scan_cnt++;
sp->Nth_scan=_mid->scan_cnt;
}
if(data==NULL||data_len<=0||scanner==NULL) if(data==NULL||data_len<=0||scanner==NULL)
{ {
return 0; return 0;
} }
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id); struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id);
if(sp->feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC, &start);
}
_mid=grab_mid(mid, sp->feather, sp->thread_num,0);
if(sp->version!=sp->feather->maat_version) if(sp->version!=sp->feather->maat_version)
{ {
@@ -1804,11 +1955,12 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
} }
_mid=grab_mid(mid, sp->feather,sp->thread_num, 1); _mid=grab_mid(mid, sp->feather,sp->thread_num, 1);
struct scan_region_hit_wraper region_hit_wraper; struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, sp->virtual_table_id); scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, hit_region_cnt,
_mid->is_last_region, sp->virtual_table_id, sp->Nth_scan);
compile_ret=region_compile(sp->feather,_mid->inner, compile_ret=region_compile(sp->feather,_mid->inner,
&region_hit_wraper, &region_hit_wraper,
result,compile_result,rule_num, result, rule_num,
sp->thread_num); sp->thread_num);
assert(_mid->is_last_region<2); assert(_mid->is_last_region<2);
if(_mid->is_last_region==1) if(_mid->is_last_region==1)
@@ -1819,17 +1971,19 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
{ {
if(sp->scan_buff!=NULL) if(sp->scan_buff!=NULL)
{ {
*detail_ret=fill_region_hit_detail(sp->scan_buff,_mid->inner, *detail_ret=fill_region_hit_detail(sp->scan_buff, _mid->inner,
region_result,hit_region_cnt, region_result, hit_region_cnt,
compile_result,compile_ret, result, compile_ret,
hit_detail,detail_num, scanner); sp->Nth_scan,
hit_detail, detail_num, scanner);
} }
else else
{ {
*detail_ret=fill_region_hit_detail(data,_mid->inner, *detail_ret=fill_region_hit_detail(data, _mid->inner,
region_result,hit_region_cnt, region_result, hit_region_cnt,
compile_result,compile_ret, result, compile_ret,
hit_detail,detail_num, scanner); sp->Nth_scan,
hit_detail, detail_num, scanner);
} }
} }
} }
@@ -1980,7 +2134,20 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
int do_query=0; int do_query=0;
int hit_region_cnt=0,compile_ret=0; int hit_region_cnt=0,compile_ret=0;
_compile_result_t compile_result[rule_num];//dynamic array struct _OUTER_scan_status_t* _mid=NULL;
struct timespec start, end;
if(sp->feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
_mid=grab_mid(mid, sp->feather, sp->thread_num, 0);
if(sp->Nth_scan==0)
{
_mid->scan_cnt++;
sp->Nth_scan=_mid->scan_cnt;
}
if(data==NULL||data_len<=0) if(data==NULL||data_len<=0)
{ {
return 0; return 0;
@@ -1995,12 +2162,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
GIE_handle_t* GIE_handle=table_rt->similar.gie_handle; GIE_handle_t* GIE_handle=table_rt->similar.gie_handle;
unsigned long long digest_len=0; unsigned long long digest_len=0;
char* digest_buff=NULL; char* digest_buff=NULL;
struct _OUTER_scan_status_t* _mid=NULL;
struct timespec start,end;
if(sp->feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1); alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1);
pthread_mutex_lock(&(sp->fuzzy_mutex)); pthread_mutex_lock(&(sp->fuzzy_mutex));
sp->process_offset+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset); sp->process_offset+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset);
@@ -2043,10 +2205,11 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
} }
_mid=grab_mid(mid,sp->feather, sp->thread_num,1); _mid=grab_mid(mid,sp->feather, sp->thread_num,1);
struct scan_region_hit_wraper region_hit_wraper; struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_GIE(&region_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, sp->virtual_table_id); scan_region_hit_wraper_build_with_GIE(&region_hit_wraper, region_result, hit_region_cnt,
_mid->is_last_region, sp->virtual_table_id, sp->Nth_scan);
compile_ret=region_compile(sp->feather,_mid->inner, compile_ret=region_compile(sp->feather,_mid->inner,
&region_hit_wraper, &region_hit_wraper,
result,compile_result,rule_num, result, rule_num,
sp->thread_num); sp->thread_num);
assert(_mid->is_last_region<2); assert(_mid->is_last_region<2);
if(_mid->is_last_region==1) if(_mid->is_last_region==1)
@@ -2160,87 +2323,32 @@ int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCA
} }
return 0; return 0;
} }
int Maat_similar_scan_string(Maat_feather_t feather,int table_id int Maat_get_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_SCAN_OPT type, void* value, int size)
,const char* data,int data_len
,struct Maat_rule_t*result,int rule_num
,scan_status_t* mid,int thread_num)
{ {
int hit_region_cnt=0,compile_ret=0;
struct _OUTER_scan_status_t* _mid=NULL;
_compile_result_t compile_result[rule_num];
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_scanner* my_scanner=NULL; struct _OUTER_scan_status_t* _mid=NULL;
Maat_table_desc* p_table=NULL; struct Maat_hit_path_t* paths;
struct timespec start,end; _mid=grab_mid(mid, _feather, 0, 0);
if(_feather->perf_on==1) if(_mid==NULL||_mid->inner==NULL)
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
int virtual_table_id=0;
p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
return -1;
}
my_scanner=_feather->scanner;
if(my_scanner==NULL)
{ {
return 0; return 0;
} }
GIE_result_t* region_result=my_scanner->gie_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; switch(type)
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id);
if(table_rt->origin_rule_num==0)
{ {
return 0; case MAAT_GET_SCAN_HIT_PATH:
if(value==NULL||size<=0||size%sizeof(struct Maat_hit_path_t)!=0)
{
return -1;
}
paths=(struct Maat_hit_path_t*)value;
break;
default:
return -1;
} }
GIE_handle_t* gie_handle=table_rt->similar.gie_handle; return 0;
INC_SCANNER_REF(my_scanner,thread_num);
alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
hit_region_cnt=GIE_query(gie_handle, data, data_len, region_result, MAX_SCANNER_HIT_NUM);
if(hit_region_cnt<0)
{
DEC_SCANNER_REF(my_scanner, thread_num);
_feather->scan_err_cnt++;
return -1;
}
else if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
{
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_GIE(&region_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, virtual_table_id);
compile_ret=region_compile(_feather,_mid->inner,
&region_hit_wraper,
result,compile_result,rule_num,
thread_num);
assert(_mid->is_last_region<2);
if(_mid->is_last_region==1)
{
_mid->is_last_region=2;
}
}
DEC_SCANNER_REF(my_scanner,thread_num);
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_rt,0,&start, &end,thread_num);
}
else
{
maat_stat_table(table_rt,0,NULL, NULL,thread_num);
}
if(compile_ret==0&&hit_region_cnt>0)
{
return -2;
}
return compile_ret;
} }
int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* value,int size)
int Maat_read_state(Maat_feather_t feather, enum MAAT_STATE_OPT type, void* value, int size)
{ {
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int * int_val=(int *)value; int * int_val=(int *)value;
@@ -2285,6 +2393,27 @@ int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* valu
return 0; return 0;
} }
void Maat_clean_status(scan_status_t* mid)
{
struct _OUTER_scan_status_t* _mid=NULL;
if(*mid==NULL)
{
return;
}
_mid=(struct _OUTER_scan_status_t*)(*mid);
alignment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1);
if(_mid->inner!=NULL)
{
scan_hit_status_free(_mid->inner);
_mid->inner=NULL;
alignment_int64_array_add(_mid->feather->inner_mid_cnt, _mid->thread_num, -1);
}
_mid->feather=NULL;
free(_mid);
*mid=NULL;
return;
}
int Maat_helper_read_column(const char* line, int Nth_column, size_t *column_offset, size_t *column_len) int Maat_helper_read_column(const char* line, int Nth_column, size_t *column_offset, size_t *column_len)
{ {
return get_column_pos(line, Nth_column, column_offset, column_len); return get_column_pos(line, Nth_column, column_offset, column_len);

2
src/entry/bool_matcher.cpp
View File

@@ -160,7 +160,7 @@ struct bool_matcher * bool_matcher_new(struct bool_expr * exprs, size_t expr_num
return matcher; return matcher;
} }
int bool_matcher_match(struct bool_matcher * matcher, unsigned int thread_id, unsigned long long * item_ids, size_t item_num, void ** result, size_t size) int bool_matcher_match(struct bool_matcher * matcher, unsigned int thread_id, const unsigned long long * item_ids, size_t item_num, void ** result, size_t size)
{ {
if(matcher==NULL) return -1; if(matcher==NULL) return -1;
if(thread_id>=matcher->max_thread_num) return -1; if(thread_id>=matcher->max_thread_num) return -1;

22
src/inc_internal/Maat_rule_internal.h
View File

@@ -20,6 +20,7 @@
#include <pthread.h> #include <pthread.h>
#include <iconv.h> #include <iconv.h>
#include <openssl/md5.h> #include <openssl/md5.h>
#include <sys/queue.h>
@@ -165,13 +166,24 @@ struct _compile_result_t
int compile_id; int compile_id;
struct bool_expr group_set; struct bool_expr group_set;
}; };
struct _INNER_scan_status_t struct Maat_hit_path_inner
{ {
size_t cur_hit_group_cnt; int Nth_hit_region;
struct Maat_hit_path_t path;
TAILQ_ENTRY(Maat_hit_path_inner) entries;
};
TAILQ_HEAD(hit_path_q, Maat_hit_path_inner);
struct scan_hit_status
{
size_t hit_path_cnt;
struct hit_path_q hit_path_qhead;
size_t all_hit_group_cnt; size_t all_hit_group_cnt;
size_t all_hit_group_array_sz; size_t all_hit_group_array_sz;
struct dynamic_array_t* cur_hit_groups;
unsigned long long *all_hit_group_array; unsigned long long *all_hit_group_array;
char not_grp_compile_hitted_flag; char not_grp_compile_hitted_flag;
}; };
struct _OUTER_scan_status_t struct _OUTER_scan_status_t
@@ -181,7 +193,8 @@ struct _OUTER_scan_status_t
unsigned char is_set_district; unsigned char is_set_district;
unsigned char is_last_region; unsigned char is_last_region;
int district_id; int district_id;
struct _INNER_scan_status_t* inner; int scan_cnt;
struct scan_hit_status* inner;
}; };
enum maat_garbage_type enum maat_garbage_type
{ {
@@ -202,6 +215,7 @@ struct _stream_para_t
{ {
struct _Maat_feather_t* feather; struct _Maat_feather_t* feather;
const struct Maat_table_desc* p_real_table; const struct Maat_table_desc* p_real_table;
int Nth_scan;
int virtual_table_id; int virtual_table_id;
int version; int version;
int thread_num; int thread_num;

2
src/inc_internal/bool_matcher.h
View File

@@ -45,7 +45,7 @@ extern "C"
/*注意：本函数调用会交换bool_exprs中元素的位置*/ /*注意：本函数调用会交换bool_exprs中元素的位置*/
struct bool_matcher * bool_matcher_new(struct bool_expr * exprs, size_t expr_num, unsigned int max_thread_num, size_t * mem_size); struct bool_matcher * bool_matcher_new(struct bool_expr * exprs, size_t expr_num, unsigned int max_thread_num, size_t * mem_size);
int bool_matcher_match(struct bool_matcher * matcher, unsigned int thread_id, unsigned long long * item_ids, size_t item_num, void ** result, size_t size); int bool_matcher_match(struct bool_matcher * matcher, unsigned int thread_id, const unsigned long long * item_ids, size_t item_num, void ** result, size_t size);
void bool_matcher_free(struct bool_matcher * matcher); void bool_matcher_free(struct bool_matcher * matcher);

95
test/test_maatframe.cpp
View File

@@ -459,6 +459,53 @@ TEST(StringScan, ExprPlusWithHex)
return; return;
} }
TEST(StringScan, CharsetWindows1251)
{
int table_id=0,ret=0;
int read_size=0,pass_flag=0;
struct Maat_rule_t result[4];
scan_status_t mid=NULL;
//const char* fn="./testdata/mesa_logo.jpg";
const char* table_name="KEYWORDS_TABLE";
const char* fn="./testdata/charsetWindows1251.txt";
FILE* fp=fopen(fn,"r");
ASSERT_FALSE(fp==NULL);
char scan_data[4096]={0};
table_id=Maat_table_register(g_feather,table_name);
ASSERT_GT(table_id, 0);
struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10);
stream_para_t sp=Maat_stream_scan_string_start(g_feather,table_id,0);
int detail_ret=0;
ASSERT_FALSE(sp==NULL);
while(0==feof(fp))
{
read_size=fread(scan_data,1,sizeof(scan_data),fp);
ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data,read_size
,result,4,hit_detail,10
,&detail_ret,&mid);
if(ret>0)
{
pass_flag=1;
break;
}
}
EXPECT_EQ(pass_flag, 1);
EXPECT_EQ(result[0].config_id, 164);
Maat_stream_scan_string_end(&sp);
free(hit_detail);
fclose(fp);
Maat_clean_status(&mid);
return;
}
#define TEST_IPSCAN #define TEST_IPSCAN
TEST(IPScan, IPv4_mask) TEST(IPScan, IPv4_mask)
{ {
@@ -629,6 +676,7 @@ TEST(IPScan, IPv6_CIDR)
return; return;
} }
#define TEST_NOTLogic 1 #define TEST_NOTLogic 1
TEST(NOTLogic, OneRegion) TEST(NOTLogic, OneRegion)
@@ -661,53 +709,6 @@ TEST(NOTLogic, OneRegion)
} }
TEST(StringScan, CharsetWindows1251)
{
int table_id=0,ret=0;
int read_size=0,pass_flag=0;
struct Maat_rule_t result[4];
scan_status_t mid=NULL;
//const char* fn="./testdata/mesa_logo.jpg";
const char* table_name="KEYWORDS_TABLE";
const char* fn="./testdata/charsetWindows1251.txt";
FILE* fp=fopen(fn,"r");
ASSERT_FALSE(fp==NULL);
char scan_data[4096]={0};
table_id=Maat_table_register(g_feather,table_name);
ASSERT_GT(table_id, 0);
struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10);
stream_para_t sp=Maat_stream_scan_string_start(g_feather,table_id,0);
int detail_ret=0;
ASSERT_FALSE(sp==NULL);
while(0==feof(fp))
{
read_size=fread(scan_data,1,sizeof(scan_data),fp);
ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data,read_size
,result,4,hit_detail,10
,&detail_ret,&mid);
if(ret>0)
{
pass_flag=1;
break;
}
}
EXPECT_EQ(pass_flag, 1);
EXPECT_EQ(result[0].config_id, 164);
Maat_stream_scan_string_end(&sp);
free(hit_detail);
fclose(fp);
Maat_clean_status(&mid);
return;
}
TEST(NOTLogic, ScanNotAtLast) TEST(NOTLogic, ScanNotAtLast)
{ {
const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-144."; const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-144.";