gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

代码适配Maat command、maat json。

Browse Source
This commit is contained in:
zhengchao
2020-06-13 21:05:42 +08:00
parent 7e1cb56d4f
commit 2c80ba4c0a
22 changed files with 935 additions and 1513 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
inc/Maat_command.h
View File

@@ -1,7 +1,7 @@
#ifndef H_MAAT_COMMAND_H_INCLUDE
#define H_MAAT_COMMAND_H_INCLUDE
#ifndef __cplusplus
#error("This file should be compiled with C++ compiler")
#ifdef __cplusplus
extern "C"{
#endif
#include "Maat_rule.h"
enum MAAT_OPERATION
@@ -106,17 +106,22 @@ struct Maat_region_t
struct Maat_rgn_sim_t similarity_rule;
};
};
struct Maat_group_t
struct Maat_cmd_region
{
const char* table_name;
const char* virtual_table_name;
int group_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number.
int parent_id;
int not_flag;
enum MAAT_GROUP_RELATION parent_type;
int region_num;
struct Maat_region_t *regions;
int region_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must appoint a unique number.
enum MAAT_REGION_TYPE region_type;
union
{
struct Maat_rgn_str_t expr_rule;
struct Maat_rgn_addr_t ip_rule;
struct Maat_rgn_intv_t interval_rule;
struct Maat_rgn_digest_t digest_rule;
struct Maat_rgn_sim_t similarity_rule;
};
};
struct Maat_cmd_t
{
//This Struct MUST alloced by Maat_create_cmd(), then released by Maat_free_cmd().
@@ -126,7 +131,7 @@ struct Maat_cmd_t
int label_id; //>0, to be indexed and quried by Maat_cmd_select; =0 not index
struct Maat_group_t* groups;// Add regions with Maat_add_region2cmd
};
struct Maat_line_t
struct Maat_cmd_line
{
const char* table_name;
const char* table_line;
@@ -134,37 +139,15 @@ struct Maat_line_t
int label_id;
int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout.
};
struct Maat_cmd_t* Maat_create_cmd(const struct Maat_rule_t* rule, int group_num);
int Maat_cmd_set_opt(struct Maat_cmd_t* cmd, enum MAAT_RULE_OPT type, const char* val, int size);
//input: which_group 0~group_num
//input: region can be freed after added.
void Maat_add_region2cmd(struct Maat_cmd_t* cmd,int which_group,const struct Maat_region_t* region);
void Maat_free_cmd(struct Maat_cmd_t* cmd);
int Maat_format_cmd(struct Maat_cmd_t* cmd, char* buffer, int size);
//Input string of REGION_EXPR and REGION_SIMILARITY need to be escapeed.
char* Maat_str_escape(char* dst,int size,const char*src);
//Deletion failed due to not complete synchronize with Redis.
//To make sure the delete command is excecuted, user should try again after MAAT_OPT_SCANDIR_INTERVAL_MS ms.
//Returns number of successfully updated rule.
//The following functions are NOT thread safe.
int Maat_cmd(Maat_feather_t feather,struct Maat_cmd_t* cmd,enum MAAT_OPERATION op);
//pipeline model
int Maat_cmd_append(Maat_feather_t feather,struct Maat_cmd_t* cmd,enum MAAT_OPERATION op);
//Return number of successfully updated rule.
//Return -1 for failed.
int Maat_cmd_commit(Maat_feather_t feather);
int Maat_cmd_set_group(Maat_feather_t feather, int group_id, const struct Maat_region_t* region, enum MAAT_OPERATION op);
//Returns number of successfully updated rule.
//Return -1 for failed.
int Maat_cmd_set_line(Maat_feather_t feather,const struct Maat_line_t* line_rule, enum MAAT_OPERATION op);
int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_rule, int line_num ,enum MAAT_OPERATION op);
int Maat_cmd_set_line(Maat_feather_t feather,const struct Maat_cmd_line* line_rule, enum MAAT_OPERATION op);
int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_rule, int line_num ,enum MAAT_OPERATION op);
int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value, size_t size, enum MAAT_OPERATION op);
//Return the value of key after the increment.
@@ -180,13 +163,38 @@ int Maat_cmd_key_select(Maat_feather_t feather, int label_id, struct Maat_cmd_ke
int Maat_cmd_select(Maat_feather_t feather, int label_id, int * output_ids, unsigned int size);
int Maat_cmd_flushDB(Maat_feather_t feather);
int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num);
int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_region_t* region, int group_id);
int Maat_command_raw_set_group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_group_t* group);
struct Maat_cmd_group2group
{
const char* table_name;
int group_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number.
int superior_group_id;
};
struct Maat_cmd_group2compile
{
const char* table_name;
const char* virtual_table_name;
int group_id;
int compile_id;
int Nth_clause;
int not_flag;
};
int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_region* region, int group_id);
int Maat_command_raw_set_group2group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g);
int Maat_command_raw_set_group2compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c);
//@param expire_after: expired after $expire_after$ seconds, set to 0 for never timeout.
//@param label_id: bigger than 0 means this compile rule is to be indexed and quried by Maat_cmd_select; =0 not index
int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int clause_num, int label_id, int expire_after);
int Maat_cmd_get_new_group_id(Maat_feather_t feather);
int Maat_cmd_get_new_region_id(Maat_feather_t feather);
#ifdef __cplusplus
} //end extern"C"
#endif
#endif

22
inc/Maat_rule.h
View File

@@ -5,16 +5,14 @@
* Her feather was the measure that determined whether the souls (considered
* to reside in the heart) of the departed would reach the paradise of afterlife
* successfully.
* Author: zhengchao@iie.ac.cn, MESA
* Version 2018-12-07 Plugin Extra Data.
* NOTE: MUST compile with G++
* All right reserved by Institute of Infomation Engineering,Chinese Academic of Science 2014~2018
* Author: zhengchao, MESA
* Version 2020-06-13 version 3.0.0
*********************************************************
*/
#ifndef H_MAAT_RULE_H_INCLUDE
#define H_MAAT_RULE_H_INCLUDE
#ifndef __cplusplus
#error("This file should be compiled with C++ compiler")
#ifdef __cplusplus
extern "C"{
#endif
#include <MESA/stream.h>
enum MAAT_CHARSET
@@ -56,10 +54,10 @@ struct Maat_rule_t
{
int config_id;
int service_id;
char do_log;
char do_blacklist;
char action;
char reserved;
unsigned char do_log;
unsigned char do_blacklist;
unsigned char action;
unsigned char reserved;
int serv_def_len;
char service_defined[MAX_SERVICE_DEFINE_LEN];
};
@@ -328,6 +326,10 @@ enum MAAT_RULE_OPT
};
int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum MAAT_RULE_OPT type, void* value, int size);
#ifdef __cplusplus
}//end extern "C"
#endif
#endif // H_MAAT_RULE_H_INCLUDE

2
src/CMakeLists.txt
View File

@@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
include_directories(/opt/MESA/include/MESA/)

382
src/entry/Maat_api.cpp
View File

@@ -9,6 +9,8 @@
#include "Maat_rule.h"
#include "Maat_rule_internal.h"
#include "Maat_utils.h"
#include "Maat_garbage_collection.h"
#include "Maat_hierarchy.h"
#include "dynamic_array.h"
#include "alignment_int64.h"
#include "config_monitor.h"
@@ -33,8 +35,8 @@ inline int scan_status_should_compile_NOT(struct _OUTER_scan_status_t* _mid)
{
if( _mid &&
_mid->is_last_region==1 &&
_mid->inner &&
_mid->inner->not_grp_compile_hitted_flag)
_mid->compile_mid &&
Maat_hierarchy_compile_mid_has_NOT_clause(_mid->compile_mid))
{
return 1;
}
@@ -43,24 +45,7 @@ inline int scan_status_should_compile_NOT(struct _OUTER_scan_status_t* _mid)
return 0;
}
}
int exprid2region_id(struct Maat_group_inner* group_rule, int expr_id, int* district_id, struct Maat_scanner* scanner)
{
int region_id=-1;
struct Maat_region_inner* region_rule=NULL;
assert(group_rule->group_id>=0);
int array_idx=(int)(long)HASH_fetch_by_id(scanner->exprid_hash, expr_id);
pthread_mutex_lock(&(group_rule->mutex));
assert(array_idx<group_rule->region_boundary);
region_rule=(struct Maat_region_inner*)dynamic_array_read(group_rule->regions, array_idx);
if(region_rule)
{
assert(expr_id>=region_rule->expr_id_lb&&expr_id<=region_rule->expr_id_ub);
region_id=region_rule->region_id;
*district_id=region_rule->district_id;
}
pthread_mutex_unlock(&(group_rule->mutex));
return region_id;
}
//return 1 if insert a unique id
//return 0 if id is duplicated
@@ -166,17 +151,16 @@ size_t Maat_rule_sort_by_evaluation_order(Maat_feather_t feather, struct Maat_ru
memcpy(copy_rule_array, rule_array, sizeof(struct Maat_rule_t)*n_rule);
struct Maat_compile_inner *p=NULL;
struct Maat_compile_rule *p=NULL;
size_t i=0, j=0;
for(i=0; i<n_rule; i++)
{
p=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule_array[i].config_id);
if(p && 0==pthread_rwlock_tryrdlock(&(p->rwlock)))//rule maybe already deleted.
p=(struct Maat_compile_rule*)Maat_hierarchy_compile_read_user_data(_feather->scanner->hier, rule_array[i].config_id);
if(p)
{
compile_sort_para_set(sort_para+i, p, copy_rule_array+i);
compile_sort_para_set(sort_para+j, p, copy_rule_array+j);
j++;
pthread_rwlock_unlock(&(p->rwlock));
}
}
qsort(sort_para, j, sizeof(struct compile_sort_para),
@@ -195,174 +179,12 @@ struct scan_region_hit_wraper
int* virtual_table_ids;
size_t elem_size;
size_t n_elem;
size_t group_offset;
size_t user_data_offset;
size_t expr_id_offset;
int virtual_table_id;
int is_last_region;
};
struct scan_hit_status* scan_hit_status_new(void)
{
struct scan_hit_status* inner_mid=NULL;
inner_mid=ALLOC(struct scan_hit_status, 1);
TAILQ_INIT(&inner_mid->hit_path_qhead);
return inner_mid;
}
void scan_hit_status_free(struct scan_hit_status* p)
{
struct Maat_hit_path_inner * tmp = TAILQ_FIRST(&p->hit_path_qhead);
while(tmp != NULL)
{
TAILQ_REMOVE(&p->hit_path_qhead, tmp, entries);
free(tmp);
p->hit_path_cnt--;
tmp = TAILQ_FIRST(&p->hit_path_qhead);
}
assert(p->hit_path_cnt==0);
free(p->all_hit_group_array);
p->all_hit_group_array=NULL;
free(p);
}
void hit_path_init(struct Maat_hit_path_t* hit_path)
{
hit_path->Nth_scan=-1;
hit_path->region_id=-1;
hit_path->sub_group_id=-1;
hit_path->top_group_id=-1;
hit_path->vt_id=-1;
hit_path->compile_id=-1;
}
size_t scan_hit_status_select_hit_path_inner(const struct hit_path_q *hit_path_qhead, struct Maat_hit_path_t* condition,
struct Maat_hit_path_inner* hit_paths, size_t n_path)
{
struct Maat_hit_path_inner* p=NULL;
size_t i=0;
TAILQ_FOREACH(p, hit_path_qhead, entries)
{
if((condition->compile_id==p->path.compile_id||condition->compile_id<0)
&& (condition->Nth_scan==p->path.Nth_scan||condition->Nth_scan<0)
&& (condition->region_id=p->path.region_id||condition->region_id<0)
&& (condition->sub_group_id=p->path.sub_group_id||condition->sub_group_id<0)
&& (condition->top_group_id=p->path.top_group_id||condition->top_group_id<0))
{
if(i<n_path)
{
hit_paths[i]=*p;
i++;
}
else if(hit_paths==NULL)//count only
{
i++;
}
}
}
return i;
}
void scan_hit_status_update_by_group(struct scan_hit_status* hit_status, struct Maat_group_inner* group_rule, int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result)
{
size_t i=0;
int ret=0;
struct Maat_hit_path_inner* hit_path=NULL;
if(group_rule->top_group_cnt==0)
{
hit_path=ALLOC(struct Maat_hit_path_inner, 1);
hit_path_init(&(hit_path->path));
hit_path->Nth_hit_region=Nth_region_result;
hit_path->path.Nth_scan=Nth_scan;
hit_path->path.region_id=region_id;
hit_path->path.sub_group_id=group_rule->group_id;
hit_path->path.vt_id=virtual_table_id;
TAILQ_INSERT_TAIL(&hit_status->hit_path_qhead, hit_path, entries);
hit_status->hit_path_cnt++;
}
else
{
for(i=0; i<group_rule->top_group_cnt; i++)
{
hit_path=ALLOC(struct Maat_hit_path_inner, 1);
hit_path_init(&(hit_path->path));
hit_path->Nth_hit_region=Nth_region_result;
hit_path->path.Nth_scan=Nth_scan;
hit_path->path.region_id=region_id;
hit_path->path.sub_group_id=group_rule->group_id;
hit_path->path.top_group_id=group_rule->top_groups[i];
hit_path->path.vt_id=virtual_table_id;
TAILQ_INSERT_TAIL(&hit_status->hit_path_qhead, hit_path, entries);
hit_status->hit_path_cnt++;
ret=insert_set_id(&(hit_status->all_hit_group_array),
&(hit_status->all_hit_group_array_sz),
hit_status->all_hit_group_cnt,
TO_RELATION_ID(virtual_table_id, group_rule->top_groups[i]));
hit_status->all_hit_group_cnt+=ret;
}
}
return;
}
size_t scan_hit_status_update_by_compile(struct scan_hit_status* hit_status, struct Maat_compile_inner* compile_rule, int Nth_scan)
{
size_t i=0, j=0;
struct Maat_hit_path_inner* p=NULL, *q=NULL;
struct bool_expr a_set;
unsigned char has_not=0;
struct Maat_hit_path_t condition;
size_t n_exsited_path=0;
struct hit_path_q shared_path_qhead;
TAILQ_INIT(&shared_path_qhead);
make_group_set(compile_rule, &a_set, &has_not);
for(i=0; i<a_set.item_num; i++)
{
TAILQ_FOREACH(p, &hit_status->hit_path_qhead, entries)
{
n_exsited_path=0;
if(TO_RELATION_ID(p->path.vt_id, p->path.top_group_id)==a_set.items[i].item_id
&& p->path.compile_id!=compile_rule->compile_id)
{
if(p->path.compile_id<0)
{
p->path.compile_id=compile_rule->compile_id;
}
else //current path already have a compile as endpoint, so we duplicate a new path.
{
condition=p->path;
condition.compile_id=compile_rule->compile_id;
n_exsited_path=scan_hit_status_select_hit_path_inner(&shared_path_qhead, &condition, NULL, 0);
if(n_exsited_path==0)
{
q=ALLOC(struct Maat_hit_path_inner, 1);
*q=*p;
q->path.compile_id=compile_rule->compile_id;
TAILQ_INSERT_TAIL(&shared_path_qhead, q, entries);
hit_status->hit_path_cnt++;
}
}
if(p->path.Nth_scan==Nth_scan && n_exsited_path==0)//Compile was satisfied by new region hits.
{
j++;
}
}
}
}
p = TAILQ_FIRST(&shared_path_qhead);
while(p != NULL)
{
TAILQ_REMOVE(&shared_path_qhead, p, entries);
TAILQ_INSERT_TAIL(&hit_status->hit_path_qhead, p, entries);
p = TAILQ_FIRST(&shared_path_qhead);
}
return j;
}
size_t scan_hit_status_read_group_ids(const struct scan_hit_status* hit_status, const unsigned long long** const grp_ids)
{
*grp_ids=hit_status->all_hit_group_array;
return hit_status->all_hit_group_cnt;
}
void scan_region_hit_wraper_build_with_rulescan(struct scan_region_hit_wraper* region_hit, scan_result_t* rulescan_rslt, size_t n_rslt, int is_last_region, int virtual_table_id, int Nth_scan)
{
@@ -371,7 +193,7 @@ void scan_region_hit_wraper_build_with_rulescan(struct scan_region_hit_wraper* r
region_hit->n_elem=n_rslt;
region_hit->elem_size=sizeof(scan_result_t);
region_hit->expr_id_offset=offsetof(scan_result_t, expr_id);
region_hit->group_offset=offsetof(scan_result_t, tag);
region_hit->user_data_offset=offsetof(scan_result_t, tag);
region_hit->is_last_region=is_last_region;
region_hit->virtual_table_id=virtual_table_id;
region_hit->Nth_scan=Nth_scan;
@@ -385,7 +207,7 @@ void scan_region_hit_wraper_build_with_GIE(struct scan_region_hit_wraper* region
region_hit->n_elem=n_rslt;
region_hit->elem_size=sizeof(GIE_result_t);
region_hit->expr_id_offset=offsetof(GIE_result_t, id);
region_hit->group_offset=offsetof(GIE_result_t, tag);
region_hit->user_data_offset=offsetof(GIE_result_t, tag);
region_hit->is_last_region=is_last_region;
region_hit->virtual_table_id=virtual_table_id;
region_hit->Nth_scan=Nth_scan;
@@ -393,35 +215,26 @@ void scan_region_hit_wraper_build_with_GIE(struct scan_region_hit_wraper* region
return;
}
int region_compile(_Maat_feather_t*feather, struct scan_hit_status *_mid, struct Maat_hierarchy_compile_mid* compile_mid, const struct scan_region_hit_wraper* region_hit_wraper, struct Maat_rule_t* result, int size,int thread_num)
int region_compile(_Maat_feather_t*feather, struct Maat_hierarchy_compile_mid* compile_mid, const struct scan_region_hit_wraper* region_hit_wraper, struct Maat_rule_t* result, int size,int thread_num)
{
int is_last_region=region_hit_wraper->is_last_region;
void* region_hit=region_hit_wraper->elem_array;
size_t region_type_size=region_hit_wraper->elem_size;
size_t expr_id_offset=region_hit_wraper->expr_id_offset;
size_t group_offset=region_hit_wraper->group_offset;
size_t user_data_offset=region_hit_wraper->user_data_offset;
size_t region_hit_num=region_hit_wraper->n_elem;
unsigned int expr_id=0;
int region_id=0;
int scan_ret=0, result_cnt=0;
int tmp=0;
size_t i=0;
size_t r_in_c_cnt=0;
int scan_ret=0;
int i=0;
struct Maat_compile_rule* compile_rule_array[MAX_SCANNER_HIT_NUM];
struct Maat_compile_rule* compile_rule=NULL;
int virtual_table_id=0;
const unsigned long long* hit_group_ids=NULL;
size_t hit_group_id_cnt=0;
struct Maat_region_inner* region=NULL;
int region_ids[MAX_SCANNER_HIT_NUM];
for(i=0;i<region_hit_num;i++)
for(i=0; (size_t)i<region_hit_num;i++)
{
region=*(struct Maat_region_inner**)((char*)region_hit+region_type_size*i+group_offset);
region_ids[i]=region->region_id;
region=*(struct Maat_region_inner**)((char*)region_hit+region_type_size*i+user_data_offset);
if(region_hit_wraper->virtual_table_ids)
{
virtual_table_id=region_hit_wraper->virtual_table_ids[i];
@@ -430,64 +243,48 @@ int region_compile(_Maat_feather_t*feather, struct scan_hit_status *_mid, struct
{
virtual_table_id=region_hit_wraper->virtual_table_id;
}
Maat_hierarchy_compile_mid_udpate(compile_mid, region_id, virtual_table_id, region_hit_wraper->Nth_scan, i);
Maat_hierarchy_compile_mid_udpate(compile_mid, region->region_id, virtual_table_id, region_hit_wraper->Nth_scan, i);
}
scan_ret=Maat_hierarchy_region_compile(compile_mid, int * region_ids, int * virtual_tables, size_t n_region, int Nth_scan, compile_rule_array, MAX_SCANNER_HIT_NUM);
scan_ret=Maat_hierarchy_region_compile(compile_mid, is_last_region, (void**)compile_rule_array, MAX_SCANNER_HIT_NUM);
if(scan_ret>1)
{
qsort(compile_rule_array, scan_ret, sizeof(struct Maat_compile_rule*),
compare_compile_rule);
}
for(i=0; i<(unsigned int)scan_ret&&result_cnt<size; i++)
for(i=0; i<scan_ret&&i<size; i++)
{
compile_rule=compile_rule_array[i];
assert(compile_rule->magic_num==COMPILE_RULE_MAGIC);
if(compile_rule->not_group_cnt>0 && !is_last_region)
{
_mid->not_grp_compile_hitted_flag=1;
}
else
{
r_in_c_cnt=scan_hit_status_update_by_compile(_mid, compile_rule, region_hit_wraper->Nth_scan);
if(r_in_c_cnt>0 || //compile hitted becasue of new reigon
region_hit_num==0) //or hit a compile that refer a NOT-logic group in previous scan.
{
fill_maat_rule(&(result[result_cnt]), &(compile_rule->compile->head),
compile_rule->compile->service_defined, compile_rule->compile->head.serv_def_len);
result_cnt++;
}
}
fill_maat_rule(&(result[i]), &(compile_rule->head),
compile_rule->service_defined, compile_rule->head.serv_def_len);
}
if(result_cnt>0)
if(scan_ret>0)
{
alignment_int64_array_add(feather->hit_cnt, thread_num, 1);
}
if(region_hit_num==0&&result_cnt>0)
if(region_hit_num==0&&scan_ret>0)
{
alignment_int64_array_add(feather->not_grp_hit_cnt, thread_num, 1);
}
return result_cnt;
return scan_ret;
}
int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num, Maat_scanner* scanner)
int match_district(struct _OUTER_scan_status_t *_mid, scan_result_t *region_hit, int region_hit_num)
{
struct Maat_group_inner* group_rule=NULL;
struct Maat_region_inner* region=NULL;
int i=0;
int district_id=-1,region_id=-1;
int ret_region_num=region_hit_num;
while(i<ret_region_num)
{
group_rule=(struct Maat_group_inner*)(region_hit[i].tag);
region_id=exprid2region_id(group_rule, region_hit[i].expr_id,&district_id, scanner);
if(region_id>0&&district_id!=_mid->district_id)
region=(struct Maat_region_inner*)(region_hit[i].tag);
if(region->district_id!=_mid->district_id)
{
ret_region_num--;
//todo: replace memmove with high performance solution.
@@ -568,7 +365,7 @@ int hit_pos_RS2Maat(struct sub_item_pos_t* maat_sub_item,int size,rule_result_t*
}
return k;
}
int fill_region_hit_detail(const char* scan_buff, const scan_hit_status* _mid,
int fill_region_hit_detail(const char* scan_buff, const Maat_hierarchy_compile_mid* mid,
scan_result_t *region_hit, int region_cnt,
struct Maat_rule_t* compile, int compile_cnt,
int Nth_scan,
@@ -578,13 +375,12 @@ int fill_region_hit_detail(const char* scan_buff, const scan_hit_status* _mid,
int i=0;
size_t j=0;
char r_in_c_flag[region_cnt];
int district_id=-1; //Indifference
memset(r_in_c_flag, 0, sizeof(r_in_c_flag));
struct Maat_group_inner* group_rule=NULL;
struct Maat_region_inner* region=NULL;
struct Maat_hit_path_inner hit_paths[MAX_SCANNER_HIT_NUM];
struct Maat_hierarchy_hit_path* hit_paths[MAX_SCANNER_HIT_NUM];
struct Maat_hit_path_t condition;
size_t hit_path_cnt=0;
int Nth_hit_region=0;
@@ -593,14 +389,14 @@ int fill_region_hit_detail(const char* scan_buff, const scan_hit_status* _mid,
for(i=0; i<compile_cnt&&i<detail_num; i++)
{
hit_detail[i].config_id=compile[i].config_id;
hit_path_init(&condition);
Maat_hit_path_init(&condition);
condition.Nth_scan=Nth_scan;
condition.compile_id=compile[i].config_id;
hit_path_cnt=scan_hit_status_select_hit_path_inner(&_mid->hit_path_qhead, &condition, hit_paths, MAX_SCANNER_HIT_NUM);
hit_path_cnt=Maat_hierarchy_hit_path_select0(mid, &condition, hit_paths, MAX_SCANNER_HIT_NUM);
for(j=0; j<hit_path_cnt && j<MAAT_MAX_HIT_RULE_NUM; j++)
{
Nth_hit_region=hit_paths[j].Nth_hit_region;
hit_detail[i].region_pos[j].region_id=hit_paths[j].path.region_id;
Nth_hit_region=hit_paths[j]->Nth_hit_region;
hit_detail[i].region_pos[j].region_id=hit_paths[j]->path.region_id;
hit_detail[i].region_pos[j].sub_item_num=region_hit[Nth_hit_region].rnum;
hit_pos_RS2Maat(hit_detail[i].region_pos[j].sub_item_pos, MAAT_MAX_EXPR_ITEM_NUM,
region_hit[Nth_hit_region].result, region_hit[Nth_hit_region].rnum, scan_buff);
@@ -613,10 +409,10 @@ int fill_region_hit_detail(const char* scan_buff, const scan_hit_status* _mid,
{
if(r_in_c_flag[j]==0)
{
group_rule=(struct Maat_group_inner*)(region_hit[j].tag);
region=(struct Maat_region_inner*)(region_hit[j].tag);
hit_detail[i].config_id=-2;
hit_detail[i].hit_region_cnt=1;
hit_detail[i].region_pos[0].region_id=exprid2region_id(group_rule, region_hit[j].expr_id, &district_id, scanner);
hit_detail[i].region_pos[0].region_id=region->region_id;
hit_detail[i].region_pos[0].sub_item_num=region_hit[j].rnum;
hit_pos_RS2Maat(hit_detail[i].region_pos[0].sub_item_pos,MAAT_MAX_EXPR_ITEM_NUM,
region_hit[j].result, region_hit[j].rnum, scan_buff);
@@ -647,11 +443,10 @@ struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* fe
if(is_hit_region==1)
{
if(_mid->inner==NULL)
if(_mid->compile_mid==NULL)
{
_mid->compile_mid=Maat_hierarchy_compile_mid_new(feather->scanner->hier, thread_num);
_mid->inner=scan_hit_status_new();
alignment_int64_array_add(feather->inner_mid_cnt,thread_num,1);
alignment_int64_array_add(feather->compile_mid_cnt, thread_num, 1);
}
}
return _mid;
@@ -704,7 +499,7 @@ int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char
}
ret=json2iris(json_buff,
maat_json_fn,
feather->compile_tn, feather->group_tn,
feather->compile_tn, feather->group2compile_tn, feather->group2group_tn,
NULL,
feather->json_ctx.iris_file,
sizeof(feather->json_ctx.iris_file),
@@ -748,7 +543,8 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
goto failed;
}
Maat_table_get_compile_table_name(feather->table_mgr, feather->compile_tn, sizeof(feather->compile_tn));
Maat_table_get_group_table_name(feather->table_mgr, feather->group_tn, sizeof(feather->group_tn));
Maat_table_get_group2compile_table_name(feather->table_mgr, feather->group2compile_tn, sizeof(feather->group2compile_tn));
Maat_table_get_group2group_table_name(feather->table_mgr, feather->group2group_tn, sizeof(feather->group2group_tn));
feather->logger=logger;
feather->scan_thread_num=max_thread_num;
@@ -759,7 +555,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
feather->rule_scan_type=2;
feather->thread_call_cnt=alignment_int64_array_alloc(max_thread_num);
feather->outer_mid_cnt=alignment_int64_array_alloc(max_thread_num);
feather->inner_mid_cnt=alignment_int64_array_alloc(max_thread_num);
feather-> compile_mid_cnt=alignment_int64_array_alloc(max_thread_num);
feather->hit_cnt=alignment_int64_array_alloc(max_thread_num);
feather->not_grp_hit_cnt=alignment_int64_array_alloc(max_thread_num);
feather->maat_version=0;
@@ -1223,24 +1019,20 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
return 1;
}
void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user)
void rule_ex_data_new_cb(void * user_data, void * param)
{
struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user;
struct Maat_compile_inner *relation=(struct Maat_compile_inner *)data;
struct Maat_compile_rule* compile_rule=relation->compile;
struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)param;
struct Maat_compile_rule* compile=(struct Maat_compile_rule*)user_data;
MAAT_RULE_EX_DATA ad=NULL;
if(compile_rule==NULL|| compile_rule->ref_table->table_id!=ex_desc->table_id)
if(compile->ref_table->table_id!=ex_desc->table_id)
{
return;
}
pthread_rwlock_wrlock(&(relation->rwlock));
ad=rule_ex_data_new(&(relation->compile->head),
relation->compile->service_defined,
ad=rule_ex_data_new(&(compile->head),
compile->service_defined,
ex_desc);
compile_rule->ads[ex_desc->idx]=ad;
pthread_rwlock_unlock(&(relation->rwlock));
compile->ads[ex_desc->idx]=ad;
return;
}
int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name,
@@ -1272,7 +1064,7 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table
if(_feather->scanner!=NULL)
{
MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_ex_desc);
Maat_hierarchy_compile_user_data_iterate(_feather->scanner->hier, rule_ex_data_new_cb, compile_ex_desc);
}
pthread_mutex_unlock(&(_feather->background_update_mutex));
@@ -1282,24 +1074,19 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table
MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx)
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct Maat_compile_inner *relation=NULL;
struct Maat_compile_rule* compile=NULL;
const struct compile_table_schema* compile_desc=NULL;
const struct compile_ex_data_idx* ex_desc=NULL;
MAAT_RULE_EX_DATA ad=NULL;
relation=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(relation==NULL||relation->compile==NULL)
compile=(struct Maat_compile_rule*)Maat_hierarchy_compile_read_user_data(_feather->scanner->hier, rule->config_id);
if(compile==NULL)
{
return NULL;
}
pthread_rwlock_rdlock(&(relation->rwlock));
compile=relation->compile;
compile_desc=&(compile->ref_table->compile);
assert(idx<compile_desc->ex_data_num);
ex_desc=compile_desc->ex_desc+idx;
ex_desc->dup_func(ex_desc->idx, &ad, compile->ads+idx, ex_desc->argl,ex_desc->argp);
pthread_rwlock_unlock(&(relation->rwlock));
return ad;
}
@@ -1498,7 +1285,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
}
if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS)
{
hit_region_cnt=match_district(_mid,region_result,hit_region_cnt, my_scanner);
hit_region_cnt=match_district(_mid, region_result, hit_region_cnt);
}
if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
{
@@ -1510,11 +1297,10 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, hit_region_cnt,
_mid->is_last_region, virtual_table_id, _mid->scan_cnt);
compile_ret=region_compile(_feather,_mid->inner,
compile_ret=region_compile(_feather, _mid->compile_mid,
&region_hit_wraper,
result, rule_num,
thread_num);
Maat_hierarchy_region_compile(struct Maat_hierarchy_compile_mid * mid, int * region_ids, int * virtual_tables, size_t n_region, int Nth_scan, void * * user_data_array, size_t ud_array_sz)
assert(_mid->is_last_region<2);
if(_mid->is_last_region==1)
{
@@ -1522,7 +1308,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
}
if(hit_region_cnt>0&&hit_detail!=NULL&&_feather->rule_scan_type!=0)
{
*detail_ret=fill_region_hit_detail(data, _mid->inner,
*detail_ret=fill_region_hit_detail(data, _mid->compile_mid,
region_result, hit_region_cnt,
result, compile_ret,
_mid->scan_cnt,
@@ -1621,7 +1407,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, region_ret,
_mid->is_last_region, virtual_table_id, _mid->scan_cnt);
compile_ret=region_compile(_feather,_mid->inner,
compile_ret=region_compile(_feather,_mid->compile_mid,
&region_hit_wraper,
result, rule_num,
thread_num);
@@ -1706,7 +1492,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_GIE(&region_hit_wraper, region_result, hit_region_cnt,
_mid->is_last_region, virtual_table_id, _mid->scan_cnt);
compile_ret=region_compile(_feather,_mid->inner,
compile_ret=region_compile(_feather,_mid->compile_mid,
&region_hit_wraper,
result, rule_num,
thread_num);
@@ -1940,7 +1726,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
{
region_hit_wraper.virtual_table_ids=region_rslt_virtual_table_id;
}
compile_ret=region_compile(_feather,_mid->inner,
compile_ret=region_compile(_feather,_mid->compile_mid,
&region_hit_wraper,
result, rule_num,
thread_num);
@@ -2150,7 +1936,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
}
if(hit_region_cnt>0&&sp->p_real_table->table_type==TABLE_TYPE_EXPR_PLUS)
{
hit_region_cnt=match_district(_mid, region_result, hit_region_cnt, scanner);
hit_region_cnt=match_district(_mid, region_result, hit_region_cnt);
}
if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
{
@@ -2163,7 +1949,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, hit_region_cnt,
_mid->is_last_region, sp->virtual_table_id, _mid->scan_cnt);
compile_ret=region_compile(sp->feather,_mid->inner,
compile_ret=region_compile(sp->feather, _mid->compile_mid,
&region_hit_wraper,
result, rule_num,
sp->thread_num);
@@ -2176,7 +1962,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
{
if(sp->scan_buff!=NULL)
{
*detail_ret=fill_region_hit_detail(sp->scan_buff, _mid->inner,
*detail_ret=fill_region_hit_detail(sp->scan_buff, _mid->compile_mid,
region_result, hit_region_cnt,
result, compile_ret,
_mid->scan_cnt,
@@ -2184,7 +1970,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
}
else
{
*detail_ret=fill_region_hit_detail(data, _mid->inner,
*detail_ret=fill_region_hit_detail(data, _mid->compile_mid,
region_result, hit_region_cnt,
result, compile_ret,
_mid->scan_cnt,
@@ -2408,7 +2194,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
struct scan_region_hit_wraper region_hit_wraper;
scan_region_hit_wraper_build_with_GIE(&region_hit_wraper, region_result, hit_region_cnt,
_mid->is_last_region, sp->virtual_table_id, _mid->scan_cnt);
compile_ret=region_compile(sp->feather,_mid->inner,
compile_ret=region_compile(sp->feather, _mid->compile_mid,
&region_hit_wraper,
result, rule_num,
sp->thread_num);
@@ -2460,21 +2246,19 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum
{
int ret=0;
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct Maat_compile_inner *compile_inner=NULL;
struct Maat_compile_rule* compile=NULL;
switch(type)
{
case MAAT_RULE_SERV_DEFINE:
compile_inner=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(compile_inner==NULL)
compile=(struct Maat_compile_rule*)Maat_hierarchy_compile_read_user_data(_feather->scanner->hier, rule->config_id);
if(compile)
{
ret=0;
ret=MIN(size, compile->head.serv_def_len);
memcpy(value, compile->service_defined, ret);
}
else
{
pthread_rwlock_rdlock(&(compile_inner->rwlock));
ret=MIN(size,compile_inner->compile->head.serv_def_len);
memcpy(value,compile_inner->compile->service_defined,ret);
pthread_rwlock_unlock(&(compile_inner->rwlock));
ret=0;
}
break;
default:
@@ -2530,12 +2314,12 @@ int Maat_get_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_S
struct _OUTER_scan_status_t* _mid=NULL;
struct Maat_hit_path_t* paths;
struct Maat_hit_path_t condition;
struct Maat_hit_path_inner *path_inner;
struct Maat_hierarchy_hit_path **path_inner=NULL;
size_t n_read=0, i=0;
int ret=0;
_mid=grab_mid(mid, _feather, 0, 0);
if(_mid->inner==NULL)
if(_mid->compile_mid==NULL)
{
return 0;
}
@@ -2548,12 +2332,12 @@ int Maat_get_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_S
}
paths=(struct Maat_hit_path_t*)value;
n_read=size/sizeof(struct Maat_hit_path_t);
path_inner=ALLOC(struct Maat_hit_path_inner, n_read);
hit_path_init(&condition);
n_read=scan_hit_status_select_hit_path_inner(&_mid->inner->hit_path_qhead, &condition, path_inner, n_read);
path_inner=ALLOC(struct Maat_hierarchy_hit_path*, n_read);
Maat_hit_path_init(&condition);
n_read=Maat_hierarchy_hit_path_select0(_mid->compile_mid, &condition, path_inner, n_read);
for(i=0; i<n_read; i++)
{
paths[i]=path_inner[i].path;
paths[i]=path_inner[i]->path;
}
free(path_inner);
ret=n_read;
@@ -2618,11 +2402,11 @@ void Maat_clean_status(scan_status_t* mid)
}
_mid=(struct _OUTER_scan_status_t*)(*mid);
alignment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1);
if(_mid->inner!=NULL)
if(_mid->compile_mid!=NULL)
{
scan_hit_status_free(_mid->inner);
_mid->inner=NULL;
alignment_int64_array_add(_mid->feather->inner_mid_cnt, _mid->thread_num, -1);
Maat_hierarchy_compile_mid_free(_mid->compile_mid);
_mid->compile_mid=NULL;
alignment_int64_array_add(_mid->feather->compile_mid_cnt, _mid->thread_num, -1);
}
_mid->feather=NULL;
free(_mid);

575
src/entry/Maat_command.cpp
View File

@@ -244,20 +244,22 @@ int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq)
line[i]='0';
return 0;
}
void serialize_group(const struct Maat_group_t* p_group, enum MAAT_OPERATION op, char* buff, size_t sz)
void serialize_group2group(enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g, char* buff, size_t sz)
{
if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD;
const char* vt_name="null";
if(p_group->virtual_table_name!=NULL)
{
vt_name=p_group->virtual_table_name;
snprintf(buff, sz, "%d\t%d\t%d", g2g->group_id,
g2g->superior_group_id,
op);
return;
}
snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d\t%s", p_group->group_id,
p_group->parent_id,
void serialize_group2compile(enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c, char* buff, size_t sz)
{
snprintf(buff, sz, "%d\t%d\t%d\t%d\t%s\t%d", g2c->group_id,
g2c->compile_id,
op,
p_group->not_flag,
p_group->parent_type,
vt_name);
g2c->not_flag,
g2c->virtual_table_name?g2c->virtual_table_name:"null",
g2c->Nth_clause==0?1:g2c->Nth_clause);
return;
}
void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int group_num, enum MAAT_OPERATION op, char* buff, size_t sz)
@@ -276,7 +278,7 @@ void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_serv
group_num);
return;
}
void serialize_region(const struct Maat_region_t* p, int group_id, char* buff, size_t sz)
void serialize_region(const struct Maat_cmd_region* p, int group_id, char* buff, size_t sz)
{
UNUSED size_t ret=0;
switch(p->region_type)
@@ -842,174 +844,7 @@ int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rul
}
return 0;
}
int calculate_serial_rule_num(struct _Maat_cmd_inner_t* _cmd,int * new_region_cnt, int* new_group_cnt)
{
int serial_num=0;
int i=0;
struct Maat_cmd_t* cmd=&(_cmd->cmd);
serial_num++;//compile rule
for(i=0;i<cmd->group_num;i++)
{
serial_num++;
if(cmd->groups[i].regions==NULL)
{
continue;
}
if(_cmd->op==MAAT_OP_ADD)
{
*new_region_cnt+=cmd->groups[i].region_num;
(*new_group_cnt)++;
}
//for MAAT_OP_DEL, if the group's refcnt>1, group->region_num=0.
//so it's OK to add.
serial_num+=cmd->groups[i].region_num;
}
return serial_num;
}
int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _cmd)
{
int i=0,j=0,grp_idx=0;
struct Maat_cmd_t* cmd=&(_cmd->cmd);
struct Maat_group_t* group_cmd=NULL;
struct Maat_region_t* region_cmd=NULL;
struct Maat_compile_inner *relation=NULL;
struct Maat_group_inner* group_inner=NULL;
struct Maat_region_inner* region_inner=NULL;
void* logger=feather->logger;
int config_id=cmd->compile.config_id;
if(feather->scanner==NULL)
{
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_command
,"MAAT not ready.");
return -1;
}
relation=(struct Maat_compile_inner *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id);
//Operation on relation is thread safe, no immediate memory free when delete a compile rule or a scanner.
//In another words, if the relation is accessable from compile means, its was valid in at least 10 seconds (garbage bury).
if(relation==NULL)
{
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_command
,"config %d not exist."
,config_id);
return -1;
}
pthread_rwlock_rdlock(&(relation->rwlock));
cmd->group_num=relation->group_cnt;
assert(cmd->groups==NULL);
cmd->groups=ALLOC(struct Maat_group_t, cmd->group_num);
for(i=0;i<relation->group_boundary;i++)
{
group_inner=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i);
if(group_inner==NULL)
{
continue;
}
group_cmd=&(cmd->groups[grp_idx]);
group_cmd->group_id=group_inner->group_id;
if(group_inner->ref_by_parent_cnt>1)
{
continue;
}
group_cmd->region_num=group_inner->region_cnt;
group_cmd->regions=ALLOC(struct Maat_region_t, group_cmd->region_num);
for(j=0;j<group_inner->region_boundary;j++)
{
region_inner=(struct Maat_region_inner*)dynamic_array_read(group_inner->regions,j);
if(region_inner==NULL)
{
continue;
}
region_cmd=&(group_cmd->regions[j]);
region_cmd->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, region_inner->table_id));
region_cmd->region_id=region_inner->region_id;
//NOTICE: region_type only avilable when OP_ADD,
region_cmd->region_type=REGION_EXPR;
}
grp_idx++;
}
pthread_rwlock_unlock(&(relation->rwlock));
return 0;
}
int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd, struct serial_rule_t* list, int size)
{
struct Maat_group_t* p_group=NULL;
struct Maat_region_t* p_region=NULL;
struct Maat_rule_t* p_m_rule=NULL;
struct Maat_cmd_t* cmd=&(_cmd->cmd);
enum MAAT_OPERATION op=_cmd->op;
int rule_num=0,i=0,j=0;
p_m_rule=&(cmd->compile);
char line[MAX_TABLE_LINE_SIZE];
time_t timeout=0;
if(_cmd->cmd.expire_after>0)
{
timeout=feather->server_time+_cmd->cmd.expire_after;
}
if(op==MAAT_OP_ADD)
{
serialize_compile(p_m_rule, _cmd->huge_service_defined, cmd->group_num, MAAT_OP_ADD, line, sizeof(line));
set_serial_rule(list+rule_num, MAAT_OP_ADD, cmd->compile.config_id, cmd->label_id, feather->compile_tn, line, timeout);
}
else
{
set_serial_rule(list+rule_num, op, cmd->compile.config_id, cmd->label_id, feather->compile_tn, NULL, timeout);
}
rule_num++;
for(i=0;i<cmd->group_num;i++)
{
p_group=&(cmd->groups[i]);
if(op==MAAT_OP_ADD)
{
if(feather->AUTO_NUMBERING_ON==1)
{
p_group->group_id=feather->base_grp_seq;
feather->base_grp_seq++;
}
p_group->parent_id=p_m_rule->config_id;
p_group->parent_type=PARENT_TYPE_COMPILE;
serialize_group(p_group, MAAT_OP_ADD, line, sizeof(line));
set_serial_rule(list+rule_num, MAAT_OP_ADD, p_group->group_id, 0, feather->group_tn, line, timeout);
}
else
{
set_serial_rule(list+rule_num,op,p_group->group_id,0,feather->group_tn,NULL,timeout);
}
rule_num++;
if(p_group->regions==NULL)//group reuse.
{
continue;
}
for(j=0;j<p_group->region_num;j++)
{
p_region=&(p_group->regions[j]);
if(op==MAAT_OP_ADD)
{
if(feather->AUTO_NUMBERING_ON==1)
{
p_region->region_id=feather->base_rgn_seq;
feather->base_rgn_seq++;
}
serialize_region(p_region, p_group->group_id, line, sizeof(line));
set_serial_rule(list+rule_num,MAAT_OP_ADD
,p_region->region_id,0,p_region->table_name,line,timeout);
}
else
{
set_serial_rule(list+rule_num,op
,p_region->region_id,0,p_region->table_name,NULL,timeout);
}
rule_num++;
}
}
assert(rule_num<=size);
return rule_num;
}
int mr_transaction_success(redisReply* data_reply)
{
if(data_reply->type==REDIS_REPLY_NIL)
@@ -1380,58 +1215,7 @@ error_out:
return success_cnt;
}
int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
{
int i=0, j=0;
const char *table_name=NULL;
int table_id=0;
struct Maat_group_t* p_group=NULL;
struct Maat_region_t* p_region=NULL;
enum MAAT_TABLE_TYPE table_type;
struct Maat_compile_inner *compile_rule=NULL;
if(feather->scanner!=NULL)
{
compile_rule=(struct Maat_compile_inner*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id);
if(compile_rule!=NULL)
{
MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module
,"Maat rule %d already exisits.",cmd->compile.config_id);
return -1;
}
}
for(i=0;i<cmd->group_num;i++)
{
p_group=&(cmd->groups[i]);
for(j=0;j<p_group->region_num;j++)
{
p_region=&(p_group->regions[j]);
table_name=p_region->table_name;
table_id=Maat_table_get_id_by_name(feather->table_mgr, table_name);
if(table_id<0)
{
MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module
,"Unknown table %s of Maat_cmd_t[%d]->group[%d]->region[%d]."
,table_name,cmd->compile.config_id,i,j);
return -1;
}
table_type=type_region2table(p_region);
if(table_type!=Maat_table_get_type_by_id(feather->table_mgr, table_id))
{
MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module
,"Table %s not support region type %d of Maat_cmd_t[%d]->group[%d]->region[%d]."
,table_name
,p_region->region_type
,cmd->compile.config_id,i,j);
return -1;
}
free((char*)p_region->table_name);
p_region->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, table_id));
}
}
return 0;
}
void check_maat_expiration(redisContext *ctx, void *logger)
{
unsigned int i=0,s_rule_num=0;
@@ -2023,70 +1807,8 @@ void _maat_empty_region(struct Maat_region_t* p)
memset(p,0,sizeof(struct Maat_region_t));
return;
}
struct Maat_cmd_t* Maat_create_cmd(const struct Maat_rule_t* rule, int group_num)
{
int i=0;
struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t*)calloc(sizeof(struct _Maat_cmd_inner_t),1);
memcpy(&(_cmd->cmd.compile),rule,sizeof(_cmd->cmd.compile));
_cmd->ref_cnt=1;
_cmd->cmd.group_num=group_num;
if(group_num>0)
{
_cmd->cmd.groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),group_num);
}
else
{
_cmd->cmd.groups=NULL;
}
for(i=0;i<group_num;i++)
{
_cmd->cmd.groups[i].regions=(struct Maat_region_t*)calloc(sizeof(struct Maat_region_t),1);
_cmd->region_size[i]=1;
}
return (struct Maat_cmd_t*)_cmd;
}
int Maat_cmd_set_opt(struct Maat_cmd_t* cmd, enum MAAT_RULE_OPT type, const char* val, int size)
{
struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t* )cmd;
int ret=-1;
switch(type)
{
case MAAT_RULE_SERV_DEFINE:
if(size>MAX_HUGE_SERVICE_DEFINE_LEN)
{
ret=-1;
}
else
{
if(_cmd->huge_service_defined!=NULL)
{
free(_cmd->huge_service_defined);
}
_cmd->huge_service_defined=(char*)calloc(sizeof(char),size+1);
memcpy(_cmd->huge_service_defined,val, size);
ret=0;
}
break;
default:
break;
}
return ret;
}
int Maat_cmd_set_group(Maat_feather_t feather,int group_id, const struct Maat_region_t* region, enum MAAT_OPERATION op)
{
_Maat_feather_t* _feather=(_Maat_feather_t*)feather;
if(_feather->AUTO_NUMBERING_ON==1)
{
return -1;
}
//struct Maat_group_inner* group_inner=NULL;
//group_inner=(struct Maat_group_inner*)HASH_fetch_by_id(_feather->scanner->group_hash, group_id);
//NOT implemented yet.
assert(0);
return 0;
}
int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_rule, int line_num ,enum MAAT_OPERATION op)
{
int i=0, j=0;
@@ -2200,7 +1922,7 @@ error_out:
free(s_rule);
return ret;
}
int Maat_cmd_set_line(Maat_feather_t feather,const struct Maat_cmd_line* line_rule, enum MAAT_OPERATION op)
{
int ret=0;
@@ -2254,203 +1976,7 @@ int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value,
}
freeReplyObject(reply);
return 1;
}
void Maat_add_region2cmd(struct Maat_cmd_t* cmd,int which_group,const struct Maat_region_t* region)
{
struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t*)cmd;
struct Maat_region_t* dst=NULL;
struct Maat_group_t* p_group=NULL;
p_group=&(cmd->groups[which_group]);
assert(which_group<cmd->group_num);
assert(region->table_name!=NULL);
if(p_group->region_num==_cmd->region_size[which_group])
{
_cmd->region_size[which_group]*=2;
p_group->regions=(struct Maat_region_t*)realloc(p_group->regions,sizeof(struct Maat_region_t)*_cmd->region_size[which_group]);
}
dst=&(p_group->regions[p_group->region_num]);
p_group->region_num++;
_maat_copy_region(dst, region);
return;
}
void Maat_free_cmd(struct Maat_cmd_t* cmd)
{
struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t*)cmd;
int i=0,j=0;
_cmd->ref_cnt--;
if(_cmd->ref_cnt>0)
{
return;
}
for(i=0;i<cmd->group_num;i++)
{
for(j=0;j<cmd->groups[i].region_num;j++)
{
_maat_empty_region(&(cmd->groups[i].regions[j]));
}
free(cmd->groups[i].regions);
cmd->groups[i].regions=NULL;
}
free(cmd->groups);
cmd->groups=NULL;
if(_cmd->huge_service_defined!=NULL)
{
free(_cmd->huge_service_defined);
_cmd->huge_service_defined=NULL;
}
_cmd->next=NULL;
free(_cmd);
return;
}
int Maat_format_cmd(struct Maat_cmd_t* rule, char* buffer, int size)
{
//TODO
return 0;
}
int Maat_cmd(Maat_feather_t feather,struct Maat_cmd_t* raw_rule,enum MAAT_OPERATION op)
{
int ret=0;
ret=Maat_cmd_append(feather,raw_rule,op);
if(ret<0)
{
return ret;
}
ret=Maat_cmd_commit(feather);
return ret;
}
int Maat_cmd_append(Maat_feather_t feather,struct Maat_cmd_t* cmd,enum MAAT_OPERATION op)
{
_Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_cmd_inner_t* _cmd=(struct _Maat_cmd_inner_t*)cmd;
int ret=0;
_cmd->op=op;
assert(op==MAAT_OP_DEL||op==MAAT_OP_ADD||op==MAAT_OP_RENEW_TIMEOUT);
assert(_cmd->next==NULL);
if(op==MAAT_OP_RENEW_TIMEOUT)
{
assert(cmd->expire_after>0);
}
switch(op)
{
case MAAT_OP_DEL:
case MAAT_OP_RENEW_TIMEOUT:
ret=reconstruct_cmd(_feather, _cmd);
break;
case MAAT_OP_ADD:
ret=fix_table_name(_feather, cmd);
break;
}
if(ret<0)
{
return -1;
}
_cmd->ref_cnt++;
if(_feather->cmd_q_cnt==0)
{
assert(_feather->cmd_qtail==NULL);
assert(_feather->cmd_qtail==_feather->cmd_qhead);
_feather->cmd_qhead=_feather->cmd_qtail=_cmd;
}
else
{
_feather->cmd_qtail->next=_cmd;
_feather->cmd_qtail=_cmd;
}
_feather->cmd_q_cnt++;
return 0;
}
int Maat_cmd_commit(Maat_feather_t feather)
{
_Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int ret=-1, i=0;
int new_region_num=0,new_group_num=0;
int serial_rule_num=0,serial_rule_idx=0;
UNUSED int transaction_success=1;
struct _Maat_cmd_inner_t* p=NULL,*n=NULL;
redisContext* write_ctx=NULL;
redisReply* data_reply=NULL;
struct serial_rule_t* s_rule=NULL;
if(_feather->input_mode!=SOURCE_REDIS)
{
return ret;
}
if(_feather->cmd_q_cnt==0)
{
return 0;
}
write_ctx=get_redis_ctx_for_write(_feather);
if(write_ctx==NULL)
{
goto error_out;
}
for(i=0,p=_feather->cmd_qhead;i<_feather->cmd_q_cnt;i++)
{
serial_rule_num+=calculate_serial_rule_num(p, &new_region_num, &new_group_num);
p=p->next;
}
_feather->server_time=redis_server_time(write_ctx);
if(!_feather->server_time)
{
goto error_out;
}
if(_feather->AUTO_NUMBERING_ON==1)
{
data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_region_id_var, new_region_num);
if(data_reply->type!=REDIS_REPLY_INTEGER)
{
freeReplyObject(data_reply);
goto error_out;
}
_feather->base_rgn_seq=data_reply->integer-new_region_num;
freeReplyObject(data_reply);
data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_group_id_var, new_group_num);
if(data_reply->type!=REDIS_REPLY_INTEGER)
{
freeReplyObject(data_reply);
goto error_out;
}
_feather->base_grp_seq=data_reply->integer-new_group_num;
freeReplyObject(data_reply);
}
s_rule=(struct serial_rule_t*)calloc(sizeof(struct serial_rule_t),serial_rule_num);
for(i=0,p=_feather->cmd_qhead;i<_feather->cmd_q_cnt;i++)
{
serial_rule_idx+=build_serial_rule(_feather,p,s_rule+serial_rule_idx, serial_rule_num-serial_rule_idx);
p=p->next;
}
assert(serial_rule_idx==serial_rule_num);
transaction_success=0;
transaction_success=exec_serial_rule(write_ctx, s_rule,serial_rule_num,_feather->server_time,_feather->logger);
assert(transaction_success==serial_rule_num);
ret=_feather->cmd_q_cnt;
_feather->cmd_acc_num+=_feather->cmd_q_cnt;
error_out:
p=_feather->cmd_qhead;
for(i=0;i<_feather->cmd_q_cnt;i++)
{
n=p->next;
Maat_free_cmd((struct Maat_cmd_t* )p);
p=n;
}
_feather->cmd_qhead=_feather->cmd_qtail=NULL;
_feather->cmd_q_cnt=0;
for(i=0;i<serial_rule_num && s_rule;i++)
{
empty_serial_rules(s_rule+i);
}
free(s_rule);
return ret;
long long Maat_cmd_incrby(Maat_feather_t feather,const char* key, int increment)
{
@@ -2613,45 +2139,62 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger)
);
}
return redis_transaction_success;
}
}
int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int clause_num, int label_id, int expire_after)
{
redisContext* write_ctx=get_redis_ctx_for_write(_feather);
_feather->server_time=redis_server_time(write_ctx);
if(!_feather->server_time)
{
return -1;
}
struct serial_rule_t s_rule;
set_serial_rule(&s_rule, op, id, 0, table_name, line, 0);
int transaction_success=0;
transaction_success=exec_serial_rule(write_ctx, &s_rule, 1, _feather->server_time, _feather->logger);
empty_serial_rules(&s_rule);
return transaction_success;
}
int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num)
{
{
struct Maat_cmd_line line_cmd;
char line[MAX_TABLE_LINE_SIZE];
serialize_compile(compile, huge_service_defined, group_num, op, line, sizeof(line));
char line[MAX_TABLE_LINE_SIZE];
serialize_compile(compile, huge_service_defined, clause_num, op, line, sizeof(line));
memset(&line_cmd, 0, sizeof(line_cmd));
line_cmd.table_name=table_name;
line_cmd.rule_id=compile->config_id;
line_cmd.table_line=line;
line_cmd.label_id=label_id;
line_cmd.expire_after=expire_after;
int ret=Maat_cmd_set_line(feather, &line_cmd, op);
return ret;
}
}
int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_region* region, int group_id)
{
{
struct Maat_cmd_line line_cmd;
char line[MAX_TABLE_LINE_SIZE];
serialize_region(region, group_id, line, sizeof(line));
serialize_region(region, group_id, line, sizeof(line));
memset(&line_cmd, 0, sizeof(line_cmd));
line_cmd.table_name=region->table_name;
line_cmd.rule_id=region->region_id;
line_cmd.table_line=line;
int ret=Maat_cmd_set_line(feather, &line_cmd, op);
return ret;
}
}
int Maat_command_raw_set_group2compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c)
{
{
struct Maat_cmd_line line_cmd;
char line[MAX_TABLE_LINE_SIZE];
serialize_group(group, op, line, sizeof(line));
char line[MAX_TABLE_LINE_SIZE];
serialize_group2compile(op, g2c, line, sizeof(line));
assert(g2c->group_id<1024*1024);
memset(&line_cmd, 0, sizeof(line_cmd));
line_cmd.table_name=g2c->table_name;
line_cmd.rule_id=g2c->compile_id*1024*1024+g2c->group_id;
line_cmd.table_line=line;
int ret=Maat_cmd_set_line(feather, &line_cmd, op);
return ret;
}
int Maat_command_raw_set_group2group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g)
{
struct Maat_cmd_line line_cmd;
char line[MAX_TABLE_LINE_SIZE];
assert(g2g->group_id<1024*1024);
serialize_group2group(op, g2g, line, sizeof(line));
memset(&line_cmd, 0, sizeof(line_cmd));
line_cmd.table_name=g2g->table_name;
line_cmd.rule_id=g2g->superior_group_id*1024*1024+g2g->group_id;
line_cmd.table_line=line;
int ret=Maat_cmd_set_line(feather, &line_cmd, op);
return ret;
}
int Maat_cmd_flushDB(Maat_feather_t feather)
{

16
src/entry/Maat_garbage_collection.cpp
View File

@@ -1,4 +1,10 @@
#include "Maat_utils.h"
#include <time.h>
#include <stdlib.h>
#include <sys/queue.h>
#include <assert.h>
struct Maat_garbage_bag
{
@@ -9,7 +15,7 @@ struct Maat_garbage_bag
void (* garbage_free)(void *garbage);
TAILQ_ENTRY(Maat_garbage_bag) entries;
};
TAILQ_HEAD(Maat_garbage_bag, Maat_garbage_q);
TAILQ_HEAD(Maat_garbage_q, Maat_garbage_bag);
struct Maat_garbage_bin
{
@@ -20,7 +26,7 @@ struct Maat_garbage_bin
struct Maat_garbage_bin* Maat_garbage_bin_new(int default_timeout)
{
struct Maat_garbage_bin* bin=ALLOC(struct Maat_garbage_bin, 1);
TAILQ_INIT(bin->garbage_q);
TAILQ_INIT(&bin->garbage_q);
bin->timeout_seconds=default_timeout;
return bin;
}
@@ -37,7 +43,7 @@ void Maat_garbage_bin_free(struct Maat_garbage_bin* bin)
free(bin);
return;
}
void Maat_garbage_bag(struct Maat_garbage_bin* bin, void* garbage, void (* func)(void *))
void Maat_garbage_bagging(struct Maat_garbage_bin* bin, void* garbage, void (* func)(void *))
{
struct Maat_garbage_bag* bag=ALLOC( struct Maat_garbage_bag, 1);
bag->create_time=time(NULL);
@@ -47,10 +53,10 @@ void Maat_garbage_bag(struct Maat_garbage_bin* bin, void* garbage, void (* func)
TAILQ_INSERT_TAIL(&bin->garbage_q, bag, entries);
bin->bag_cnt++;
}
void Maat_garbage_collect(struct Maat_garbage_bin* bin)
void Maat_garbage_collect_routine(struct Maat_garbage_bin* bin)
{
struct Maat_garbage_bag* p=NULL, *tmp=NULL;
int n_clollected=0, n_bag=0;
size_t n_clollected=0, n_bag=0;
time_t now=time(NULL);
for(p=TAILQ_FIRST(&bin->garbage_q); p!=NULL; p=tmp)

586
src/entry/Maat_hierarchy.cpp
View File

@@ -1,13 +1,24 @@
#include "uthash.h"
#include "Maat_hierarchy.h"
#include "Maat_utils.h"
#include "uthash/uthash.h"
#include "uthash/utarray.h"
#include "igraph/igraph.h"
#include "bool_matcher.h"
#include <MESA/MESA_handle_logger.h>
#include <pthread.h>
#include <assert.h>
#define module_maat_hierarchy "MAAT_HIERARCHY"
const char* sql_create_hier_table[]={"create table MAAT_HIERARCHY_REGION(region_id interger PRIMARY KEY, group_id interger, district_id interger, table_id interger, expr_id_lb interger, expr_id_ub interger)",
"create table MAAT_HIERARCHY_GROUP(group_id interger PRIMARY KEY, vertex_id interger)",
"create table MAAT_HIERARCHY_COMPILE(compile_id interger PRIMARY KEY, declared_clause_num interger, user_data_pointer interger)",
"create table MAAT_HIERARCHY_CLAUSE(compile_id interger, nth_clause interger, not_flag interger, UNIQUE(compile_id, nth_clause))",
"create table MAAT_HIERARCHY_LITERAL(group_id interger, virtual_table_id interger, clause_id interger)",
NULL};
#define TO_CLAUSE_ID(Nth_clause, compile_id) ((long long)Nth_clause<<32|compile_id)
#define TO_CLAUSE_ID_COMPATBILE(vid, gid) ((unsigned long long)vid<<32|gid)
#define TO_LITERAL_ID(vt_id, group_id) ((long long)vt_id<<32|group_id)
struct Maat_hierarchy_group
@@ -38,7 +49,7 @@ struct Maat_hierarchy_region
struct Maat_hierarchy_literal_id
{
int group_id;
int vt_id;
int virtual_table_id;
};
struct Maat_hierarchy_clause_id
{
@@ -80,6 +91,11 @@ struct Maat_hierarchy_compile
UT_array *literal_ids;
};
static void _group_vertex_free(struct Maat_hierarchy_group* group)
{
free(group->top_group_ids);
free(group);
}
@@ -114,6 +130,115 @@ struct Maat_hierarchy
void* logger;
};
static struct Maat_hierarchy_literal* Maat_hierarchy_literal_new(struct Maat_hierarchy* hier, int group_id, int vt_id)
{
struct Maat_hierarchy_literal* literal=ALLOC(struct Maat_hierarchy_literal, 1);
literal->literal_id.group_id=group_id;
literal->literal_id.virtual_table_id=vt_id;
literal->hash_clause_by_id=NULL;
HASH_ADD(hh, hier->hash_literal_by_id, literal_id, sizeof(literal->literal_id), literal);
return literal;
}
static void Maat_hierarchy_literal_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_literal* literal)
{
struct Maat_hierarchy_clause* clause=NULL, *tmp=NULL;
HASH_ITER(hh, literal->hash_clause_by_id, clause, tmp)
{
HASH_DELETE(hh, literal->hash_clause_by_id, clause);
free(clause);
}
HASH_DELETE(hh, hier->hash_literal_by_id, literal);
free(literal);
}
static int Maat_hierarchy_literal_join_clause(struct Maat_hierarchy_literal* literal, int not_flag, int Nth_clause, int compile_id)
{
struct Maat_hierarchy_clause* clause=NULL;
struct Maat_hierarchy_clause_id clause_id={Nth_clause, compile_id};
HASH_FIND(hh, literal->hash_clause_by_id, &clause_id, sizeof(clause_id), clause);
if(clause)
{
return -1;
}
clause=ALLOC(struct Maat_hierarchy_clause, 1);
clause->clause_id=clause_id;
clause->not_flag=not_flag;
HASH_ADD(hh, literal->hash_clause_by_id, clause_id, sizeof(clause->clause_id), clause);
return 0;
}
static int Maat_hierarchy_literal_leave_clause(struct Maat_hierarchy_literal* literal, int Nth_clause, int compile_id)
{
struct Maat_hierarchy_clause* clause=NULL;
struct Maat_hierarchy_clause_id clause_id={Nth_clause, compile_id};
HASH_FIND(hh, literal->hash_clause_by_id, &clause_id, sizeof(clause_id), clause);
if(!clause)
{
return -1;
}
HASH_DELETE(hh, literal->hash_clause_by_id, clause);
free(clause);
return 0;
}
int compare_literal_id(const void *pa, const void *pb)
{
struct Maat_hierarchy_literal_id *la=(struct Maat_hierarchy_literal_id *)pa;
struct Maat_hierarchy_literal_id *lb=(struct Maat_hierarchy_literal_id *)pb;
return TO_LITERAL_ID(la->virtual_table_id, la->group_id)-TO_LITERAL_ID(lb->virtual_table_id, lb->group_id);
}
static struct Maat_hierarchy_compile* Maat_hierarchy_compile_new(struct Maat_hierarchy* hier, int compile_id)
{
struct Maat_hierarchy_compile* compile=NULL;
compile=ALLOC(struct Maat_hierarchy_compile, 1);
compile->compile_id=compile_id;
HASH_ADD_INT(hier->hash_compile_by_id, compile_id, compile);
utarray_new(compile->literal_ids, &ut_literal_id_icd);
return compile;
}
static void Maat_hierarchy_compile_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_compile* compile)
{
HASH_DEL(hier->hash_compile_by_id, compile);
if(hier->compile_user_data_free)
{
hier->compile_user_data_free(compile->user_data);
}
utarray_free(compile->literal_ids);
free(compile);
}
static struct Maat_hierarchy_region* Maat_hierarchy_region_new(struct Maat_hierarchy* hier, int region_id, int group_id, int table_id, struct Maat_hierarchy_group* parent_group, void* user_data)
{
struct Maat_hierarchy_region* region=NULL;
region=ALLOC(struct Maat_hierarchy_region, 1);
region->group_id=group_id;
region->region_id=region_id;
region->table_id=table_id;
region->ref_parent_group=parent_group;
region->user_data=user_data;
HASH_ADD_INT(hier->hash_region_by_id, region_id, region);
parent_group->ref_by_region_cnt++;
return region;
}
static void Maat_hierarchy_region_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_region* region)
{
HASH_DELETE(hh, hier->hash_region_by_id, region);
region->ref_parent_group->ref_by_region_cnt--;
if(region->user_data)
{
hier->region_user_data_free(region->user_data);
region->user_data=NULL;
}
free(region);
return;
}
struct Maat_hierarchy* Maat_hierarchy_new(int thread_num, void* mesa_handle_logger)
{
struct Maat_hierarchy* hier=ALLOC(struct Maat_hierarchy, 1);
@@ -134,11 +259,10 @@ struct Maat_hierarchy* Maat_hierarchy_new(int thread_num, void* mesa_handle_logg
}
void Maat_hierarchy_free(struct Maat_hierarchy* hier)
{
int ret=0;
struct Maat_hierarchy_compile* compile=NULL, tmp_compile=NULL;
struct Maat_hierarchy_group* group=NULL, tmp_group=NULL;
struct Maat_hierarchy_literal* literal=NULL, tmp_literal=NULL;
struct Maat_hierarchy_region* region=NULL, tmp_region=NULL;
struct Maat_hierarchy_compile* compile=NULL, *tmp_compile=NULL;
struct Maat_hierarchy_group* group=NULL, *tmp_group=NULL;
struct Maat_hierarchy_literal* literal=NULL, *tmp_literal=NULL;
struct Maat_hierarchy_region* region=NULL, *tmp_region=NULL;
pthread_rwlock_wrlock(&hier->rwlock);
//Reference: https://troydhanson.github.io/uthash/userguide.html#_what_can_it_do
@@ -190,31 +314,6 @@ void Maat_hierarchy_set_region_user_data_free_func(struct Maat_hierarchy* hier,
}
#define TO_CLAUSE_ID(Nth_clause, compile_id) ((long long)Nth_clause<<32|compile_id)
#define TO_CLAUSE_ID_COMPATBILE(vid, gid) ((unsigned long long)vid<<32|gid)
#define TO_LITERAL_ID(vt_id, group_id) ((long long)vt_id<<32|group_id)
static struct Maat_hierarchy_compile* Maat_hierarchy_compile_new(struct Maat_hierarchy* hier, int policy_id)
{
struct Maat_hierarchy_compile* compile=NULL;
compile=ALLOC(struct Maat_hierarchy_compile, 1);
compile->compile_id=policy_id;
HASH_ADD_INT(hier->hash_compile_by_id, policy_id, compile);
utarray_new(compile->literal_ids, &ut_literal_id_icd);
return compile;
}
static void Maat_hierarchy_compile_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_compile* compile)
{
HASH_DEL(hier->hash_compile_by_id, compile);
if(hier->compile_user_data_free)
{
hier->compile_user_data_free(compile->user_data);
}
utarray_free(compile->literal_ids);
free(compile);
}
int Maat_hierarchy_compile_add(struct Maat_hierarchy* hier, int compile_id, int declared_clause_num, void* user_data)
{
int ret=0;
@@ -231,7 +330,7 @@ int Maat_hierarchy_compile_add(struct Maat_hierarchy* hier, int compile_id, int
}
else
{
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module,
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
"Add compile %d failed, compile is already exisited.",
compile_id);
ret=-1;
@@ -254,7 +353,7 @@ int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id)
}
else
{
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module,
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
"Remove compile %d failed, compile is not exisited.",
compile_id);
ret=-1;
@@ -263,8 +362,7 @@ int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id)
pthread_rwlock_unlock(&hier->rwlock);
return ret;
}
void* Maat_hierarchy_compile_dettach_user_data(struct Maat_hierarchy* hier, int compile_id)
static void* Maat_hier_get_user_data(struct Maat_hierarchy* hier, int compile_id, int is_dettach)
{
struct Maat_hierarchy_compile* compile=NULL;
void* ret=NULL;
@@ -274,12 +372,39 @@ void* Maat_hierarchy_compile_dettach_user_data(struct Maat_hierarchy* hier, int
if(compile)
{
ret=compile->user_data;
if(is_dettach)
{
compile->user_data=NULL;
}
}
pthread_rwlock_unlock(&hier->rwlock);
return ret;
}
}
void* Maat_hierarchy_compile_dettach_user_data(struct Maat_hierarchy* hier, int compile_id)
{
void* user_data=NULL;
user_data=Maat_hier_get_user_data(hier, compile_id, 1);
return user_data;
}
void* Maat_hierarchy_compile_read_user_data(struct Maat_hierarchy* hier, int compile_id)
{
void* user_data=NULL;
user_data=Maat_hier_get_user_data(hier, compile_id, 0);
return user_data;
}
void Maat_hierarchy_compile_user_data_iterate(struct Maat_hierarchy* hier, void (*callback)(void *user_data, void* apram), void* param)
{
struct Maat_hierarchy_compile* compile=NULL, *tmp_compile=NULL;
pthread_rwlock_rdlock(&hier->rwlock);
HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile)
{
callback(compile->user_data, param);
}
pthread_rwlock_unlock(&hier->rwlock);
return;
}
struct Maat_hierarchy_group* Maat_hierarchy_group_new(struct Maat_hierarchy* hier, int group_id)
{
struct Maat_hierarchy_group* group=NULL;
@@ -315,10 +440,17 @@ static size_t effective_vertices_count(igraph_vector_t *vids)
}
return i;
}
size_t print_igraph_vector(igraph_vector_t *v, char* buff, size_t sz) {
long int i;
int printed=0;
for (i=0; i<igraph_vector_size(v); i++) {
printed+=snprintf(buff+printed, sz-printed, " %li", (long int) VECTOR(*v)[i]);
}
return printed;
}
static void Maat_hierarchy_group_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_group* group)
{
int ret=0;
igraph_vector_t v;
char buff[4096];
assert(group->ref_by_compile_cnt==0&&group->ref_by_group_cnt==0);
@@ -327,7 +459,7 @@ static void Maat_hierarchy_group_free(struct Maat_hierarchy* hier, struct Maat_h
if(igraph_vector_size(&v)>0)
{
print_igraph_vector(&v, buff, sizeof(buff));
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module,
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
"Del group %d exception, still reached by %s.",
group->vertex_id, buff);
assert(0);
@@ -340,62 +472,11 @@ static void Maat_hierarchy_group_free(struct Maat_hierarchy* hier, struct Maat_h
HASH_DELETE(hh_group_id, hier->hash_group_by_id, group);
HASH_DELETE(hh_vertex_id, hier->hash_group_by_vertex, group);
free(group->top_group_ids);
free(group);
_group_vertex_free(group);
return;
}
static struct Maat_hierarchy_literal* Maat_hierarchy_literal_new(struct Maat_hierarchy* hier, int group_id, int vt_id)
{
struct Maat_hierarchy_literal* literal=ALLOC(struct Maat_hierarchy_literal, 1);
literal->literal_id.group_id=group_id;
literal->literal_id.vt_id=vt_id;
literal->hash_clause_by_id=NULL;
HASH_ADD(hh, hier->hash_literal_by_id, literal_id, sizeof(literal->literal_id), literal);
return literal;
}
static void Maat_hierarchy_literal_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_literal* literal)
{
struct Maat_hierarchy_clause* clause=NULL, *tmp=NULL;
HASH_ITER(hh, literal->hash_clause_by_id, clause, tmp)
{
HASH_DELETE(hh, literal->hash_clause_by_id, clause);
free(clause);
}
HASH_DELETE(hh, hier->hash_literal_by_id, literal);
free(literal);
}
static int Maat_hierarchy_literal_join_clause(struct Maat_hierarchy_literal* literal, int not_flag, int Nth_clause, int compile_id)
{
struct Maat_hierarchy_clause* clause=NULL;
struct Maat_hierarchy_clause_id clause_id={Nth_clause, compile_id};
HASH_FIND(hh, literal->hash_clause_by_id, &clause_id, sizeof(clause_id), clause);
if(clause)
{
return -1;
}
clause=ALLOC(struct Maat_hierarchy_clause, 1);
clause->clause_id=clause_id;
clause->not_flag=not_flag;
HASH_ADD(hh, literal->hash_clause_by_id, clause_id, sizeof(clause->clause_id), clause);
return 0;
}
static int Maat_hierarchy_literal_leave_clause(struct Maat_hierarchy_literal* literal, int Nth_clause, int compile_id)
{
struct Maat_hierarchy_clause* clause=NULL;
struct Maat_hierarchy_clause_id clause_id={Nth_clause, compile_id};
HASH_FIND(hh, literal->hash_clause_by_id, &clause_id, sizeof(clause_id), clause);
if(!clause)
{
return -1;
}
HASH_DELETE(hh, literal->hash_clause_by_id, clause);
free(clause);
return 0;
}
int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int Nth_clause, int compile_id)
{
int ret=0;
@@ -404,7 +485,7 @@ int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_i
struct Maat_hierarchy_literal_id literal_id={group_id, vt_id};
pthread_rwlock_wrlock(&hier->rwlock);
HASH_FIND(hh_group_id, hier->hash_group_by_id, group_id, sizeof(group_id), group);
HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
if(!group)
{
group=Maat_hierarchy_group_new(hier, group_id);
@@ -414,11 +495,11 @@ int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_i
{
literal=Maat_hierarchy_literal_new(hier, group_id, vt_id);
}
ret=Maat_hierarchy_literal_join_clause(literal, Nth_clause, compile_id);
ret=Maat_hierarchy_literal_join_clause(literal, not_flag, Nth_clause, compile_id);
pthread_rwlock_unlock(&hier->rwlock);
if(ret<0)
{
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module,
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
"Add group %d vt_id %d to clause %d of compile %d failed, group is already exisited.",
group_id, vt_id, Nth_clause, compile_id);
ret=-1;
@@ -433,16 +514,16 @@ int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_i
}
int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int Nth_clause, int compile_id)
{
struct Maat_hierarchy_compile* compile=NULL;
struct Maat_hierarchy_group* group=NULL;
struct Maat_hierarchy_literal* literal=NULL;
struct Maat_hierarchy_literal_id literal_id={group_id, vt_id};
int ret=0;
pthread_rwlock_wrlock(&hier->rwlock);
HASH_FIND(hh_group_id, hier->hash_group_by_id, group_id, sizeof(group_id), group);
HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
if(!group)
{
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module,
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
"Remove group %d from compile %d failed, group is not exisited.",
group_id, compile_id);
goto error_out;
@@ -450,7 +531,7 @@ int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int gr
HASH_FIND(hh, hier->hash_literal_by_id, &literal_id, sizeof(literal_id), literal);
if(!literal)
{
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module,
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
"Remove group %d from compile %d failed, literal is not exisited.",
group_id, compile_id);
goto error_out;
@@ -458,7 +539,7 @@ int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int gr
ret=Maat_hierarchy_literal_leave_clause(literal, Nth_clause, compile_id);
if(ret<0)
{
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module,
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
"Remove group %d vt_id %d from clause %d of compile %d failed, clause is not exisited.",
group_id, vt_id, Nth_clause, compile_id);
goto error_out;
@@ -468,7 +549,7 @@ int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int gr
error_out:
pthread_rwlock_unlock(&hier->rwlock);
return NULL;
return -1;
}
@@ -476,7 +557,7 @@ int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id,
{
int ret=0;
igraph_integer_t edge_id;
struct Maat_hierarchy_group* group=NULL, superior_group=NULL;
struct Maat_hierarchy_group* group=NULL, *superior_group=NULL;
pthread_rwlock_wrlock(&hier->rwlock);
HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
@@ -493,7 +574,7 @@ int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id,
ret=igraph_get_eid(&hier->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0);
if(edge_id>0)//No duplicated edges between two groups.
{
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module,
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
"Add group %d to group %d failed, relation already exisited.",
group->group_id, superior_group->group_id);
ret=-1;
@@ -511,15 +592,14 @@ int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id,
int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id)
{
int ret=0;
igraph_integer_t edge_id;
struct Maat_hierarchy_group* group=NULL, superior_group=NULL;
struct Maat_hierarchy_group* group=NULL, *superior_group=NULL;
//No hash write operation, LOCK protection is unnecessary.
HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
if(group==NULL)
{
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module,
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
"Del group %d from group %d failed, group %d not exisited.",
group_id, superior_group_id, group_id);
return -1;
@@ -527,7 +607,7 @@ int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int grou
HASH_FIND(hh_group_id, hier->hash_group_by_id, &superior_group_id, sizeof(superior_group_id), superior_group);
if(superior_group==NULL)
{
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module,
MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
"Del group %d from group %d failed, superior group %d not exisited.",
group_id, superior_group_id, superior_group_id);
return -1;
@@ -552,40 +632,15 @@ int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int grou
if(ret!=IGRAPH_SUCCESS||edge_num_before-edge_num_after!=1)
{
assert(0)
assert(0);
return -1;
}
else
{
group->ref_by_group_cnt--;
return 0;
}
}
static struct Maat_hierarchy_region* Maat_hierarchy_region_new(struct Maat_hierarchy* hier, int region_id, int group_id, int table_id, struct Maat_hierarchy_group* parent_group, void* user_data)
{
struct Maat_hierarchy_region* region=NULL;
region=ALLOC(struct Maat_hierarchy_region, 1);
region->group_id=group_id
region->region_id=region_id;
region->table_id=table_id;
region->ref_parent_group=parent_group;
region->user_data=user_data;
HASH_ADD_INT(hier->hash_region_by_id, region_id, region);
parent_group->ref_by_region_cnt++;
return region;
}
static void Maat_hierarchy_region_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_region* region)
{
HASH_DELETE(hh, hier->hash_region_by_id, region);
region->ref_parent_group->ref_by_region_cnt--;
if(region->user_data)
{
hier->region_user_data_free(region->user_data);
region->user_data=NULL;
}
free(region);
return;
}
int Maat_hierarchy_add_region_to_group(struct Maat_hierarchy* hier, int group_id, int region_id, int table_id, void* user_data)
{
@@ -627,7 +682,7 @@ void* Maat_hierarchy_region_dettach_user_data(struct Maat_hierarchy* hier, int r
HASH_FIND(hh, hier->hash_region_by_id, &region_id, sizeof(region_id), region);
if(region)
{
*ret=region->user_data;
ret=region->user_data;
region->user_data=NULL;
}
pthread_rwlock_unlock(&hier->rwlock);
@@ -637,7 +692,6 @@ int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int gro
{
struct Maat_hierarchy_group* group=NULL;
struct Maat_hierarchy_region* region=NULL;
int ret=0;
pthread_rwlock_wrlock(&hier->rwlock);
HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
if(!group)
@@ -646,7 +700,7 @@ int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int gro
"Remove region %d from group %d failed, group is not existed.",
region_id,
group_id);
goto error_out
goto error_out;
}
HASH_FIND(hh, hier->hash_region_by_id, &region_id, sizeof(region_id), region);
if(!region)
@@ -659,7 +713,7 @@ int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int gro
}
assert(region->group_id==group->group_id);
Maat_hierarchy_region_free(hier, region)
Maat_hierarchy_region_free(hier, region);
pthread_rwlock_unlock(&hier->rwlock);
return 0;
@@ -681,7 +735,8 @@ static struct bool_matcher* Maat_hierarchy_build_bool_matcher(struct Maat_hierar
//STEP 1, for building a new bool matcher, we need to reset previous clause states of each compile.
HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile)
{
memset(compile->clause_states);
memset(compile->clause_states, 0, sizeof(compile->clause_states));
utarray_clear(compile->literal_ids);
}
@@ -696,14 +751,14 @@ static struct bool_matcher* Maat_hierarchy_build_bool_matcher(struct Maat_hierar
{
if(compile->clause_states[clause->clause_id.Nth_clause].in_use==1)
{
assert(compile->clause_states[clause->clause_id.Nth_clause].not_flag==clause.not_flag);
assert(compile->clause_states[clause->clause_id.Nth_clause].not_flag==clause->not_flag);
}
else
{
compile->clause_states[clause->clause_id.Nth_clause].in_use=1;
compile->clause_states[clause->clause_id.Nth_clause].not_flag=clause.not_flag;
compile->clause_states[clause->clause_id.Nth_clause].not_flag=clause->not_flag;
}
utarray_sort
utarray_push_back(compile->literal_ids, &literal->literal_id);
}
}
}
@@ -732,7 +787,7 @@ static struct bool_matcher* Maat_hierarchy_build_bool_matcher(struct Maat_hierar
bool_expr_array[expr_cnt].item_num=j;
expr_cnt++;
}
utarray_sort(compile->literal_ids, compare_literal_id);
}
//Final STEP, build the bool matcher.
@@ -754,7 +809,7 @@ static struct bool_matcher* Maat_hierarchy_build_bool_matcher(struct Maat_hierar
}
static int Maat_hierarchy_build_top_groups(struct Maat_hierarchy* hier)
{
struct Maat_hierarchy_group* group=NULL, tmp=NULL;
struct Maat_hierarchy_group* group=NULL, *tmp=NULL;
struct Maat_hierarchy_group* superior_group=NULL;
int tmp_vid=0;
size_t i=0, top_group_cnt=0;
@@ -780,7 +835,7 @@ static int Maat_hierarchy_build_top_groups(struct Maat_hierarchy* hier)
pthread_rwlock_wrlock(&hier->rwlock);
free(group->top_group_ids);
Maat_hierarchy_group_free(hier, group->group_id);
Maat_hierarchy_group_free(hier, group);
pthread_rwlock_unlock(&hier->rwlock);
continue;
}
@@ -792,13 +847,13 @@ static int Maat_hierarchy_build_top_groups(struct Maat_hierarchy* hier)
{
//fast path, group is only referenced by compile rules.
top_group_cnt=1;
temp_group_ids=ALLOC(long long, top_group_cnt);
temp_group_ids=ALLOC(int, top_group_cnt);
temp_group_ids[0]=group->group_id;
}
else
{
igraph_vector_t *vids=&(hier->dfs_vids);
igraph_dfs(&hier->group_graph), group->vertex_id, IGRAPH_OUT,
igraph_dfs(&hier->group_graph, group->vertex_id, IGRAPH_OUT,
0, vids, NULL, NULL, NULL, NULL, NULL, NULL);
temp_group_ids=ALLOC(int, effective_vertices_count(vids));
@@ -810,7 +865,7 @@ static int Maat_hierarchy_build_top_groups(struct Maat_hierarchy* hier)
{
break;
}
HASH_FIND(hh_vertex_id, hier->hash_group_by_vertex, tmp_vid, sizeof(tmp_vid), superior_group);
HASH_FIND(hh_vertex_id, hier->hash_group_by_vertex, &tmp_vid, sizeof(tmp_vid), superior_group);
if(superior_group->ref_by_compile_cnt>0)//including itself
{
temp_group_ids[top_group_cnt]=superior_group->group_id;
@@ -837,7 +892,7 @@ static int Maat_hierarchy_build_top_groups(struct Maat_hierarchy* hier)
int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier)
{
int ret=0;
struct bool_matcher* new_bm=NULL, old_bm=NULL;
struct bool_matcher* new_bm=NULL, *old_bm=NULL;
new_bm=Maat_hierarchy_build_bool_matcher(hier);
old_bm=hier->bm;
@@ -849,13 +904,8 @@ int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier)
ret=Maat_hierarchy_build_top_groups(hier);
return ret;
}
struct Maat_hierarchy_hit_path_inner
{
int Nth_hit_region;
struct Maat_hit_path_t path;
TAILQ_ENTRY(Maat_hierarchy_hit_path_inner) entries;
};
TAILQ_HEAD(hit_path_q, Maat_hierarchy_hit_path_inner);
TAILQ_HEAD(hit_path_q, Maat_hierarchy_hit_path);
struct Maat_hierarchy_compile_mid
{
@@ -863,7 +913,7 @@ struct Maat_hierarchy_compile_mid
int thread_num;
int Nth_scan;
size_t this_scan_region_hits;
int not_grp_compile_hitted_flag;
int not_clause_hitted_flag;
size_t hit_path_cnt;
struct hit_path_q hit_path_qhead;
@@ -878,11 +928,11 @@ struct Maat_hierarchy_compile_mid* Maat_hierarchy_compile_mid_new(struct Maat_hi
TAILQ_INIT(&mid->hit_path_qhead);
mid->thread_num=thread_num;
mid->ref_hier=hier;
return;
return mid;
}
void Maat_hierarchy_compile_mid_free(struct Maat_hierarchy_compile_mid* mid)
{
struct Maat_hierarchy_hit_path_inner * tmp = TAILQ_FIRST(&mid->hit_path_qhead);
struct Maat_hierarchy_hit_path * tmp = TAILQ_FIRST(&mid->hit_path_qhead);
while(tmp != NULL)
{
TAILQ_REMOVE(&mid->hit_path_qhead, tmp, entries);
@@ -896,6 +946,11 @@ void Maat_hierarchy_compile_mid_free(struct Maat_hierarchy_compile_mid* mid)
mid->ref_hier=NULL;
free(mid);
}
int Maat_hierarchy_compile_mid_has_NOT_clause(struct Maat_hierarchy_compile_mid* mid)
{
return mid->not_clause_hitted_flag;
}
//return 1 if insert a unique id
//return 0 if id is duplicated
//return -1 if set is full
@@ -924,12 +979,58 @@ int insert_clause_id(unsigned long long **set, size_t* size, size_t cnt, unsigne
return 0;
}
}
void Maat_hit_path_init(struct Maat_hit_path_t* hit_path)
{
hit_path->Nth_scan=-1;
hit_path->region_id=-1;
hit_path->sub_group_id=-1;
hit_path->top_group_id=-1;
hit_path->virtual_table_id=-1;
hit_path->compile_id=-1;
}
static size_t hit_path_select(const struct hit_path_q *hit_path_qhead, struct Maat_hit_path_t* condition,
struct Maat_hierarchy_hit_path** hit_paths, size_t n_path)
{
struct Maat_hierarchy_hit_path* p=NULL;
size_t i=0;
TAILQ_FOREACH(p, hit_path_qhead, entries)
{
if((condition->compile_id==p->path.compile_id||condition->compile_id<0)
&& (condition->Nth_scan==p->path.Nth_scan||condition->Nth_scan<0)
&& (condition->region_id=p->path.region_id||condition->region_id<0)
&& (condition->sub_group_id=p->path.sub_group_id||condition->sub_group_id<0)
&& (condition->top_group_id=p->path.top_group_id||condition->top_group_id<0))
{
if(i<n_path)
{
hit_paths[i]=p;
i++;
}
else if(hit_paths==NULL)//count only
{
i++;
}
}
}
return i;
}
size_t Maat_hierarchy_hit_path_select0(const struct Maat_hierarchy_compile_mid* mid, struct Maat_hit_path_t* condition,
struct Maat_hierarchy_hit_path** hit_paths, size_t n_path)
{
size_t ret=0;
ret=hit_path_select(&mid->hit_path_qhead, condition, hit_paths, n_path);
return ret;
}
void Maat_hierarchy_compile_mid_udpate(struct Maat_hierarchy_compile_mid* mid, int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result)
{
size_t i=0;
int ret=0;
struct Maat_hit_path_inner* hit_path=NULL;
int ret=0, i=0;
struct Maat_hierarchy_hit_path* hit_path=NULL;
struct Maat_hierarchy_region* region=NULL;
struct Maat_hierarchy_group* group=NULL;
@@ -945,18 +1046,18 @@ void Maat_hierarchy_compile_mid_udpate(struct Maat_hierarchy_compile_mid* mid, i
mid->Nth_scan=Nth_scan;
}
pthread_rwlock_rdlock(&hier->rwlock);
HASH_FIND_INT(hier->hash_region_by_id, region_id, region);
HASH_FIND_INT(hier->hash_region_by_id, &region_id, region);
group=region->ref_parent_group;
if(group->top_group_cnt==0)
{
hit_path=ALLOC(struct Maat_hit_path_inner, 1);
hit_path_init(&(hit_path->path));
hit_path=ALLOC(struct Maat_hierarchy_hit_path, 1);
Maat_hit_path_init(&(hit_path->path));
hit_path->Nth_hit_region=Nth_region_result;
hit_path->path.Nth_scan=Nth_scan;
hit_path->path.region_id=region_id;
hit_path->path.sub_group_id=group->group_id;
hit_path->path.vt_id=virtual_table_id;
hit_path->path.virtual_table_id=virtual_table_id;
TAILQ_INSERT_TAIL(&mid->hit_path_qhead, hit_path, entries);
mid->hit_path_cnt++;
}
@@ -964,27 +1065,27 @@ void Maat_hierarchy_compile_mid_udpate(struct Maat_hierarchy_compile_mid* mid, i
{
for(i=0; i<group->top_group_cnt; i++)
{
hit_path=ALLOC(struct Maat_hit_path_inner, 1);
hit_path_init(&(hit_path->path));
hit_path=ALLOC(struct Maat_hierarchy_hit_path, 1);
Maat_hit_path_init(&(hit_path->path));
hit_path->Nth_hit_region=Nth_region_result;
hit_path->path.Nth_scan=Nth_scan;
hit_path->path.region_id=region_id;
hit_path->path.sub_group_id=group->group_id;
hit_path->path.top_group_id=group->top_group_ids[i];
hit_path->path.vt_id=virtual_table_id;
hit_path->path.virtual_table_id=virtual_table_id;
TAILQ_INSERT_TAIL(&mid->hit_path_qhead, hit_path, entries);
mid->hit_path_cnt++;
literal_id.vt_id=virtual_table_id;
literal_id.virtual_table_id=virtual_table_id;
literal_id.group_id=group->top_group_ids[i];
HASH_FIND(hh, hier->hash_literal_by_id, &literal_id, literal);
HASH_FIND(hh, hier->hash_literal_by_id, &literal_id, sizeof(literal_id), literal);
if(!literal)
{
continue;
}
HASH_ITER(hh, literal->hash_clause_by_id, clause, tmp_clause)
{
ret=insert_clause_id(mid->all_hit_clause_array, mid->all_hit_clause_array_sz, mid->all_hit_clause_cnt, TO_CLAUSE_ID(clause->clause_id.Nth_clause, clause->clause_id.compile_id));
ret=insert_clause_id(&mid->all_hit_clause_array, &mid->all_hit_clause_array_sz, mid->all_hit_clause_cnt, TO_CLAUSE_ID(clause->clause_id.Nth_clause, clause->clause_id.compile_id));
mid->all_hit_clause_cnt+=ret;
}
}
@@ -992,113 +1093,86 @@ void Maat_hierarchy_compile_mid_udpate(struct Maat_hierarchy_compile_mid* mid, i
pthread_rwlock_unlock(&hier->rwlock);
return;
}
int compare_literal_id(const void *pa, const void *pb)
{
struct Maat_hierarchy_literal_id *la=(struct Maat_hierarchy_literal_id *)pa;
struct Maat_hierarchy_literal_id *lb=(struct Maat_hierarchy_literal_id *)pb;
return TO_LITERAL_ID(la->vt_id, la->group_id)-TO_LITERAL_ID(lb->vt_id, lb->group_id);
}
static size_t Maat_hierarchy_compile_mid_update_by_compile(struct Maat_hierarchy_compile_mid* mid, struct Maat_hierarchy_compile* compile)
{
int i=0;
size_t r_in_c_cnt=0;
struct Maat_hierarchy_hit_path_inner* p;
struct Maat_hierarchy_literal_id literal_id={0,0}, *l;
for(i=0; i<mid->this_scan_region_hits; i++)
{
size_t r_in_c_cnt=0, this_scan_hit_region_cnt=0;
struct Maat_hierarchy_hit_path* p=NULL, *q=NULL;
struct Maat_hierarchy_literal_id literal_id={0,0}, *l=NULL;
struct Maat_hit_path_t condition;
size_t n_exsited_path=0;
struct hit_path_q new_path_qhead;
TAILQ_INIT(&new_path_qhead);
}
TAILQ_FOREACH(p, &mid->hit_path_qhead, entries)
{
literal_id={p->path.top_group_id, p->path.virtual_table_id};
n_exsited_path=0;
if(p->path.Nth_scan==mid->Nth_scan)
{
this_scan_hit_region_cnt++;
}
if(p->path.compile_id==compile->compile_id)
{
continue;
}
literal_id.group_id=p->path.top_group_id;
literal_id.virtual_table_id=p->path.virtual_table_id;
l=(struct Maat_hierarchy_literal_id*)utarray_find(compile->literal_ids, &literal_id, compare_literal_id);
if(!l)
{
continue;
}
assert(*l==literal_id);
}
mid->this_scan_region_hits=0;
size_t i=0, j=0;
struct Maat_hit_path_inner* p=NULL, *q=NULL;
struct bool_expr a_set;
unsigned char has_not=0;
struct Maat_hit_path_t condition;
size_t n_exsited_path=0;
struct hit_path_q shared_path_qhead;
TAILQ_INIT(&shared_path_qhead);
struct Maat_hierarchy* hier=mid->ref_hier;
struct Maat_hierarchy_compile* compile=NULL;
make_group_set(compile_rule, &a_set, &has_not);
for(i=0; i<a_set.item_num; i++)
{
TAILQ_FOREACH(p, &hit_status->hit_path_qhead, entries)
{
n_exsited_path=0;
if(TO_RELATION_ID(p->path.vt_id, p->path.top_group_id)==a_set.items[i].item_id
&& p->path.compile_id!=compile_rule->compile_id)
{
assert(l->group_id==literal_id.group_id && l->virtual_table_id==literal_id.virtual_table_id);
if(p->path.compile_id<0)
{
p->path.compile_id=compile_rule->compile_id;
p->path.compile_id=compile->compile_id;
}
else //current path already have a compile as endpoint, so we duplicate a new path.
else //p->path.compile_id!=compile->compile_id, current literal already in a path that has a compile, may be a new path.
{
condition=p->path;
condition.compile_id=compile_rule->compile_id;
n_exsited_path=scan_hit_status_select_hit_path_inner(&shared_path_qhead, &condition, NULL, 0);
condition.compile_id=compile->compile_id;
n_exsited_path=hit_path_select(&new_path_qhead, &condition, NULL, 0);
if(n_exsited_path==0)
{
q=ALLOC(struct Maat_hit_path_inner, 1);
q=ALLOC(struct Maat_hierarchy_hit_path, 1);
*q=*p;
q->path.compile_id=compile_rule->compile_id;
TAILQ_INSERT_TAIL(&shared_path_qhead, q, entries);
hit_status->hit_path_cnt++;
q->path.compile_id=compile->compile_id;
TAILQ_INSERT_TAIL(&new_path_qhead, q, entries);
mid->hit_path_cnt++;
}
}
if(p->path.Nth_scan==mid->Nth_scan && n_exsited_path==0)//Compile was satisfied by new region hits.
{
j++;
r_in_c_cnt++;
}
}
}
}
p = TAILQ_FIRST(&shared_path_qhead);
assert(this_scan_hit_region_cnt==mid->this_scan_region_hits);
mid->this_scan_region_hits=0;
p = TAILQ_FIRST(&new_path_qhead);
while(p != NULL)
{
TAILQ_REMOVE(&shared_path_qhead, p, entries);
TAILQ_INSERT_TAIL(&hit_status->hit_path_qhead, p, entries);
p = TAILQ_FIRST(&shared_path_qhead);
TAILQ_REMOVE(&new_path_qhead, p, entries);
TAILQ_INSERT_TAIL(&mid->hit_path_qhead, p, entries);
p = TAILQ_FIRST(&new_path_qhead);
}
return j;
return r_in_c_cnt;
}
int Maat_hierarchy_region_compile(struct Maat_hierarchy_compile_mid* mid, int is_last_compile, void** user_data_array, size_t ud_array_sz)
{
int bool_match_ret=0, ret=0;
size_t n_clauses=0;
int bool_match_ret=0, i=0;
struct Maat_hierarchy* hier=mid->ref_hier;
struct Maat_hierarchy_literal_id literal_id={0,0};
unsigned long long * clause_ids=NULL;
const struct Maat_hierarchy_region* region=NULL;
struct Maat_hierarchy_group* group=NULL;
struct Maat_hierarchy_literal* literal=NULL;
struct Maat_hierarchy_clause* clause=NULL, *tmp_clause=NULL;
struct Maat_hierarchy_compile* compile[ud_array_sz];
struct Maat_hierarchy_compile* compile_array[ud_array_sz];
size_t i=0, r_in_c_cnt=0, this_scan_region_hits=mid->this_scan_region_hits;
size_t r_in_c_cnt=0, this_scan_region_hits=mid->this_scan_region_hits;
size_t ud_result_cnt=0;
if(!hier->bm)
{
@@ -1108,21 +1182,21 @@ int Maat_hierarchy_region_compile(struct Maat_hierarchy_compile_mid* mid, int is
pthread_rwlock_rdlock(&hier->rwlock);
bool_match_ret=bool_matcher_match(hier->bm, mid->thread_num,
mid->all_hit_clause_array, mid->all_hit_clause_cnt,
compile, ud_array_sz);
(void**)compile_array, ud_array_sz);
for(i=0; i<bool_match_ret; i++)
{
r_in_c_cnt=Maat_hierarchy_compile_mid_update_by_compile(mid, compile);
if(compile[i].not_clause_cnt>0 && !is_last_compile)
r_in_c_cnt=Maat_hierarchy_compile_mid_update_by_compile(mid, compile_array[i]);
if(compile_array[i]->not_clause_cnt>0 && !is_last_compile)
{
mid->not_grp_compile_hitted_flag=1;
mid->not_clause_hitted_flag=1;
}
else
{
if(r_in_c_cnt>0 || //compile hitted becasue of new reigon
this_scan_region_hits==0) //or hit a compile that refer a NOT-logic group in previous scan.
{
user_data_array[ud_result_cnt]=compile[i]->user_data;
ud_result_cnt++
user_data_array[ud_result_cnt]=compile_array[i]->user_data;
ud_result_cnt++;
}
}
}

76
src/entry/Maat_rule.cpp
View File

@@ -21,6 +21,9 @@
#include "Maat_rule.h"
#include "Maat_rule_internal.h"
#include "Maat_utils.h"
#include "Maat_hierarchy.h"
#include "Maat_garbage_collection.h"
#include "json2iris.h"
#include "cJSON.h"
#include "dynamic_array.h"
@@ -28,8 +31,6 @@
#include "config_monitor.h"
#include "map_str2int.h"
#include "rulescan.h"
#include "bool_matcher.h"
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
@@ -712,6 +713,18 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule)
op_expr->rule_type=p_rule->rule_type;
return;
}
void Maat_region_inner_free(struct Maat_region_inner* region)
{
free(region);
}
void Maat_region_inner_cancel_last_expr_id(struct Maat_region_inner* region)
{
assert(region->expr_id_cnt==region->expr_id_ub-region->expr_id_lb);
region->expr_id_ub--;
region->expr_id_cnt--;
return;
}
struct Maat_scanner* create_maat_scanner(unsigned int version, _Maat_feather_t *feather)
{
@@ -721,8 +734,8 @@ struct Maat_scanner* create_maat_scanner(unsigned int version, _Maat_feather_t *
struct Maat_scanner* scanner=NULL;
scanner=ALLOC(struct Maat_scanner, 1);
scanner->hier=Maat_hierarchy_new(scan_thread_num, feather->logger);
Maat_hierarchy_set_compile_user_data_free_func(scanner->hier, destroy_compile_rule);
Maat_hierarchy_set_region_user_data_free_func(scanner->hier, Maat_region_inner_free);
Maat_hierarchy_set_compile_user_data_free_func(scanner->hier, (void (*)(void*))destroy_compile_rule);
Maat_hierarchy_set_region_user_data_free_func(scanner->hier, (void (*)(void*))Maat_region_inner_free);
scanner->district_map=map_create();
@@ -1038,18 +1051,7 @@ void Maat_region_inner_add_expr_id(struct Maat_region_inner* region, int expr_id
region->expr_id_cnt++;
return;
}
void Maat_region_inner_free(struct Maat_region_inner* region)
{
free(region);
}
void Maat_region_inner_cancel_last_expr_id(struct Maat_region_inner* region)
{
assert(region->expr_id_cnt==region->expr_id_ub-region->expr_id_lb);
region->expr_id_ub--;
region->expr_id_cnt--;
return;
}
int add_expr_rule(struct Maat_table_schema* table,struct db_str_rule_t* db_rule,struct Maat_scanner *scanner,void* logger)
{
@@ -1307,7 +1309,7 @@ int add_ip_rule(struct Maat_table_schema* table, struct db_ip_rule_t* db_ip_rule
scan_rule_t* p_rule=NULL;
struct op_expr_t* op_expr=NULL;
struct Maat_region_inner* u_para=NULL;
int expr_id=0, district_id=-1, ret=0;
int expr_id=0, ret=0;
u_para=Maat_region_inner_new(db_ip_rule->group_id, db_ip_rule->region_id, table->table_id, -1);
ret=Maat_hierarchy_add_region_to_group(scanner->hier, db_ip_rule->group_id, db_ip_rule->region_id, table->table_id, u_para);
@@ -1386,7 +1388,6 @@ int add_digest_rule(struct Maat_table_schema* table, struct db_digest_rule* db_r
int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id, int rule_type, struct Maat_scanner *maat_scanner, void* logger)
{
int i=0;
int expr_num=0;
struct Maat_table_runtime* table_rt=NULL;
struct op_expr_t* op_expr=NULL;
@@ -1426,7 +1427,7 @@ int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id
assert(0);
break;
}
Maat_garbage_bag(maat_scanner->ref_garbage_bin, region, Maat_region_inner_free);
Maat_garbage_bagging(maat_scanner->ref_garbage_bin, region, (void (*)(void*))Maat_region_inner_free);
return 0;
}
@@ -1437,7 +1438,7 @@ void update_group2compile_rule(struct Maat_table_schema* table, const char* tabl
int ret=0;
char virtual_table_name[MAX_TABLE_NAME_LEN]={0};
memset(&db_g2c_rule, 0, sizeof(db_g2c_rule));
ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%d\t%s", &(db_g2c_rule.group_id),
ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%s\t%d", &(db_g2c_rule.group_id),
&(db_g2c_rule.compile_id),
&(db_g2c_rule.is_valid),
&(db_g2c_rule.not_flag),
@@ -1486,7 +1487,7 @@ void update_group2compile_rule(struct Maat_table_schema* table, const char* tabl
if(ret==0)
{
table_rt->origin_rule_num++;
if(db_group2compile_rule.not_flag)
if(db_g2c_rule.not_flag)
{
table_rt->group2compile.not_flag_group++;
}
@@ -1528,10 +1529,6 @@ void update_group2group_rule(struct Maat_table_schema* table, const char* table_
if(ret==0)
{
table_rt->origin_rule_num++;
if(db_group2compile_rule.not_flag)
{
table_rt->group2compile.not_flag_group++;
}
scanner->to_update_group_cnt++;
}
}
@@ -1627,16 +1624,6 @@ void update_expr_rule(struct Maat_table_schema* table,const char* table_line,str
table->udpate_err_cnt++;
goto error_out;
}
ret=sync_region(scanner->region_hash,
maat_str_rule->region_id,
maat_str_rule->group_id,
table->table_name[table->updating_name],
maat_str_rule->is_valid,logger);
if(ret<0)
{
table->udpate_err_cnt++;
goto error_out;
}
if(maat_str_rule->is_valid==FALSE)
{
@@ -1876,16 +1863,6 @@ void update_ip_rule(struct Maat_table_schema* table, const char* table_line, str
goto error_out;
}
}
ret=sync_region(scanner->region_hash,
ip_rule->region_id,
ip_rule->group_id,
table->table_name[table->updating_name],
ip_rule->is_valid,logger);
if(ret<0)
{
table->udpate_err_cnt++;
goto error_out;
}
if(ip_rule->is_valid==FALSE)
{
ret=del_region_rule(table,
@@ -2045,7 +2022,7 @@ void update_compile_rule(struct Maat_table_schema* table,const char* table_line
ret=Maat_hierarchy_compile_remove(scanner->hier, m_rule_tmp.config_id);
assert(ret==0);
table_rt->origin_rule_num--;
Maat_garbage_bag(scanner->ref_garbage_bin, p_compile, destroy_compile_rule);
Maat_garbage_bagging(scanner->ref_garbage_bin, p_compile, (void (*)(void*))destroy_compile_rule);
}
else
{
@@ -2246,7 +2223,6 @@ void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char*
void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* logger)
{
struct bool_matcher *tmp1=NULL,*tmp2=NULL;
MESA_htable_handle tmp_map=NULL;
struct Maat_table_runtime* table_rt=NULL;
struct ip_matcher* old_ip_matcher=NULL;
@@ -2295,7 +2271,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
case TABLE_TYPE_IP_PLUGIN:
Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt);
old_ip_matcher=Maat_table_runtime_dettach_old_ip_matcher(table_rt);
Maat_garbage_bag(scanner->ref_garbage_bin, old_ip_matcher, ip_matcher_free);
Maat_garbage_bagging(scanner->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
break;
default:
break;
@@ -2307,7 +2283,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
tmp_map=scanner->district_map;
scanner->district_map=scanner->tmp_district_map;
scanner->tmp_district_map=NULL;
Maat_garbage_bag(scanner->ref_garbage_bin, tmp_map, map_destroy)
Maat_garbage_bagging(scanner->ref_garbage_bin, tmp_map, (void (*)(void*))map_destroy);
}
scanner->last_update_time=time(NULL);
scanner->gie_update_q_size=0;
@@ -2592,7 +2568,7 @@ void *thread_rule_monitor(void *arg)
old_scanner->version, feather->scanner->version);
}
feather->zombie_rs_stream+=alignment_int64_array_sum(old_scanner->ref_cnt,old_scanner->max_thread_num);
Maat_garbage_bag(feather->garbage_bin, old_scanner, destroy_maat_scanner);
Maat_garbage_bagging(feather->garbage_bin, old_scanner, (void (*)(void*))destroy_maat_scanner);
}
feather->update_tmp_scanner=NULL;
feather->maat_version=feather->scanner->version;
@@ -2628,7 +2604,7 @@ void *thread_rule_monitor(void *arg)
Maat_garbage_bin_free(feather->garbage_bin);
alignment_int64_array_free(feather->thread_call_cnt);
alignment_int64_array_free(feather->inner_mid_cnt);
alignment_int64_array_free(feather-> compile_mid_cnt);
alignment_int64_array_free(feather->outer_mid_cnt);
alignment_int64_array_free(feather->hit_cnt);
alignment_int64_array_free(feather->not_grp_hit_cnt);

2
src/entry/Maat_stat.cpp
View File

@@ -182,7 +182,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
}
active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num);
inner_mid_cnt=alignment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num);
inner_mid_cnt=alignment_int64_array_sum(feather->compile_mid_cnt,feather->scan_thread_num);
not_grp_hit_cnt=alignment_int64_array_sum(feather->not_grp_hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);

18
src/entry/Maat_table.cpp
View File

@@ -682,12 +682,12 @@ int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char
}
return 0;
}
int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
int Maat_table_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
{
int i=0;
for(i=0; i< MAX_TABLE_NUM; i++)
{
if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP)
if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP2COMPILE)
{
strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz);
return 1;
@@ -695,6 +695,20 @@ int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char*
}
return 0;
}
int Maat_table_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
{
int i=0;
for(i=0; i< MAX_TABLE_NUM; i++)
{
if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP2GROUP)
{
strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz);
return 1;
}
}
return 0;
}
const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id)
{
if(table_id>MAX_TABLE_NUM)

150
src/entry/json2iris.cpp
View File

@@ -47,6 +47,8 @@ struct iris_description_t
char index_path[MAX_PATH_LINE];
struct iris_table_t* group_table;
struct iris_table_t* group2group_table;
struct iris_table_t* group2compile_table;
struct iris_table_t* compile_table;
MESA_htable_handle group_name_map;
MESA_htable_handle iris_table_map;
@@ -123,7 +125,7 @@ static int get_region_seq(struct iris_description_t* iris_cfg)
return sequence;
}
int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_key, const char* encrypt_algo, const char*compile_tn,const char* group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger)
int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_key, const char* encrypt_algo, const char*compile_tn, const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger)
{
memset(iris_cfg,0,sizeof(struct iris_description_t));
snprintf(iris_cfg->tmp_iris_dir,sizeof(iris_cfg->tmp_iris_dir),"%s_iris_tmp",json_file);
@@ -190,7 +192,8 @@ int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_k
map_register(iris_cfg->str2int_map, "case plain",2);
iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn, TABLE_TYPE_COMPILE);
iris_cfg->group_table=query_table_info(iris_cfg, group_tn, TABLE_TYPE_GROUP);
iris_cfg->group2compile_table=query_table_info(iris_cfg, group2compile_tn, TABLE_TYPE_GROUP2COMPILE);
iris_cfg->group2group_table=query_table_info(iris_cfg, group2compile_tn, TABLE_TYPE_GROUP2GROUP);
if(encrypt_key && encrypt_algo)
{
@@ -795,12 +798,42 @@ int write_region_rule(cJSON* region_json, int compile_id, int group_id, iris_des
int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void * logger)
{
int compile_id=-1,cmd_cnt=0,ret=-1;
int compile_id=-1,cmd_cnt=0,ret=-1, clause_num=0, group_num=0, i=0, nth_clause=0;
cJSON* item=NULL;
struct iris_table_t* table_info=NULL;
cJSON* g_rules=cJSON_GetObjectItem(compile, "groups");
int group_cnt=cJSON_GetArraySize(g_rules);
cJSON_AddNumberToObject(compile, "group_num", group_cnt);
cJSON* group_array=NULL, *group_obj=NULL;
int* clause_ids=NULL;
group_array=cJSON_GetObjectItem(compile, "groups");
group_num=cJSON_GetArraySize(group_array);
clause_ids=ALLOC(int, group_num);
cJSON_ArrayForEach(group_obj, group_array);
{
item=cJSON_GetObjectItem(compile,"nth_clause");
if(item)
{
nth_clause=item->valueint;
for(i=0; i<clause_num; i++)
{
if(clause_ids[i]==nth_clause)
{
break;
}
}
if(i==clause_num)
{
clause_ids[clause_num]=nth_clause;
clause_num++;
}
}
}
free(clause_ids);
clause_ids=NULL;
if(clause_num==0)
{
clause_num=cJSON_GetArraySize(group_array);
}
cJSON_AddNumberToObject(compile, "clause_num", clause_num);
struct traslate_command_t compile_cmd[MAX_COLUMN_NUM];
memset(compile_cmd,0,sizeof(compile_cmd));
@@ -842,7 +875,7 @@ int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void *
compile_cmd[cmd_cnt].str2int_flag=1;
cmd_cnt++;
compile_cmd[cmd_cnt].json_string="group_num";
compile_cmd[cmd_cnt].json_string="clause_num";
compile_cmd[cmd_cnt].json_type=cJSON_Number;
cmd_cnt++;
@@ -878,15 +911,25 @@ int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void *
compile_id=item->valueint;
return compile_id;
}
int write_group_line(int group_id, int parent_id, int group_not_flag, int parent_type, const char* virtual_table, struct iris_description_t *p_iris, void * logger)
int write_group2compile_line(int group_id, int compile_id, int group_not_flag, int Nth_clause, const char* virtual_table, struct iris_description_t *p_iris, void * logger)
{
char buff[1024*4];
struct iris_table_t* table=p_iris->group_table;
snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%d\t%s\n", group_id, parent_id, group_not_flag, parent_type, virtual_table);
struct iris_table_t* table=p_iris->group2compile_table;
snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_id, compile_id, group_not_flag, virtual_table, Nth_clause);
table->write_pos+=memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff));
table->line_count++;
return 0;
}
int write_group2group_line(int group_id, int superior_gorup_id, struct iris_description_t *p_iris, void * logger)
{
char buff[1024*4];
struct iris_table_t* table=p_iris->group2group_table;
snprintf(buff, sizeof(buff), "%d\t%d\t1\n", group_id, superior_gorup_id);
table->write_pos+=memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff));
table->line_count++;
return 0;
}
void table_idx_write_cb(const uchar * key, uint size, void * data, void * user)
{
struct iris_description_t *p_iris=(struct iris_description_t *)user;
@@ -957,11 +1000,12 @@ static struct group_info_t* group_info_add_unsafe(struct iris_description_t* p_i
}
return group_info;
}
int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger)
int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, int Nth_group, struct iris_description_t *p_iris, void* logger)
{
const char* _str_parent_type[2]={"compile", "group"};
int ret=0;
int group_not_flag=0;
int ret=0, i=0;
int group_not_flag=0, Nth_clause=0;
cJSON *region_json=NULL, *item=NULL;
cJSON *sub_groups=NULL, *region_rule=NULL;
const char* group_name=NULL, *virtual_table=NULL;
@@ -976,7 +1020,8 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
{
group_name=item->valuestring;
}
if(parent_type==PARENT_TYPE_COMPILE)
{
item=cJSON_GetObjectItem(group_json, "virtual_table");
if(item==NULL||item->type!=cJSON_String)
{
@@ -995,7 +1040,17 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
{
group_not_flag=item->valueint;
}
if(parent_type==PARENT_TYPE_GROUP)
item=cJSON_GetObjectItem(group_json,"Nth_clause");
if(item==NULL||item->type!=cJSON_Number)
{
Nth_clause=Nth_group;
}
else
{
Nth_clause=item->valueint;
}
}
else
{
group_not_flag=0;
}
@@ -1021,14 +1076,16 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
if(sub_groups!=NULL)
{
//recursively
i=0;
cJSON_ArrayForEach(item, sub_groups)
{
ret=write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, p_iris, logger);
i++;
ret=write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, i, p_iris, logger);
if(ret<0)
{
return -1;
}
}
}
if(region_json==NULL && sub_groups==NULL)
@@ -1037,7 +1094,14 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
"A group of compile rule %d has neither regions, sub groups, nor refered another exisited group.", tracking_compile_id);
}
}
ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, virtual_table, p_iris, logger);
if(parent_type==PARENT_TYPE_COMPILE)
{
ret=write_group2compile_line(group_info->group_id, parent_id, group_not_flag, Nth_clause, virtual_table, p_iris, logger);
}
else
{
ret=write_group2group_line(group_info->group_id, parent_id, p_iris, logger);
}
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
@@ -1052,25 +1116,27 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
int i=0;
int compile_id=-1, compile_cnt=0, group_cnt=0;
int ret=0;
cJSON *c_rules=NULL, *g_rules=NULL, *plug_tables=NULL;
cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL, *item=NULL;
cJSON *compile_array=NULL, *group_array=NULL, *plug_tables=NULL;
cJSON *compile_obj=NULL, *group_obj=NULL, *each_plug_table=NULL, *item=NULL;
static struct group_info_t* parent_group=NULL;
const char* parent_group_name=NULL;
plug_tables=cJSON_GetObjectItem(json,"plugin_table");
if(NULL!=plug_tables)
{
i=0;
cJSON_ArrayForEach(each_plug_table, plug_tables)
{
write_plugin_line(each_plug_table, i, p_iris, logger);
i++;
}
}
g_rules=cJSON_GetObjectItem(json, "groups");//sub-group to group
if(g_rules!=NULL)
group_array=cJSON_GetObjectItem(json, "groups");//sub-group to group
if(group_array!=NULL)
{
cJSON_ArrayForEach(group_rule, g_rules)
cJSON_ArrayForEach(group_obj, group_array)
{
item=cJSON_GetObjectItem(group_rule, "parent_group");
item=cJSON_GetObjectItem(group_obj, "parent_group");
if(item==NULL || item->type!=cJSON_String)
{
parent_group_name=untitled_group_name;
@@ -1085,7 +1151,7 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
{
parent_group=group_info_add_unsafe(p_iris, p_iris->group_name_map, item->string);
}
ret=write_group_rule(group_rule, parent_group->group_id, PARENT_TYPE_GROUP, 0, p_iris, logger);
ret=write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, 0, 0, p_iris, logger);
if(ret<0)
{
return -1;
@@ -1093,47 +1159,48 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
}
}
c_rules=cJSON_GetObjectItem(json,"rules");
if(c_rules==NULL)
compile_array=cJSON_GetObjectItem(json,"rules");
if(compile_array==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"have no rules.");
return -1;
}
compile_cnt=cJSON_GetArraySize(c_rules);
compile_cnt=cJSON_GetArraySize(compile_array);
if(compile_cnt<=0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"have no rules.");
return -1;
}
cJSON_ArrayForEach(compile_rule, c_rules)
cJSON_ArrayForEach(compile_obj, compile_array)
{
compile_id=write_compile_line(compile_rule,p_iris, logger);
compile_id=write_compile_line(compile_obj,p_iris, logger);
if(compile_id<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"In %d compile rule.",i);
return -1;
}
g_rules=cJSON_GetObjectItem(compile_rule,"groups");
if(g_rules==NULL)
group_array=cJSON_GetObjectItem(compile_obj, "groups");
if(group_array==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"compile rule %d have no group.",compile_id);
return -1;
}
group_cnt=cJSON_GetArraySize(g_rules);
group_cnt=cJSON_GetArraySize(group_array);
if(group_cnt<=0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"compile rule %d have no groups.",compile_id);
return -1;
}
cJSON_ArrayForEach(group_rule, g_rules)
i=1;
cJSON_ArrayForEach(group_obj, group_array)
{
ret=write_group_rule(group_rule, compile_id, PARENT_TYPE_COMPILE, compile_id, p_iris, logger);
ret=write_group_rule(group_obj, PARENT_TYPE_COMPILE, compile_id, compile_id, i, p_iris, logger);
if(ret<0)
{
return -1;
@@ -1148,7 +1215,7 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
return 0;
}
// redis_write_ctx is used by maat_redis_tool to write json to redis.
int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger)
int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger)
{
cJSON *json=NULL, *tmp_obj=NULL;
int ret=-1;
@@ -1166,12 +1233,17 @@ int json2iris(const char* json_buff, const char* json_filename, const char*compi
compile_tn=tmp_obj->valuestring;
}
tmp_obj=cJSON_GetObjectItem(json, "group_table");
tmp_obj=cJSON_GetObjectItem(json, "group2compile_table");
if(tmp_obj)
{
group_tn=tmp_obj->valuestring;
group2compile_tn=tmp_obj->valuestring;
}
ret=set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, compile_tn, group_tn, redis_write_ctx, &iris_cfg, logger);
tmp_obj=cJSON_GetObjectItem(json, "group2group_table");
if(tmp_obj)
{
group2group_tn=tmp_obj->valuestring;
}
ret=set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, compile_tn, group2compile_tn, group2group_tn, redis_write_ctx, &iris_cfg, logger);
if(ret<0)
{
goto error_out;

4
src/inc_internal/Maat_garbage_collection.h
View File

@@ -1,8 +1,8 @@
#pragma once
struct Maat_garbage_bin{};
struct Maat_garbage_bin* Maat_garbage_bin_new(void);
struct Maat_garbage_bin* Maat_garbage_bin_new(int default_timeout);
void Maat_garbage_bin_free(struct Maat_garbage_bin* bin);
void Maat_garbage_bag(struct Maat_garbage_bin* bin, void* garbage, int timeout, void (* func)(void *));
void Maat_garbage_bagging(struct Maat_garbage_bin* bin, void* garbage, void (* func)(void *));
void Maat_garbage_collect_routine(struct Maat_garbage_bin* bin);

46
src/inc_internal/Maat_hierarchy.h
View File

@@ -1,6 +1,52 @@
#pragma once
#include "Maat_rule.h"
#include <sys/queue.h>
struct Maat_hierarchy;
struct Maat_hierarchy* Maat_hierarchy_new(int thread_num, void* mesa_handle_logger);
void Maat_hierarchy_free(struct Maat_hierarchy* hier);
void Maat_hierarchy_set_compile_user_data_free_func(struct Maat_hierarchy* hier, void (* func)(void *));
void Maat_hierarchy_set_region_user_data_free_func(struct Maat_hierarchy* hier, void (* func)(void *));
struct Maat_hierarchy_compile_mid;
struct Maat_hierarchy_compile_mid* Maat_hierarchy_compile_mid_new(struct Maat_hierarchy* hier, int thread_num);
void Maat_hierarchy_compile_mid_free(struct Maat_hierarchy_compile_mid* mid);
void Maat_hierarchy_compile_mid_udpate(struct Maat_hierarchy_compile_mid* mid, int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result);
void Maat_hierarchy_compile_user_data_iterate(struct Maat_hierarchy* hier, void (*callback)(void *user_data, void* apram), void* param);
int Maat_hierarchy_compile_mid_has_NOT_clause(struct Maat_hierarchy_compile_mid* mid);
struct Maat_hierarchy_hit_path
{
int Nth_hit_region;
struct Maat_hit_path_t path;
TAILQ_ENTRY(Maat_hierarchy_hit_path) entries;
};
void Maat_hit_path_init(struct Maat_hit_path_t* hit_path);
size_t Maat_hierarchy_hit_path_select0(const struct Maat_hierarchy_compile_mid* mid, struct Maat_hit_path_t* condition,
struct Maat_hierarchy_hit_path** hit_paths, size_t n_path);
int Maat_hierarchy_compile_add(struct Maat_hierarchy* hier, int compile_id, int declared_clause_num, void* user_data);
int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id);
void* Maat_hierarchy_compile_dettach_user_data(struct Maat_hierarchy* hier, int compile_id);
void* Maat_hierarchy_compile_read_user_data(struct Maat_hierarchy* hier, int compile_id);
void* Maat_hierarchy_region_dettach_user_data(struct Maat_hierarchy* hier, int region_id);
int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int Nth_clause, int compile_id);
int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int Nth_clause, int compile_id);
int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id);
int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id);
int Maat_hierarchy_add_region_to_group(struct Maat_hierarchy* hier, int group_id, int region_id, int table_id, void* user_data);
int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int group_id, int region_id);
int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier);
int Maat_hierarchy_region_compile(struct Maat_hierarchy_compile_mid* mid, int is_last_compile, void** user_data_array, size_t ud_array_sz);

95
src/inc_internal/Maat_rule_internal.h
View File

@@ -11,10 +11,8 @@
#include <MESA/field_stat2.h>
#include <MESA/rulescan.h>
#include "dynamic_array.h"
#include "bool_matcher.h"
#include "hiredis.h"
#include "IPMatcher.h"
#include "igraph/igraph.h"
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
#include "alignment_int64.h"
@@ -139,13 +137,6 @@ struct op_expr_t
int rule_type;
};
struct region_group_relation
{
int region_id;
int group_id;
int array_idx;
};
struct Maat_region_inner
{
int region_id;
@@ -159,51 +150,6 @@ struct Maat_region_inner
};
struct Maat_group_inner
{
int group_id;
int table_id;
int region_boundary;
int region_cnt;
int ref_by_parent_cnt;
int ref_by_children_cnt;
char* group_name;
int ref_by_compile_cnt;
int vertex_id;
size_t top_group_cnt;
long long* top_groups;
dynamic_array_t *regions;
pthread_mutex_t mutex;
};
struct _compile_result_t
{
int compile_id;
struct bool_expr group_set;
};
struct Maat_hit_path_inner
{
int Nth_hit_region;
struct Maat_hit_path_t path;
TAILQ_ENTRY(Maat_hit_path_inner) entries;
};
TAILQ_HEAD(hit_path_q, Maat_hit_path_inner);
struct scan_hit_status
{
size_t hit_path_cnt;
struct hit_path_q hit_path_qhead;
size_t all_hit_group_cnt;
size_t all_hit_group_array_sz;
unsigned long long *all_hit_group_array;
char not_grp_compile_hitted_flag;
};
struct _OUTER_scan_status_t
{
struct _Maat_feather_t* feather;
@@ -213,18 +159,8 @@ struct _OUTER_scan_status_t
int district_id;
int scan_cnt;
struct Maat_hierarchy_compile_mid* compile_mid;
struct scan_hit_status* inner;
};
enum maat_garbage_type
{
GARBAGE_SCANNER=0,
GARBAGE_GROUP_RULE,
GARBAGE_COMPILE_RULE,
GARBAGE_COMPILE_INNER,
GARBAGE_BOOL_MATCHER,
GARBAGE_MAP_STR2INT,
GARBAGE_IP_MATCHER
};
struct iconv_handle_t
{
int is_initialized;
@@ -356,6 +292,8 @@ struct _Maat_feather_t
char instance_name[MAX_TABLE_NAME_LEN];
char table_info_fn[MAX_TABLE_NAME_LEN];
char compile_tn[MAX_TABLE_NAME_LEN];
char group2compile_tn[MAX_TABLE_NAME_LEN];
char group2group_tn[MAX_TABLE_NAME_LEN];
char group_tn[MAX_TABLE_NAME_LEN];
pthread_mutex_t background_update_mutex;
char decrypt_key[MAX_TABLE_NAME_LEN];
@@ -386,7 +324,7 @@ struct _Maat_feather_t
int fs_status_id[MAX_MAAT_STAT_NUM];
int fs_column_id[MAX_MAAT_STAT_NUM];
mcore_long_t outer_mid_cnt;
mcore_long_t inner_mid_cnt;
mcore_long_t compile_mid_cnt;
mcore_long_t hit_cnt;
mcore_long_t thread_call_cnt;//size indicate by scan_thread_num,
mcore_long_t not_grp_hit_cnt;
@@ -401,25 +339,7 @@ struct _Maat_feather_t
long long cmd_acc_num;
long long line_cmd_acc_num;
};
struct _maat_garbage_t
{
enum maat_garbage_type type;
time_t create_time;
int expire_after;
int ok_times;
union
{
struct Maat_scanner* scanner;
struct Maat_group_inner* group_rule;
struct Maat_compile_rule* compile_rule;
struct Maat_compile_inner * compile_group_relation;
struct bool_matcher* bool_matcher;
struct ip_matcher* a_ip_matcher;
void * raw;
MESA_htable_handle str2int_map;
char* filename;
};
};
struct foreign_key
{
int column;
@@ -439,10 +359,7 @@ struct serial_rule_t //rm= Redis Maat
struct foreign_key* f_keys;
};
int parse_accept_tag(const char* value, struct rule_tag** result, void* logger);
void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(struct Maat_compile_inner* compile_rule, struct bool_expr* a_set, unsigned char *has_not);
void maat_start_cb(long long new_version,int update_type,void*u_para);
int maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para);

4
src/inc_internal/Maat_table.h
View File

@@ -154,7 +154,9 @@ int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
Maat_finish_callback_t *finish,//u_para
void* u_para);
int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id);
enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id);

2
src/inc_internal/json2iris.h
View File

@@ -1,6 +1,6 @@
#ifndef H_MAAT_JSON2IRIS_H_INCLUDE
#define H_MAAT_JSON2IRIS_H_INCLUDE
int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger);
int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger);
int set_file_rulenum(const char* path, int rulenum, void* logger);
#endif

2
src/version.map
View File

@@ -1,4 +1,4 @@
VERS_2.4{
VERS_3.0{
global:
extern "C++" {
*MAAT_FRAME_VERSION_*;

8
test/CMakeLists.txt
View File

@@ -1,5 +1,5 @@
add_executable(maat_demo maat_demo.cpp)
target_link_libraries(maat_demo maat_frame_shared)
#add_executable(maat_demo maat_demo.cpp)
#target_link_libraries(maat_demo maat_frame_shared)
add_executable(test_igraph test_igraph.cpp)
target_link_libraries(test_igraph igraph-static)
@@ -7,8 +7,8 @@ target_link_libraries(test_igraph igraph-static)
add_executable(test_maatframe test_maatframe.cpp)
target_link_libraries(test_maatframe maat_frame_shared gtest)
add_executable(perf_test_maatframe perf_test_maatframe.cpp)
target_link_libraries(perf_test_maatframe maat_frame_shared gtest)
#add_executable(perf_test_maatframe perf_test_maatframe.cpp)
#target_link_libraries(perf_test_maatframe maat_frame_shared gtest)
configure_file(table_info.conf table_info.conf COPYONLY)
configure_file(t2_tableinfo.conf t2_tableinfo.conf COPYONLY)

8
test/maat_demo.cpp
View File

@@ -678,8 +678,8 @@ int test_table_conjunction(Maat_feather_t feather,const char* table_name,const c
#define TEST_CMD_LINE_NUM 200*1000
void test_set_cmd_line(Maat_feather_t feather)
{
struct Maat_line_t **p_line=(struct Maat_line_t **)calloc(sizeof(struct Maat_line_t *), TEST_CMD_LINE_NUM);
struct Maat_line_t *line_rule=(struct Maat_line_t *)calloc(sizeof(struct Maat_line_t), TEST_CMD_LINE_NUM);
struct Maat_cmd_line **p_line=(struct Maat_cmd_line **)calloc(sizeof(struct Maat_cmd_line *), TEST_CMD_LINE_NUM);
struct Maat_cmd_line *line_rule=(struct Maat_cmd_line *)calloc(sizeof(struct Maat_cmd_line), TEST_CMD_LINE_NUM);
int i=0;
const char* line="1\t192.168.0.1\t4444444444\t1";
memset(&line_rule,0,sizeof(line_rule));
@@ -695,14 +695,14 @@ void test_set_cmd_line(Maat_feather_t feather)
}
Maat_cmd_set_lines(feather,(const struct Maat_line_t **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
Maat_cmd_set_lines(feather,(const struct Maat_cmd_line **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
usleep(WAIT_FOR_EFFECTIVE_US);
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
line_rule[i].table_line=NULL;
}
Maat_cmd_set_lines(feather, (const struct Maat_line_t **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
Maat_cmd_set_lines(feather, (const struct Maat_cmd_line **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
free(p_line);
free(line_rule);

8
test/perf_test_maatframe.cpp
View File

@@ -157,8 +157,8 @@ TEST_F(MaatCMDPerfTest, SetLinesOneMillion)
const int TEST_CMD_LINE_NUM=1000*1000;
Maat_feather_t feather=MaatCMDPerfTest::_shared_feather;
struct Maat_line_t **p_line=(struct Maat_line_t **)calloc(sizeof(struct Maat_line_t *), TEST_CMD_LINE_NUM);
struct Maat_line_t *line_rule=(struct Maat_line_t *)calloc(sizeof(struct Maat_line_t), TEST_CMD_LINE_NUM);
struct Maat_cmd_line **p_line=(struct Maat_cmd_line **)calloc(sizeof(struct Maat_cmd_line *), TEST_CMD_LINE_NUM);
struct Maat_cmd_line *line_rule=(struct Maat_cmd_line *)calloc(sizeof(struct Maat_cmd_line), TEST_CMD_LINE_NUM);
int i=0;
const char* line="1\t192.168.0.1\t4444444444\t1";
int seq=(int)Maat_cmd_incrby(feather,"TEST_PLUG_SEQ", TEST_CMD_LINE_NUM);
@@ -176,7 +176,7 @@ TEST_F(MaatCMDPerfTest, SetLinesOneMillion)
long long version_before=0;
Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
Maat_cmd_set_lines(feather,(const struct Maat_line_t **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
Maat_cmd_set_lines(feather,(const struct Maat_cmd_line **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
wait_for_cmd_effective_copy(feather, version_before);
@@ -184,7 +184,7 @@ TEST_F(MaatCMDPerfTest, SetLinesOneMillion)
{
line_rule[i].table_line=NULL;
}
Maat_cmd_set_lines(feather, (const struct Maat_line_t **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
Maat_cmd_set_lines(feather, (const struct Maat_cmd_line **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
free(p_line);
free(line_rule);

272
test/test_maatframe.cpp
View File

@@ -2029,20 +2029,18 @@ void* MaatCmdTest::logger;
int MaatCmdTest::linger_timeout;
int test_add_expr_command(Maat_feather_t feather,const char* region_table,int config_id, int timeout,int label_id, const char* keywords)
{
struct Maat_cmd_t* cmd=NULL;
struct Maat_rule_t rule;
char huge_serv_def[1024*2];
memset(huge_serv_def,'s',sizeof(huge_serv_def));
struct Maat_region_t region;
int group_num=1,ret=0;
memset(&rule,0,sizeof(rule));
rule.config_id=config_id;
strcpy(rule.service_defined,"maat_command");
//MUST acqire by function, because Maat_cmd_t has some hidden members.
cmd=Maat_create_cmd(&rule, group_num);
cmd->expire_after=timeout;
cmd->label_id=label_id;
memset(huge_serv_def,'s',sizeof(huge_serv_def)-1);
struct Maat_rule_t compile;
memset(&compile,0,sizeof(compile));
compile.config_id=config_id;
strcpy(compile.service_defined,"maat_command");
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile, "COMPILE", huge_serv_def, 1, label_id, timeout);
struct Maat_cmd_region region;
memset(&region,0,sizeof(region));
region.region_id=Maat_cmd_get_new_region_id(feather);
region.region_type=REGION_EXPR;
region.table_name=region_table;
region.expr_rule.district=NULL;
@@ -2050,58 +2048,56 @@ int test_add_expr_command(Maat_feather_t feather,const char* region_table,int co
region.expr_rule.expr_type=EXPR_TYPE_AND;
region.expr_rule.match_method=MATCH_METHOD_SUB;
region.expr_rule.hex_bin=UNCASE_PLAIN;
Maat_cmd_set_opt(cmd, MAAT_RULE_SERV_DEFINE, huge_serv_def, sizeof(huge_serv_def));
Maat_add_region2cmd(cmd, 0, &region);
//use pipeline model.
ret=Maat_cmd_append(feather, cmd, MAAT_OP_ADD);
if(ret<0)
{
printf("Add Maat command %d failed.\n",rule.config_id);
Maat_free_cmd(cmd);
return 0;
}
//cmd has been saved in feather, so free cmd before commit is allowed.
Maat_free_cmd(cmd);
struct Maat_cmd_group2compile g2c;
memset(&g2c, 0, sizeof(g2c));
g2c.group_id=Maat_cmd_get_new_group_id(feather);
g2c.table_name="GROUP";
g2c.compile_id=config_id;
g2c.Nth_clause=1;
Maat_command_raw_set_group2compile(feather, MAAT_OP_ADD, &g2c);
Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region, g2c.group_id);
return 0;
}
int del_command(Maat_feather_t feather, int config_id)
{
struct Maat_cmd_t* cmd=NULL;
struct Maat_rule_t rule;
int ret=0;
memset(&rule,0,sizeof(rule));
rule.config_id=config_id;
cmd=Maat_create_cmd(&rule, 0);
ret=Maat_cmd(feather, cmd, MAAT_OP_DEL);
if(ret<0)
{
printf("Delete Maat command %d failed.\n",rule.config_id);
}
Maat_free_cmd(cmd);
return 0;
struct Maat_rule_t compile;
memset(&compile,0,sizeof(compile));
compile.config_id=config_id;
ret=Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile, "COMPILE", NULL, 1, 0, 0);
return ret;
}
TEST_F(MaatCmdTest, SetIP)
{
struct Maat_cmd_t* cmd=NULL;
struct Maat_rule_t rule;
struct Maat_rule_t compile;
int config_id=0, timeout=4;
long long version_before=0;
const char* region_table="IP_CONFIG";
struct Maat_region_t region;
int group_num=1,ret=0;
memset(&rule,0,sizeof(rule));
int ret=0;
memset(&compile, 0, sizeof(compile));
Maat_feather_t feather=MaatCmdTest::_shared_feather;
//MUST acquire by Maat_cmd_incrby to guarantee a unique compile ID.
config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
rule.config_id=config_id;
compile.config_id=config_id;
strcpy(rule.service_defined,"maat_command");
strcpy(compile.service_defined, "maat_command");
//MUST acqire by function, because Maat_cmd_t has some hidden members.
cmd=Maat_create_cmd(&rule, group_num);
cmd->expire_after=timeout;
cmd->label_id=0; //no lable
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile, "COMPILE", NULL, 1, 0, timeout);
struct Maat_cmd_group2compile g2c;
memset(&g2c, 0, sizeof(g2c));
g2c.group_id=Maat_cmd_get_new_group_id(feather);
g2c.compile_id=compile.config_id;
g2c.Nth_clause=1;
Maat_command_raw_set_group2compile(feather, MAAT_OP_ADD, &g2c);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
EXPECT_EQ(ret, 0);
struct Maat_cmd_region region;
memset(&region, 0, sizeof(region));
region.region_type=REGION_IP;
region.table_name=region_table;
@@ -2117,16 +2113,11 @@ TEST_F(MaatCmdTest, SetIP)
region.ip_rule.dst_port=80;
region.ip_rule.mask_dst_port=65535;
region.ip_rule.protocol=0;//means any protocol should hit.
Maat_add_region2cmd(cmd, 0, &region);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
EXPECT_EQ(ret, 0);
ret=Maat_cmd(feather, cmd, MAAT_OP_ADD);
ret=Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region, g2c.group_id);
EXPECT_GE(ret, 0);
Maat_free_cmd(cmd);
cmd=NULL;
wait_for_cmd_effective(feather, version_before);
struct ipaddr ipv4_addr;
struct stream_tuple4_v4 v4_addr;
ipv4_addr.addrtype=ADDR_TYPE_IPV4;
@@ -2151,18 +2142,9 @@ TEST_F(MaatCmdTest, SetIP)
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
//reset timeout.
cmd=Maat_create_cmd(&rule, 0);
cmd->expire_after=timeout+5;
ret=Maat_cmd(feather, cmd, MAAT_OP_RENEW_TIMEOUT);
EXPECT_EQ(ret ,1);
Maat_command_raw_set_compile(feather, MAAT_OP_RENEW_TIMEOUT, &compile, "COMPILE", NULL, 1, 0, timeout+5);
// RENEW_TIMEOUT doesn't change MAAT_VERSION, thus we cannot use wait_for_cmd_effective.
// wait_for_cmd_effective(feather, version_before);
Maat_free_cmd(cmd);
cmd=NULL;
sleep(timeout+1);
sleep(timeout+5);
ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,&result,1, &mid,0);
EXPECT_EQ(ret, 1);
@@ -2200,8 +2182,6 @@ TEST_F(MaatCmdTest, SetExpr)
test_add_expr_command(feather,table_name,config_id-1, 0, label_id, keywords);
test_add_expr_command(feather,table_name,config_id, 0, label_id, keywords);
ret=Maat_cmd_commit(feather);
EXPECT_TRUE(ret>=0);
usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_after, sizeof(version_after));
@@ -2240,8 +2220,6 @@ TEST_F(MaatCmdTest, SetExpr)
timeout=1;
config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
test_add_expr_command(feather,table_name,config_id, timeout, label_id, keywords);
ret=Maat_cmd_commit(feather);
EXPECT_TRUE(ret>=0);
usleep(timeout*1000*1000+WAIT_FOR_EFFECTIVE_US);
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
@@ -2268,8 +2246,7 @@ TEST_F(MaatCmdTest, RuleIDRecycle)
int label_id=5211, ret=0;
test_add_expr_command(feather,table_name,rule_id, 0, label_id, keywords);
ret=Maat_cmd_commit(feather);
EXPECT_TRUE(ret>=0);
usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
&result, NULL, 1,
@@ -2287,8 +2264,6 @@ TEST_F(MaatCmdTest, RuleIDRecycle)
EXPECT_EQ(ret, 0);
test_add_expr_command(feather,table_name,rule_id, 0, label_id, keywords);
ret=Maat_cmd_commit(feather);
EXPECT_TRUE(ret>=0);
usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
memset(&result, 0, sizeof(result));
@@ -2325,8 +2300,6 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder)
expect_ruleid[i]=rule_id+1-repeat_times+i;
test_add_expr_command(feather,table_name,rule_id+1-repeat_times+i, 0, label_id, keywords);
}
ret=Maat_cmd_commit(feather);
EXPECT_TRUE(ret>=0);
usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
memset(&result, 0, sizeof(result));
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
@@ -2358,27 +2331,31 @@ TEST_F(MaatCmdTest, SubGroup)
int table_id=0;
table_id=Maat_table_register(feather, table_name);
struct Maat_rule_t compile1, compile2;
struct Maat_group_t group1, group2, group3;
struct Maat_region_t region1,region2;
// struct Maat_group_t group1, group2, group3;
struct Maat_cmd_group2compile group1, group2_g2c;
struct Maat_cmd_group2group group2, group3;
struct Maat_cmd_region region1, region2;
memset(&compile1, 0, sizeof(compile1));
compile1.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1);
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1, 0, 0);
memset(&compile2, 0, sizeof(compile2));
compile2.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile2, compile_table_name, NULL, 1);
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile2, compile_table_name, NULL, 1, 0, 0);
//group1->compile1
memset(&group1, 0, sizeof(group1));
group1.group_id=Maat_cmd_get_new_group_id(feather);
group1.table_name=group_table_name;
group1.parent_id=compile1.config_id;
group1.parent_type=PARENT_TYPE_COMPILE;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
group1.compile_id=compile1.config_id;
group1.Nth_clause=1;
group1.not_flag=0;
group1.virtual_table_name="null";
Maat_command_raw_set_group2compile(feather, MAAT_OP_ADD, &group1);
//group1->compile2
group1.parent_id=compile2.config_id;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
group1.compile_id=compile2.config_id;
Maat_command_raw_set_group2compile(feather, MAAT_OP_ADD, &group1);
/*group2->group1->compile1
\
@@ -2387,9 +2364,10 @@ TEST_F(MaatCmdTest, SubGroup)
memset(&group2, 0, sizeof(group2));
group2.group_id=Maat_cmd_get_new_group_id(feather);
group2.table_name=group_table_name;
group2.parent_id=group1.group_id;
group2.parent_type=PARENT_TYPE_GROUP;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2);
group2.superior_group_id=group1.group_id;
Maat_command_raw_set_group2group(feather, MAAT_OP_ADD, &group2);
/*region1->group2->group1->compile1
\
@@ -2421,7 +2399,7 @@ TEST_F(MaatCmdTest, SubGroup)
\
\--X--compile2
*/
Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1);
Maat_command_raw_set_group2compile(feather, MAAT_OP_DEL, &group1);
sleep(1);
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1),
result, NULL, 4,
@@ -2435,12 +2413,16 @@ TEST_F(MaatCmdTest, SubGroup)
\
\->compile2
*/
group1.parent_id=compile1.config_id;
Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1);
Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile1, compile_table_name, NULL, 1);
group2.parent_type=PARENT_TYPE_COMPILE;
group2.parent_id=compile2.config_id;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2);
group1.compile_id=compile1.config_id;
Maat_command_raw_set_group2compile(feather, MAAT_OP_DEL, &group1);
Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile1, compile_table_name, NULL, 1, 0, 0);
memset(&group2_g2c, 0, sizeof(group2_g2c));
group2_g2c.group_id=group2.group_id;
group2_g2c.table_name=group2.table_name;
group2_g2c.compile_id=compile2.config_id;
Maat_command_raw_set_group2compile(feather, MAAT_OP_ADD, &group2_g2c);
sleep(1);
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1),
@@ -2454,15 +2436,15 @@ TEST_F(MaatCmdTest, SubGroup)
/*region1->group2->group1--X--
\
\->compile2
region2->group3
*/
memset(&group3, 0, sizeof(group3));
group3.group_id=Maat_cmd_get_new_group_id(feather);
group3.parent_id=group1.group_id;
group3.parent_type=PARENT_TYPE_GROUP;
group3.superior_group_id=group1.group_id;
group3.table_name=group_table_name;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group3);
Maat_command_raw_set_group2group(feather, MAAT_OP_ADD, &group3);
memset(&region2, 0, sizeof(region2));
region2.region_id=Maat_cmd_get_new_region_id(feather);
@@ -2484,13 +2466,13 @@ TEST_F(MaatCmdTest, SubGroup)
\->compile2
region2->group3
*/
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1);
group1.parent_id=compile1.config_id;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1, 0, 0);
group1.compile_id=compile1.config_id;
Maat_command_raw_set_group2compile(feather, MAAT_OP_ADD, &group1);
Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile1, compile_table_name, NULL, 1);
group1.parent_id=compile1.config_id;
Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1);
Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile1, compile_table_name, NULL, 1, 0, 0);
group1.compile_id=compile1.config_id;
Maat_command_raw_set_group2compile(feather, MAAT_OP_DEL, &group1);
sleep(1);
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1),
@@ -2503,6 +2485,7 @@ TEST_F(MaatCmdTest, SubGroup)
}
#define MaatCmdTest_RefGroup 0
/*
TEST_F(MaatCmdTest, RefGroup)
{
@@ -2536,8 +2519,8 @@ TEST_F(MaatCmdTest, RefGroup)
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
/*region1->group1->compile1
*/
//region1->group1->compile1
memset(&region1, 0, sizeof(region1));
region1.region_id=Maat_cmd_get_new_region_id(feather);
region1.region_type=REGION_EXPR;
@@ -2618,8 +2601,7 @@ TEST_F(MaatCmdTest, VirtualTable)
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
/*region1->group1->compile1
*/
//region1->group1->compile1
memset(&region1, 0, sizeof(region1));
region1.region_id=Maat_cmd_get_new_region_id(feather);
region1.region_type=REGION_EXPR;
@@ -2708,12 +2690,12 @@ TEST_F(MaatCmdTest, VirtualTable)
return;
}
*/
TEST_F(MaatCmdTest, SetLines)
{
const int TEST_CMD_LINE_NUM=4;
const struct Maat_line_t *p_line[TEST_CMD_LINE_NUM];
struct Maat_line_t line_rule[TEST_CMD_LINE_NUM];
const struct Maat_cmd_line *p_line[TEST_CMD_LINE_NUM];
struct Maat_cmd_line line_rule[TEST_CMD_LINE_NUM];
char table_line[TEST_CMD_LINE_NUM][128];
int ret=0,i=0;
Maat_feather_t feather=MaatCmdTest::_shared_feather;
@@ -2761,7 +2743,7 @@ TEST_F(MaatCmdTest, PauseUpdate)
{
Maat_feather_t feather=MaatCmdTest::_shared_feather;
int value=0, ret=0, table_id=0;
struct Maat_line_t line_rule;
struct Maat_cmd_line line_rule;
char* line=NULL;
const char* table_name="QD_ENTRY_INFO";
table_id=Maat_table_register(feather, table_name);
@@ -2900,7 +2882,7 @@ TEST_F(MaatCmdTest, SetFile)
char line[1024];
int tag=0;
struct Maat_line_t line_rule;
struct Maat_cmd_line line_rule;
line_rule.label_id=0;
line_rule.rule_id=(int)Maat_cmd_incrby(feather,"TEST_PLUG_SEQ", 1);
line_rule.table_name=table_name;
@@ -2977,8 +2959,8 @@ TEST_F(MaatCmdTest, PluginEXData)
const int TEST_CMD_LINE_NUM=4;
const struct Maat_line_t *p_line[TEST_CMD_LINE_NUM];
struct Maat_line_t line_rule[TEST_CMD_LINE_NUM];
const struct Maat_cmd_line *p_line[TEST_CMD_LINE_NUM];
struct Maat_cmd_line line_rule[TEST_CMD_LINE_NUM];
const char* table_line[TEST_CMD_LINE_NUM]={"1\t192.168.0.1\tmahuateng\t1\t0",
"2\t192.168.0.2\tliuqiangdong\t1\t0",
"3\t192.168.0.3\tmayun\t1\t0",
@@ -3035,12 +3017,13 @@ TEST_F(MaatCmdTest, HitPath)
const char* table_keywords="KEYWORDS_TABLE";
struct Maat_rule_t compile1;
struct Maat_group_t group1, group2, group21, group3, group4;
struct Maat_region_t region1, region2, region3, region4;
struct Maat_cmd_group2compile group1, group21, group3, group4;
struct Maat_cmd_group2group group2;
struct Maat_cmd_region region1, region2, region3, region4;
memset(&compile1, 0, sizeof(compile1));
compile1.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 2);
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 2, 0, 0);
//group1->compile1
@@ -3048,13 +3031,12 @@ TEST_F(MaatCmdTest, HitPath)
group1.group_id=Maat_cmd_get_new_group_id(feather);
group1.table_name=group_table_name;
group1.virtual_table_name="HTTP_REQUEST_HEADER";
group1.parent_id=compile1.config_id;
group1.parent_type=PARENT_TYPE_COMPILE;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
group1.compile_id=compile1.config_id;
Maat_command_raw_set_group2compile(feather, MAAT_OP_ADD, &group1);
/*region1->group1->compile1
*/
//region1->group1->compile1
memset(&region1, 0, sizeof(region1));
region1.region_id=Maat_cmd_get_new_region_id(feather);
region1.region_type=REGION_EXPR;
@@ -3064,38 +3046,34 @@ TEST_F(MaatCmdTest, HitPath)
region1.expr_rule.expr_type=EXPR_TYPE_STRING;
Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region1, group1.group_id);
/*
region1->group1->compile1
/
group21/
*/
// region1->group1->compile1
// /
// group21/
memset(&group21, 0, sizeof(group21));
group21.group_id=Maat_cmd_get_new_group_id(feather);
group21.table_name=group_table_name;
group21.virtual_table_name="HTTP_RESPONSE_HEADER";
group21.parent_id=compile1.config_id;
group21.parent_type=PARENT_TYPE_COMPILE;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group21);
group21.compile_id=compile1.config_id;
Maat_command_raw_set_group2compile(feather, MAAT_OP_ADD, &group21);
/*
region1->group1->compile1
/
group2->group21/
*/
// region1->group1->compile1
// /
// group2->group21/
memset(&group2, 0, sizeof(group2));
group2.group_id=Maat_cmd_get_new_group_id(feather);
group2.table_name=group_table_name;
group2.parent_id=group21.group_id;
group2.parent_type=PARENT_TYPE_GROUP;
Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2);
group2.superior_group_id=group21.group_id;
Maat_command_raw_set_group2group(feather, MAAT_OP_ADD, &group2);
/*
region1->group1->compile1
/
region2->group2->group21/
*/
// region1->group1->compile1
// /
//region2->group2->group21/
memset(&region2, 0, sizeof(region2));
region2.region_id=Maat_cmd_get_new_region_id(feather);
@@ -3179,7 +3157,7 @@ TEST_F(MaatCmdTest, HitPath)
EXPECT_EQ(hit_path[path_idx].region_id, region1.region_id);
EXPECT_EQ(hit_path[path_idx].sub_group_id, group1.group_id);
EXPECT_EQ(hit_path[path_idx].top_group_id, group1.group_id);
EXPECT_EQ(hit_path[path_idx].vt_id, table_id);
EXPECT_EQ(hit_path[path_idx].virtual_table_id, table_id);
EXPECT_EQ(hit_path[path_idx].compile_id, -1);
@@ -3211,7 +3189,7 @@ TEST_F(MaatCmdTest, HitPath)
EXPECT_EQ(hit_path[path_idx].region_id, region2.region_id);
EXPECT_EQ(hit_path[path_idx].sub_group_id, group2.group_id);
EXPECT_EQ(hit_path[path_idx].top_group_id, group21.group_id);
EXPECT_EQ(hit_path[path_idx].vt_id, table_id);
EXPECT_EQ(hit_path[path_idx].virtual_table_id, table_id);
EXPECT_EQ(hit_path[path_idx].compile_id, compile1.config_id);
const char* keywords1="In graph theory, a path in a graph is a finite or infinite \
@@ -3235,7 +3213,7 @@ that the edges be all directed in the same direction.";
EXPECT_EQ(hit_path[path_idx].region_id, region4.region_id);
EXPECT_EQ(hit_path[path_idx].sub_group_id, group4.group_id);
EXPECT_EQ(hit_path[path_idx].top_group_id, group4.group_id);
EXPECT_EQ(hit_path[path_idx].vt_id, 0);
EXPECT_EQ(hit_path[path_idx].virtual_table_id, 0);
EXPECT_EQ(hit_path[path_idx].compile_id, -1);
table_id=Maat_table_register(feather, table_ip);
@@ -3253,7 +3231,7 @@ that the edges be all directed in the same direction.";
EXPECT_EQ(hit_path[path_idx].region_id, region3.region_id);
EXPECT_EQ(hit_path[path_idx].sub_group_id, group3.group_id);
EXPECT_EQ(hit_path[path_idx].top_group_id, group3.group_id);
EXPECT_EQ(hit_path[path_idx].vt_id, 0);
EXPECT_EQ(hit_path[path_idx].virtual_table_id, 0);
EXPECT_EQ(hit_path[path_idx].compile_id, -1);
Nth_scan++;
@@ -3267,7 +3245,7 @@ that the edges be all directed in the same direction.";
EXPECT_EQ(hit_path[path_idx].region_id, region4.region_id);
EXPECT_EQ(hit_path[path_idx].sub_group_id, group4.group_id);
EXPECT_EQ(hit_path[path_idx].top_group_id, group4.group_id);
EXPECT_EQ(hit_path[path_idx].vt_id, 0);
EXPECT_EQ(hit_path[path_idx].virtual_table_id, 0);
EXPECT_EQ(hit_path[path_idx].compile_id, -1);
Maat_stream_scan_string_end(&stream_para);

2
tools/maat_redis_tool.cpp
View File

@@ -332,7 +332,7 @@ int main(int argc, char * argv[])
{
printf("open %s failed.\n", json_file);
}
ret=json2iris(json_buff, json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, NULL);
ret=json2iris(json_buff, json_file, NULL, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, NULL);
if(ret<0)
{
printf("Invalid json format.\n");