gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

通过Maat_rule_get_ex_new_index为Maat_rule_t添加自定义的数据结构。

Browse Source
This commit is contained in:
zhengchao
2018-11-07 17:12:06 +08:00
parent 9fa9da75ae
commit 05ff016043
4 changed files with 254 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
inc/Maat_rule.h
View File

@@ -1,12 +1,12 @@
/*
*****************Maat Network Flow Rule Manage Framework********
*****************Maat Deep Packet Inspection Policy Framework********
* Maat is the Goddess of truth and justice in ancient Egyptian concept.
* Her feather was the measure that determined whether the souls (considered
* to reside in the heart) of the departed would reach the paradise of afterlife
* successfully.
* Author: zhengchao@iie.ac.cn,MESA
* Version 2018-09-25 foreign key and rule tags.
* Author: zhengchao@iie.ac.cn, MESA
* Version 2018-11-06 Maat Rule Extra Data.
* NOTE: MUST compile with G++
* All right reserved by Institute of Infomation Engineering,Chinese Academic of Science 2014~2018
*********************************************************
@@ -181,6 +181,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
Maat_finish_callback_t *finish,//u_para
void* u_para);
enum MAAT_SCAN_OPT
{
MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*,SIZE= strlen(string).DEFAULT: no default.
@@ -238,10 +239,31 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
,scan_status_t* mid,int thread_num);
void Maat_clean_status(scan_status_t* mid);
struct Maat_rule_ex_t
{
};
typedef void* MAAT_RULE_EX_DATA;
// The idx parameter is the index: this will be the same value returned by Maat_rule_get_ex_new_index() when the functions were initially registered.
// Finally the argl and argp parameters are the values originally passed to the same corresponding parameters when Maat_rule_get_ex_new_index() was called.
typedef void Maat_rule_EX_new_func_t(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp);
typedef void Maat_rule_EX_free_func_t(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp);
int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name,
Maat_rule_EX_new_func_t *new_func,
Maat_rule_EX_free_func_t* free_func,
long argl, void *argp);
MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx);
enum MAAT_RULE_OPT
{
MAAT_RULE_SERV_DEFINE //VALUE is a char* buffer,SIZE= buffer size.
};
int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum MAAT_RULE_OPT type, void* value, int size);
#endif // H_MAAT_RULE_H_INCLUDE

101
src/entry/Maat_api.cpp
View File

@@ -99,6 +99,13 @@ int pickup_hit_region_from_compile(universal_bool_expr_t *compile_hit,const unsi
}
return k;
}
void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len)
{
memcpy(rule, rule_head, sizeof(struct _head_Maat_rule_t));
memcpy(rule->service_defined, srv_def, MIN(srv_def_len,MAX_SERVICE_DEFINE_LEN));
return;
}
int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num)
{
@@ -155,26 +162,26 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
for(i=0;i<scan_ret&&result_cnt<size;i++)
{
_mi_rule=array_mi_rule[i];
if(_mi_rule==NULL||_mi_rule->db_c_rule==NULL)
if(_mi_rule==NULL)
{
continue;
}
if(0==pthread_rwlock_tryrdlock(&(_mi_rule->rwlock)))
{
if(_mi_rule->is_valid==1)
{
make_group_set(_mi_rule,&(rs_result[result_cnt].group_set));
r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set),_mid->cur_hit_id,_mid->cur_hit_cnt,
region_pos, MAX_SCANNER_HIT_NUM);
if(r_in_c_cnt>0)//compile config hitted becasue of new reigon
{
memcpy(&(result[result_cnt]),&(_mi_rule->db_c_rule->m_rule_head),sizeof(struct _head_Maat_rule_t));
memcpy(result[result_cnt].service_defined
,_mi_rule->db_c_rule->service_defined
,MIN(_mi_rule->db_c_rule->m_rule_head.serv_def_len,MAX_SERVICE_DEFINE_LEN));
fill_maat_rule(&(result[result_cnt]), &(_mi_rule->db_c_rule->m_rule_head),
_mi_rule->db_c_rule->service_defined ,_mi_rule->db_c_rule->m_rule_head.serv_def_len);
rs_result[result_cnt].compile_id=_mi_rule->compile_id;
result_cnt++;
}
}
pthread_rwlock_unlock(&(_mi_rule->rwlock));
}
}
@@ -880,7 +887,7 @@ error_out:
void Maat_burn_feather(Maat_feather_t feather)
{
_Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
_feather->still_working=0;//destroy will proceed in thread_rule_monitor
return;
}
@@ -888,7 +895,7 @@ int Maat_table_register(Maat_feather_t feather,const char* table_name)
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
int table_id=-1,ret=0;
ret=map_str2int(_feather->map_tablename2id, table_name,&table_id);
ret=map_str2int(_feather->map_tablename2id, table_name, &table_id);
if(ret>0)
{
return table_id;
@@ -906,7 +913,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
int idx=0,i=0;
_Maat_table_info_t *p_table=_feather->p_table_info[table_id];
struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id];
const char* lines=NULL;
if(p_table==NULL)
{
@@ -952,6 +959,78 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return 1;
}
void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user)
{
struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user;
struct _Maat_compile_inner_t *compile_inner=(struct _Maat_compile_inner_t *)data;
MAAT_RULE_EX_DATA ad=NULL;
if(compile_inner->ref_table->table_id!=ex_desc->table_id || compile_inner->db_c_rule==NULL)
{
return;
}
pthread_rwlock_wrlock(&(compile_inner->rwlock));
ad=rule_ex_data_new(&(compile_inner->db_c_rule->m_rule_head),
compile_inner->db_c_rule->service_defined,
ex_desc);
compile_inner->ads[ex_desc->idx]=ad;
pthread_rwlock_unlock(&(compile_inner->rwlock));
return;
}
int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name,
Maat_rule_EX_new_func_t *new_func,
Maat_rule_EX_free_func_t* free_func,
long argl, void *argp)
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
int table_id=-1,ret=0, idx=-1;
ret=map_str2int(_feather->map_tablename2id, compile_table_name, &table_id);
if(ret<0)
{
return -1;
}
struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id];
if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL)
{
return -1;
}
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
if(p_table->ex_data_num==MAX_COMPILE_EX_DATA_NUM)
{
ret=-1;
goto failed;
}
idx=p_table->ex_data_num;
p_table->ex_desc[idx].idx=idx;
p_table->ex_desc[idx].table_id=table_id;
p_table->ex_desc[idx].argl=argl;
p_table->ex_desc[idx].argp=argp;
p_table->ex_desc[idx].new_func=new_func;
p_table->ex_desc[idx].free_func=free_func;
p_table->ex_data_num++;
MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, p_table->ex_desc+idx);
failed:
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return idx;
}
MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx)
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct _Maat_compile_inner_t *compile_inner=NULL;
MAAT_RULE_EX_DATA ad=NULL;
compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(compile_inner==NULL)
{
return NULL;
}
pthread_rwlock_rdlock(&(compile_inner->rwlock));
assert(idx<compile_inner->ref_table->ex_data_num);
ad=compile_inner->ads[idx];
pthread_rwlock_unlock(&(compile_inner->rwlock));
return ad;
}
int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
,enum MAAT_CHARSET charset,const char* data,int data_len
,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num
@@ -1781,17 +1860,17 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum
{
case MAAT_RULE_SERV_DEFINE:
compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
pthread_rwlock_rdlock(&(compile_inner->rwlock));
if(compile_inner==NULL)
{
ret=0;
}
else
{
pthread_rwlock_rdlock(&(compile_inner->rwlock));
ret=MIN(size,compile_inner->db_c_rule->m_rule_head.serv_def_len);
memcpy(value,compile_inner->db_c_rule->service_defined,ret);
}
pthread_rwlock_unlock(&(compile_inner->rwlock));
}
break;
default:
ret=-1;

82
src/entry/Maat_rule.cpp
View File

@@ -504,6 +504,23 @@ int HASH_delete_by_id(MESA_htable_handle hash,int id)
ret=MESA_htable_del(hash,(unsigned char*)&id, sizeof(id), NULL);
return ret;
}
MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc)
{
MAAT_RULE_EX_DATA ad=NULL;
struct Maat_rule_t rule;
fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1);
ex_desc->new_func(ex_desc->idx, &rule, srv_def, &ad, ex_desc->argl,ex_desc->argp);
return ad;
}
void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc)
{
struct Maat_rule_t rule;
fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1);
ex_desc->free_func(ex_desc->idx, &rule, srv_def, ad, ex_desc->argl,ex_desc->argp);
return;
}
_Maat_table_info_t* create_table_info(int max_thread_num)
{
struct _Maat_table_info_t*p=NULL;
@@ -832,7 +849,7 @@ struct _Maat_group_inner_t* create_group_rule(int group_id)
pthread_mutex_init(&(group->mutex), NULL);
return group;
}
void force_destroy_group_rule(struct _Maat_group_inner_t* group)
void _destroy_group_rule(struct _Maat_group_inner_t* group)
{
dynamic_array_destroy(group->regions,free);
group->region_cnt=0;
@@ -854,7 +871,7 @@ void destroy_group_rule(struct _Maat_group_inner_t* group)
{
return;
}
force_destroy_group_rule(group);
_destroy_group_rule(group);
}
void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set)
{
@@ -963,22 +980,39 @@ struct _Maat_compile_inner_t * create_compile_rule(int compile_id)
p=(struct _Maat_compile_inner_t*)calloc(sizeof(struct _Maat_compile_inner_t),1);
p->compile_id=compile_id;
p->group_cnt=0;
p->table_id=0;
p->group_boundary=1;
p->groups=dynamic_array_create(1, 1);
p->ads=ALLOC(MAAT_RULE_EX_DATA, MAX_COMPILE_EX_DATA_NUM);
pthread_rwlock_init(&(p->rwlock), NULL);
return p;
}
void force_destroy_compile_rule(struct _Maat_compile_inner_t * p)
void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule)
{
p->compile_id=-1;
dynamic_array_destroy(p->groups,NULL);
if(p->db_c_rule!=NULL)
const struct _Maat_table_info_t* table=compile_rule->ref_table;
struct db_compile_rule_t* db_compile_rule=compile_rule->db_c_rule;
int i=0;
compile_rule->compile_id=-1;
dynamic_array_destroy(compile_rule->groups,NULL);
pthread_rwlock_wrlock(&(compile_rule->rwlock));
if(db_compile_rule!=NULL)
{
free(p->db_c_rule);
free(db_compile_rule->service_defined);
free(db_compile_rule);
compile_rule->db_c_rule=NULL;
}
pthread_rwlock_destroy(&(p->rwlock));
free(p);
for(i=0; i<table->ex_data_num; i++)
{
rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, table->ex_desc+i);
compile_rule->ads[i]=NULL;
}
free(compile_rule->ads);
pthread_rwlock_unlock(&(compile_rule->rwlock));
pthread_rwlock_destroy(&(compile_rule->rwlock));
free(compile_rule);
}
void destroy_compile_rule(struct _Maat_compile_inner_t * p)
{
@@ -990,7 +1024,7 @@ void destroy_compile_rule(struct _Maat_compile_inner_t * p)
p_group=(struct _Maat_compile_inner_t*)dynamic_array_read(p->groups,i);
assert(p_group==NULL);
}
force_destroy_compile_rule(p);
_destroy_compile_rule(p);
}
scan_rule_t* create_rs_str_rule(unsigned int sub_type,enum MAAT_MATCH_METHOD match_method,int is_case_sensitive,const char* string,int len,int l_offset,int r_offset)
{
@@ -1265,8 +1299,8 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner)
return;
}
rulescan_destroy(scanner->region);
MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))force_destroy_compile_rule);
MESA_htable_destroy(scanner->group_hash, (void (*)(void*))force_destroy_group_rule);
MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))_destroy_compile_rule);
MESA_htable_destroy(scanner->group_hash, (void (*)(void*))_destroy_group_rule);
MESA_htable_destroy(scanner->region_hash, NULL);
map_destroy(scanner->district_map);
scanner->district_map=NULL;
@@ -2254,18 +2288,31 @@ int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t*
{
struct _Maat_compile_inner_t *compile_rule=NULL;
struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head);
int i=0;
compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id);
if(compile_rule==NULL)
{
compile_rule=create_compile_rule(p_maat_rule_head->config_id);
compile_rule->table_id=table->table_id;
HASH_add_by_id(scanner->compile_hash,p_maat_rule_head->config_id,compile_rule);
}
else
{
if(compile_rule->db_c_rule!=NULL)//duplicate config
{
return -1;
}
}
pthread_rwlock_wrlock(&(compile_rule->rwlock));
compile_rule->ref_table=table;
compile_rule->db_c_rule=db_compile_rule;
for(i=0; i<table->ex_data_num; i++)
{
compile_rule->ads[i]=rule_ex_data_new(p_maat_rule_head, db_compile_rule->service_defined, table->ex_desc+i);
}
compile_rule->is_valid=1;
pthread_rwlock_unlock(&(compile_rule->rwlock));
return 0;
}
@@ -2282,12 +2329,7 @@ int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t*
return -1;
}
pthread_rwlock_wrlock(&(compile_rule->rwlock));
if(compile_rule->db_c_rule!=NULL)
{
free(compile_rule->db_c_rule->service_defined);
free(compile_rule->db_c_rule);
compile_rule->db_c_rule=NULL;
}
compile_rule->is_valid=0;
pthread_rwlock_unlock(&(compile_rule->rwlock));
if(compile_rule->group_cnt==0)

115
src/inc_internal/Maat_rule_internal.h
View File

@@ -46,7 +46,7 @@ typedef int atomic_t;
#define MAX_PLUGIN_PER_TABLE 32
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_SCANNER_HIT_NUM 64
#define MAX_COMPILE_EX_DATA_NUM 8
#define MAX_GROUP_CACHE 128
#define MAX_FAILED_NUM 128
@@ -84,6 +84,66 @@ enum MAAT_TABLE_TYPE
TABLE_TYPE_PLUGIN
};
enum USER_REGION_ENCODE
{
USER_REGION_ENCODE_NONE=0,
USER_REGION_ENCODE_ESCAPE,
USER_REGION_ENCODE_BASE64
};
struct compile_ex_data_idx
{
Maat_rule_EX_new_func_t *new_func;
Maat_rule_EX_free_func_t* free_func;
long argl;
void *argp;
int idx;
int table_id;
};
struct _Maat_table_info_t
{
unsigned short table_id;
unsigned short conj_cnt;
unsigned short updating_name;
char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
enum MAAT_TABLE_TYPE table_type;
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cfg_num;
int cross_cache_size;
int quick_expr_switch;
union
{
int expr_rule_cnt; //expr_type=0,1,3
int ipv4_rule_cnt;
};
union
{
int regex_rule_cnt; //expr_type=2
int ipv6_rule_cnt;
};
struct _plugin_table_info *cb_info;
int valid_flag_column; //for plugin table
int rule_tag_column; //for plugin table;
int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table;
int n_foreign;
//for compile table
enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num;
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
unsigned long long unmatch_tag_cnt;
unsigned long long iconv_err_cnt;
int stat_line_id;
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
};
struct db_str_rule_t
{
@@ -188,10 +248,12 @@ struct _Maat_compile_inner_t
{
struct db_compile_rule_t *db_c_rule;
dynamic_array_t *groups;
int is_valid;
int compile_id;//equal to db_c_rule->m_rule.config_id
int table_id;
const struct _Maat_table_info_t* ref_table;
int group_boundary;
int group_cnt;
MAAT_RULE_EX_DATA* ads;
pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read
};
struct _compile_result_t
@@ -231,51 +293,8 @@ struct _region_stat_t
int ipv6_rule_cnt;
};
};
#define USER_REGION_ENCODE_NONE 0
#define USER_REGION_ENCODE_ESCAPE 1
#define USER_REGION_ENCODE_BASE64 2
struct _Maat_table_info_t
{
unsigned short table_id;
unsigned short conj_cnt;
unsigned short updating_name;
char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
enum MAAT_TABLE_TYPE table_type;
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cfg_num;
int cross_cache_size;
int quick_expr_switch;
union
{
int expr_rule_cnt; //expr_type=0,1,3
int ipv4_rule_cnt;
};
union
{
int regex_rule_cnt; //expr_type=2
int ipv6_rule_cnt;
};
struct _plugin_table_info *cb_info;
int valid_flag_column; //for plugin table
int rule_tag_column; //for plugin table;
int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table;
int n_foreign;
int user_region_encoding; //for compile table, USER_REGION_ENCODE_xx
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
unsigned long long unmatch_tag_cnt;
unsigned long long iconv_err_cnt;
int stat_line_id;
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
};
struct _INNER_scan_status_t
{
@@ -505,6 +524,10 @@ int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_lis
void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, int print_fn, void *logger);
void rewrite_table_line_with_foreign(struct serial_rule_t*p);
void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len);
MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc);
void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc);
void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout);
void empty_serial_rules(struct serial_rule_t* rule);