diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 4a6839f..bbf2430 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -1,12 +1,12 @@ /* -*****************Maat Network Flow Rule Manage Framework******** +*****************Maat Deep Packet Inspection Policy Framework******** * Maat is the Goddess of truth and justice in ancient Egyptian concept. * Her feather was the measure that determined whether the souls (considered * to reside in the heart) of the departed would reach the paradise of afterlife * successfully. -* Author: zhengchao@iie.ac.cn,MESA -* Version 2018-09-25 foreign key and rule tags. +* Author: zhengchao@iie.ac.cn, MESA +* Version 2018-11-06 Maat Rule Extra Data. * NOTE: MUST compile with G++ * All right reserved by Institute of Infomation Engineering,Chinese Academic of Science 2014~2018 ********************************************************* @@ -181,6 +181,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, Maat_finish_callback_t *finish,//u_para void* u_para); + enum MAAT_SCAN_OPT { MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*,SIZE= strlen(string).DEFAULT: no default. @@ -238,10 +239,31 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id ,scan_status_t* mid,int thread_num); void Maat_clean_status(scan_status_t* mid); + + +struct Maat_rule_ex_t +{ + +}; +typedef void* MAAT_RULE_EX_DATA; +// The idx parameter is the index: this will be the same value returned by Maat_rule_get_ex_new_index() when the functions were initially registered. +// Finally the argl and argp parameters are the values originally passed to the same corresponding parameters when Maat_rule_get_ex_new_index() was called. +typedef void Maat_rule_EX_new_func_t(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp); +typedef void Maat_rule_EX_free_func_t(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp); + +int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name, + Maat_rule_EX_new_func_t *new_func, + Maat_rule_EX_free_func_t* free_func, + long argl, void *argp); +MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx); + + enum MAAT_RULE_OPT { MAAT_RULE_SERV_DEFINE //VALUE is a char* buffer,SIZE= buffer size. }; int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum MAAT_RULE_OPT type, void* value, int size); + + #endif // H_MAAT_RULE_H_INCLUDE diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 8b78b3f..5053f2e 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -99,6 +99,13 @@ int pickup_hit_region_from_compile(universal_bool_expr_t *compile_hit,const unsi } return k; } +void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len) +{ + memcpy(rule, rule_head, sizeof(struct _head_Maat_rule_t)); + memcpy(rule->service_defined, srv_def, MIN(srv_def_len,MAX_SERVICE_DEFINE_LEN)); + return; +} + int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num) { @@ -155,25 +162,25 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int for(i=0;idb_c_rule==NULL) + if(_mi_rule==NULL) { continue; } if(0==pthread_rwlock_tryrdlock(&(_mi_rule->rwlock))) { - make_group_set(_mi_rule,&(rs_result[result_cnt].group_set)); - r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set),_mid->cur_hit_id,_mid->cur_hit_cnt, - region_pos, MAX_SCANNER_HIT_NUM); - if(r_in_c_cnt>0)//compile config hitted becasue of new reigon + if(_mi_rule->is_valid==1) { - memcpy(&(result[result_cnt]),&(_mi_rule->db_c_rule->m_rule_head),sizeof(struct _head_Maat_rule_t)); - memcpy(result[result_cnt].service_defined - ,_mi_rule->db_c_rule->service_defined - ,MIN(_mi_rule->db_c_rule->m_rule_head.serv_def_len,MAX_SERVICE_DEFINE_LEN)); - - rs_result[result_cnt].compile_id=_mi_rule->compile_id; - result_cnt++; + make_group_set(_mi_rule,&(rs_result[result_cnt].group_set)); + r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set),_mid->cur_hit_id,_mid->cur_hit_cnt, + region_pos, MAX_SCANNER_HIT_NUM); + if(r_in_c_cnt>0)//compile config hitted becasue of new reigon + { + fill_maat_rule(&(result[result_cnt]), &(_mi_rule->db_c_rule->m_rule_head), + _mi_rule->db_c_rule->service_defined ,_mi_rule->db_c_rule->m_rule_head.serv_def_len); + rs_result[result_cnt].compile_id=_mi_rule->compile_id; + result_cnt++; + } } pthread_rwlock_unlock(&(_mi_rule->rwlock)); } @@ -880,7 +887,7 @@ error_out: void Maat_burn_feather(Maat_feather_t feather) { - _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; _feather->still_working=0;//destroy will proceed in thread_rule_monitor return; } @@ -888,7 +895,7 @@ int Maat_table_register(Maat_feather_t feather,const char* table_name) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; int table_id=-1,ret=0; - ret=map_str2int(_feather->map_tablename2id, table_name,&table_id); + ret=map_str2int(_feather->map_tablename2id, table_name, &table_id); if(ret>0) { return table_id; @@ -906,7 +913,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; int idx=0,i=0; - _Maat_table_info_t *p_table=_feather->p_table_info[table_id]; + struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id]; const char* lines=NULL; if(p_table==NULL) { @@ -952,6 +959,78 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); return 1; } + +void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user) +{ + struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user; + struct _Maat_compile_inner_t *compile_inner=(struct _Maat_compile_inner_t *)data; + MAAT_RULE_EX_DATA ad=NULL; + if(compile_inner->ref_table->table_id!=ex_desc->table_id || compile_inner->db_c_rule==NULL) + { + return; + } + pthread_rwlock_wrlock(&(compile_inner->rwlock)); + ad=rule_ex_data_new(&(compile_inner->db_c_rule->m_rule_head), + compile_inner->db_c_rule->service_defined, + ex_desc); + compile_inner->ads[ex_desc->idx]=ad; + pthread_rwlock_unlock(&(compile_inner->rwlock)); + return; +} +int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name, + Maat_rule_EX_new_func_t *new_func, + Maat_rule_EX_free_func_t* free_func, + long argl, void *argp) +{ + struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; + int table_id=-1,ret=0, idx=-1; + ret=map_str2int(_feather->map_tablename2id, compile_table_name, &table_id); + if(ret<0) + { + return -1; + } + struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id]; + if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL) + { + return -1; + } + + pthread_mutex_lock(&(_feather->backgroud_update_mutex)); + if(p_table->ex_data_num==MAX_COMPILE_EX_DATA_NUM) + { + ret=-1; + goto failed; + } + idx=p_table->ex_data_num; + p_table->ex_desc[idx].idx=idx; + p_table->ex_desc[idx].table_id=table_id; + p_table->ex_desc[idx].argl=argl; + p_table->ex_desc[idx].argp=argp; + p_table->ex_desc[idx].new_func=new_func; + p_table->ex_desc[idx].free_func=free_func; + p_table->ex_data_num++; + MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, p_table->ex_desc+idx); +failed: + pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); + return idx; + +} +MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx) +{ + struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; + struct _Maat_compile_inner_t *compile_inner=NULL; + MAAT_RULE_EX_DATA ad=NULL; + compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); + if(compile_inner==NULL) + { + return NULL; + } + pthread_rwlock_rdlock(&(compile_inner->rwlock)); + assert(idxref_table->ex_data_num); + ad=compile_inner->ads[idx]; + pthread_rwlock_unlock(&(compile_inner->rwlock)); + return ad; +} int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id ,enum MAAT_CHARSET charset,const char* data,int data_len ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num @@ -1781,17 +1860,17 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum { case MAAT_RULE_SERV_DEFINE: compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); - pthread_rwlock_rdlock(&(compile_inner->rwlock)); if(compile_inner==NULL) { ret=0; } else { + pthread_rwlock_rdlock(&(compile_inner->rwlock)); ret=MIN(size,compile_inner->db_c_rule->m_rule_head.serv_def_len); memcpy(value,compile_inner->db_c_rule->service_defined,ret); + pthread_rwlock_unlock(&(compile_inner->rwlock)); } - pthread_rwlock_unlock(&(compile_inner->rwlock)); break; default: ret=-1; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index f6bda98..a1b3844 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -504,6 +504,23 @@ int HASH_delete_by_id(MESA_htable_handle hash,int id) ret=MESA_htable_del(hash,(unsigned char*)&id, sizeof(id), NULL); return ret; } +MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc) +{ + MAAT_RULE_EX_DATA ad=NULL; + struct Maat_rule_t rule; + fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); + ex_desc->new_func(ex_desc->idx, &rule, srv_def, &ad, ex_desc->argl,ex_desc->argp); + return ad; +} +void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc) +{ + struct Maat_rule_t rule; + fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); + ex_desc->free_func(ex_desc->idx, &rule, srv_def, ad, ex_desc->argl,ex_desc->argp); + return; +} + + _Maat_table_info_t* create_table_info(int max_thread_num) { struct _Maat_table_info_t*p=NULL; @@ -832,7 +849,7 @@ struct _Maat_group_inner_t* create_group_rule(int group_id) pthread_mutex_init(&(group->mutex), NULL); return group; } -void force_destroy_group_rule(struct _Maat_group_inner_t* group) +void _destroy_group_rule(struct _Maat_group_inner_t* group) { dynamic_array_destroy(group->regions,free); group->region_cnt=0; @@ -854,7 +871,7 @@ void destroy_group_rule(struct _Maat_group_inner_t* group) { return; } - force_destroy_group_rule(group); + _destroy_group_rule(group); } void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set) { @@ -963,22 +980,39 @@ struct _Maat_compile_inner_t * create_compile_rule(int compile_id) p=(struct _Maat_compile_inner_t*)calloc(sizeof(struct _Maat_compile_inner_t),1); p->compile_id=compile_id; p->group_cnt=0; - p->table_id=0; p->group_boundary=1; p->groups=dynamic_array_create(1, 1); + p->ads=ALLOC(MAAT_RULE_EX_DATA, MAX_COMPILE_EX_DATA_NUM); pthread_rwlock_init(&(p->rwlock), NULL); return p; } -void force_destroy_compile_rule(struct _Maat_compile_inner_t * p) +void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule) { - p->compile_id=-1; - dynamic_array_destroy(p->groups,NULL); - if(p->db_c_rule!=NULL) + const struct _Maat_table_info_t* table=compile_rule->ref_table; + struct db_compile_rule_t* db_compile_rule=compile_rule->db_c_rule; + int i=0; + compile_rule->compile_id=-1; + dynamic_array_destroy(compile_rule->groups,NULL); + + + pthread_rwlock_wrlock(&(compile_rule->rwlock)); + if(db_compile_rule!=NULL) { - free(p->db_c_rule); + free(db_compile_rule->service_defined); + free(db_compile_rule); + compile_rule->db_c_rule=NULL; } - pthread_rwlock_destroy(&(p->rwlock)); - free(p); + for(i=0; iex_data_num; i++) + { + rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, table->ex_desc+i); + compile_rule->ads[i]=NULL; + } + + free(compile_rule->ads); + pthread_rwlock_unlock(&(compile_rule->rwlock)); + + pthread_rwlock_destroy(&(compile_rule->rwlock)); + free(compile_rule); } void destroy_compile_rule(struct _Maat_compile_inner_t * p) { @@ -990,7 +1024,7 @@ void destroy_compile_rule(struct _Maat_compile_inner_t * p) p_group=(struct _Maat_compile_inner_t*)dynamic_array_read(p->groups,i); assert(p_group==NULL); } - force_destroy_compile_rule(p); + _destroy_compile_rule(p); } scan_rule_t* create_rs_str_rule(unsigned int sub_type,enum MAAT_MATCH_METHOD match_method,int is_case_sensitive,const char* string,int len,int l_offset,int r_offset) { @@ -1265,8 +1299,8 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner) return; } rulescan_destroy(scanner->region); - MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))force_destroy_compile_rule); - MESA_htable_destroy(scanner->group_hash, (void (*)(void*))force_destroy_group_rule); + MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))_destroy_compile_rule); + MESA_htable_destroy(scanner->group_hash, (void (*)(void*))_destroy_group_rule); MESA_htable_destroy(scanner->region_hash, NULL); map_destroy(scanner->district_map); scanner->district_map=NULL; @@ -2254,18 +2288,31 @@ int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* { struct _Maat_compile_inner_t *compile_rule=NULL; struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head); + int i=0; compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id); if(compile_rule==NULL) { compile_rule=create_compile_rule(p_maat_rule_head->config_id); - compile_rule->table_id=table->table_id; HASH_add_by_id(scanner->compile_hash,p_maat_rule_head->config_id,compile_rule); } - if(compile_rule->db_c_rule!=NULL)//duplicate config + else { - return -1; + if(compile_rule->db_c_rule!=NULL)//duplicate config + { + return -1; + } } + + pthread_rwlock_wrlock(&(compile_rule->rwlock)); + compile_rule->ref_table=table; compile_rule->db_c_rule=db_compile_rule; + for(i=0; iex_data_num; i++) + { + compile_rule->ads[i]=rule_ex_data_new(p_maat_rule_head, db_compile_rule->service_defined, table->ex_desc+i); + } + compile_rule->is_valid=1; + pthread_rwlock_unlock(&(compile_rule->rwlock)); + return 0; } @@ -2282,12 +2329,7 @@ int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* return -1; } pthread_rwlock_wrlock(&(compile_rule->rwlock)); - if(compile_rule->db_c_rule!=NULL) - { - free(compile_rule->db_c_rule->service_defined); - free(compile_rule->db_c_rule); - compile_rule->db_c_rule=NULL; - } + compile_rule->is_valid=0; pthread_rwlock_unlock(&(compile_rule->rwlock)); if(compile_rule->group_cnt==0) diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 7ce5d08..53b8acf 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -46,7 +46,7 @@ typedef int atomic_t; #define MAX_PLUGIN_PER_TABLE 32 #define MAX_FOREIGN_CLMN_NUM 8 #define MAX_SCANNER_HIT_NUM 64 - +#define MAX_COMPILE_EX_DATA_NUM 8 #define MAX_GROUP_CACHE 128 #define MAX_FAILED_NUM 128 @@ -84,6 +84,66 @@ enum MAAT_TABLE_TYPE TABLE_TYPE_PLUGIN }; +enum USER_REGION_ENCODE +{ + USER_REGION_ENCODE_NONE=0, + USER_REGION_ENCODE_ESCAPE, + USER_REGION_ENCODE_BASE64 +}; +struct compile_ex_data_idx +{ + Maat_rule_EX_new_func_t *new_func; + Maat_rule_EX_free_func_t* free_func; + long argl; + void *argp; + int idx; + int table_id; +}; +struct _Maat_table_info_t +{ + unsigned short table_id; + unsigned short conj_cnt; + unsigned short updating_name; + char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN]; + enum MAAT_TABLE_TYPE table_type; + enum MAAT_CHARSET src_charset; + enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; + int src_charset_in_dst; + int do_charset_merge; + int cfg_num; + int cross_cache_size; + int quick_expr_switch; + union + { + int expr_rule_cnt; //expr_type=0,1,3 + int ipv4_rule_cnt; + }; + union + { + int regex_rule_cnt; //expr_type=2 + int ipv6_rule_cnt; + }; + struct _plugin_table_info *cb_info; + int valid_flag_column; //for plugin table + int rule_tag_column; //for plugin table; + int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table; + int n_foreign; + + //for compile table + enum USER_REGION_ENCODE user_region_encoding; + int ex_data_num; + struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; +//for stat>>>>>>>> + unsigned long long udpate_err_cnt; + unsigned long long unmatch_tag_cnt; + unsigned long long iconv_err_cnt; + int stat_line_id; + mcore_long_t scan_cnt; + mcore_long_t scan_cpu_time; //nano + mcore_long_t input_bytes; + mcore_long_t stream_num; + mcore_long_t hit_cnt; +}; struct db_str_rule_t { @@ -188,10 +248,12 @@ struct _Maat_compile_inner_t { struct db_compile_rule_t *db_c_rule; dynamic_array_t *groups; + int is_valid; int compile_id;//equal to db_c_rule->m_rule.config_id - int table_id; + const struct _Maat_table_info_t* ref_table; int group_boundary; int group_cnt; + MAAT_RULE_EX_DATA* ads; pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read }; struct _compile_result_t @@ -231,51 +293,8 @@ struct _region_stat_t int ipv6_rule_cnt; }; }; -#define USER_REGION_ENCODE_NONE 0 -#define USER_REGION_ENCODE_ESCAPE 1 -#define USER_REGION_ENCODE_BASE64 2 -struct _Maat_table_info_t -{ - unsigned short table_id; - unsigned short conj_cnt; - unsigned short updating_name; - char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN]; - enum MAAT_TABLE_TYPE table_type; - enum MAAT_CHARSET src_charset; - enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; - int src_charset_in_dst; - int do_charset_merge; - int cfg_num; - int cross_cache_size; - int quick_expr_switch; - union - { - int expr_rule_cnt; //expr_type=0,1,3 - int ipv4_rule_cnt; - }; - union - { - int regex_rule_cnt; //expr_type=2 - int ipv6_rule_cnt; - }; - struct _plugin_table_info *cb_info; - int valid_flag_column; //for plugin table - int rule_tag_column; //for plugin table; - int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table; - int n_foreign; - int user_region_encoding; //for compile table, USER_REGION_ENCODE_xx -//for stat>>>>>>>> - unsigned long long udpate_err_cnt; - unsigned long long unmatch_tag_cnt; - unsigned long long iconv_err_cnt; - int stat_line_id; - mcore_long_t scan_cnt; - mcore_long_t scan_cpu_time; //nano - mcore_long_t input_bytes; - mcore_long_t stream_num; - mcore_long_t hit_cnt; -}; + struct _INNER_scan_status_t { @@ -505,6 +524,10 @@ int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_lis void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, int print_fn, void *logger); void rewrite_table_line_with_foreign(struct serial_rule_t*p); +void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len); +MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc); +void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc); + void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout); void empty_serial_rules(struct serial_rule_t* rule);