gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature-rule-extra-data-index' into 'master'

Browse Source 
Feature rule extra data index

See merge request MESA_framework/maat!11
This commit is contained in:
郑超
2018-11-15 19:49:12 +08:00
parent 9fa9da75ae 70c0536a7b
commit acdbc1c7c5
15 changed files with 3472 additions and 722 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
inc/Maat_rule.h
View File

@@ -1,12 +1,12 @@
/*
*****************Maat Network Flow Rule Manage Framework********
*****************Maat Deep Packet Inspection Policy Framework********
* Maat is the Goddess of truth and justice in ancient Egyptian concept.
* Her feather was the measure that determined whether the souls (considered
* to reside in the heart) of the departed would reach the paradise of afterlife
* successfully.
* Author: zhengchao@iie.ac.cn,MESA
* Version 2018-09-25 foreign key and rule tags.
* Author: zhengchao@iie.ac.cn, MESA
* Version 2018-11-06 Maat Rule Extra Data.
* NOTE: MUST compile with G++
* All right reserved by Institute of Infomation Engineering,Chinese Academic of Science 2014~2018
*********************************************************
@@ -181,6 +181,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
Maat_finish_callback_t *finish,//u_para
void* u_para);
enum MAAT_SCAN_OPT
{
MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*,SIZE= strlen(string).DEFAULT: no default.
@@ -238,10 +239,31 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
,scan_status_t* mid,int thread_num);
void Maat_clean_status(scan_status_t* mid);
typedef void* MAAT_RULE_EX_DATA;
// The idx parameter is the index: this will be the same value returned by Maat_rule_get_ex_new_index() when the functions were initially registered.
// Finally the argl and argp parameters are the values originally passed to the same corresponding parameters when Maat_rule_get_ex_new_index() was called.
typedef void Maat_rule_EX_new_func_t(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
MAAT_RULE_EX_DATA* ad, long argl, void *argp);
typedef void Maat_rule_EX_free_func_t(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
MAAT_RULE_EX_DATA* ad, long argl, void *argp);
typedef void Maat_rule_EX_dup_func_t(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp);
int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name,
Maat_rule_EX_new_func_t* new_func,
Maat_rule_EX_free_func_t* free_func,
Maat_rule_EX_dup_func_t* dup_func,
long argl, void *argp);
//returned data is duplicated by dup_func of Maat_rule_get_ex_new_index, caller is responsible to free the data.
MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx);
enum MAAT_RULE_OPT
{
MAAT_RULE_SERV_DEFINE //VALUE is a char* buffer,SIZE= buffer size.
};
int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum MAAT_RULE_OPT type, void* value, int size);
#endif // H_MAAT_RULE_H_INCLUDE

123
src/entry/Maat_api.cpp
View File

@@ -99,6 +99,13 @@ int pickup_hit_region_from_compile(universal_bool_expr_t *compile_hit,const unsi
}
return k;
}
void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len)
{
memcpy(rule, rule_head, sizeof(struct _head_Maat_rule_t));
memcpy(rule->service_defined, srv_def, MIN(srv_def_len,MAX_SERVICE_DEFINE_LEN));
return;
}
int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num)
{
@@ -155,25 +162,25 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
for(i=0;i<scan_ret&&result_cnt<size;i++)
{
_mi_rule=array_mi_rule[i];
if(_mi_rule==NULL||_mi_rule->db_c_rule==NULL)
if(_mi_rule==NULL)
{
continue;
}
if(0==pthread_rwlock_tryrdlock(&(_mi_rule->rwlock)))
{
make_group_set(_mi_rule,&(rs_result[result_cnt].group_set));
r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set),_mid->cur_hit_id,_mid->cur_hit_cnt,
region_pos, MAX_SCANNER_HIT_NUM);
if(r_in_c_cnt>0)//compile config hitted becasue of new reigon
if(_mi_rule->is_valid==1)
{
memcpy(&(result[result_cnt]),&(_mi_rule->db_c_rule->m_rule_head),sizeof(struct _head_Maat_rule_t));
memcpy(result[result_cnt].service_defined
,_mi_rule->db_c_rule->service_defined
,MIN(_mi_rule->db_c_rule->m_rule_head.serv_def_len,MAX_SERVICE_DEFINE_LEN));
rs_result[result_cnt].compile_id=_mi_rule->compile_id;
result_cnt++;
make_group_set(_mi_rule,&(rs_result[result_cnt].group_set));
r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set),_mid->cur_hit_id,_mid->cur_hit_cnt,
region_pos, MAX_SCANNER_HIT_NUM);
if(r_in_c_cnt>0)//compile config hitted becasue of new reigon
{
fill_maat_rule(&(result[result_cnt]), &(_mi_rule->db_c_rule->m_rule_head),
_mi_rule->db_c_rule->service_defined ,_mi_rule->db_c_rule->m_rule_head.serv_def_len);
rs_result[result_cnt].compile_id=_mi_rule->compile_id;
result_cnt++;
}
}
pthread_rwlock_unlock(&(_mi_rule->rwlock));
}
@@ -880,7 +887,7 @@ error_out:
void Maat_burn_feather(Maat_feather_t feather)
{
_Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
_feather->still_working=0;//destroy will proceed in thread_rule_monitor
return;
}
@@ -888,7 +895,7 @@ int Maat_table_register(Maat_feather_t feather,const char* table_name)
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
int table_id=-1,ret=0;
ret=map_str2int(_feather->map_tablename2id, table_name,&table_id);
ret=map_str2int(_feather->map_tablename2id, table_name, &table_id);
if(ret>0)
{
return table_id;
@@ -906,7 +913,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
int idx=0,i=0;
_Maat_table_info_t *p_table=_feather->p_table_info[table_id];
struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id];
const char* lines=NULL;
if(p_table==NULL)
{
@@ -952,6 +959,88 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return 1;
}
void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user)
{
struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user;
struct _Maat_compile_inner_t *compile_inner=(struct _Maat_compile_inner_t *)data;
MAAT_RULE_EX_DATA ad=NULL;
if(compile_inner->ref_table==NULL || compile_inner->ref_table->table_id!=ex_desc->table_id || compile_inner->db_c_rule==NULL)
{
return;
}
pthread_rwlock_wrlock(&(compile_inner->rwlock));
ad=rule_ex_data_new(&(compile_inner->db_c_rule->m_rule_head),
compile_inner->db_c_rule->service_defined,
ex_desc);
compile_inner->ads[ex_desc->idx]=ad;
pthread_rwlock_unlock(&(compile_inner->rwlock));
return;
}
int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name,
Maat_rule_EX_new_func_t *new_func,
Maat_rule_EX_free_func_t* free_func,
Maat_rule_EX_dup_func_t* dup_func,
long argl, void *argp)
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
int table_id=-1,ret=0, idx=-1;
ret=map_str2int(_feather->map_tablename2id, compile_table_name, &table_id);
if(ret<0)
{
return -1;
}
struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id];
if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL)
{
return -1;
}
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
if(p_table->ex_data_num==MAX_COMPILE_EX_DATA_NUM)
{
ret=-1;
goto failed;
}
idx=p_table->ex_data_num;
p_table->ex_desc[idx].idx=idx;
p_table->ex_desc[idx].table_id=table_id;
p_table->ex_desc[idx].argl=argl;
p_table->ex_desc[idx].argp=argp;
p_table->ex_desc[idx].new_func=new_func;
p_table->ex_desc[idx].free_func=free_func;
p_table->ex_desc[idx].dup_func=dup_func;
p_table->ex_data_num++;
if(_feather->scanner!=NULL)
{
MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, p_table->ex_desc+idx);
}
failed:
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return idx;
}
MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx)
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct _Maat_compile_inner_t *compile_inner=NULL;
const struct compile_ex_data_idx* ex_desc=NULL;
MAAT_RULE_EX_DATA ad=NULL;
compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(compile_inner==NULL)
{
return NULL;
}
pthread_rwlock_rdlock(&(compile_inner->rwlock));
assert(idx<compile_inner->ref_table->ex_data_num);
ex_desc=compile_inner->ref_table->ex_desc+idx;
ex_desc->dup_func(ex_desc->idx, &ad, compile_inner->ads+idx, ex_desc->argl,ex_desc->argp);
pthread_rwlock_unlock(&(compile_inner->rwlock));
return ad;
}
int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
,enum MAAT_CHARSET charset,const char* data,int data_len
,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num
@@ -1781,17 +1870,17 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum
{
case MAAT_RULE_SERV_DEFINE:
compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
pthread_rwlock_rdlock(&(compile_inner->rwlock));
if(compile_inner==NULL)
{
ret=0;
}
else
{
pthread_rwlock_rdlock(&(compile_inner->rwlock));
ret=MIN(size,compile_inner->db_c_rule->m_rule_head.serv_def_len);
memcpy(value,compile_inner->db_c_rule->service_defined,ret);
pthread_rwlock_unlock(&(compile_inner->rwlock));
}
pthread_rwlock_unlock(&(compile_inner->rwlock));
break;
default:
ret=-1;

14
src/entry/Maat_command.cpp
View File

@@ -1532,14 +1532,14 @@ void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor
, "Get %s,%d foreign keys failed: No %dth column."
, p_rule->table_name, p_rule->rule_id, foreign_columns[i]);
continue;
break;
}
if(0!=strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix)))
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor
,"Get %s,%d foreign key failed: Invalid source prefix %s."
, p_rule->table_name, p_rule->rule_id, p_foreign);
continue;
break;
}
p_rule->f_keys[i].column=foreign_columns[i];
foreign_key_size=foreign_key_size-strlen(foreign_source_prefix);
@@ -1560,7 +1560,13 @@ void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n
p_rule->f_keys[i].is_existed=1;
}
}
return;
if(i!=n_foreign)
{
p_rule->n_foreign=0;
free(p_rule->f_keys);
p_rule->f_keys=NULL;
p_rule->with_error=1;
}
}
int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger)
{
@@ -1784,7 +1790,7 @@ void redis_monitor_traverse(long long version,redisContext *c,
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Start %s update: %lld->%lld (%d entries).",
update_type==CM_UPDATE_TYPE_INC?"INC":"FULL",version,new_version,rule_num);
for(i=0;i<rule_num;i++)
{
{
if(rule_list[i].table_line==NULL||rule_list[i].with_error==1)
{
continue;

95
src/entry/Maat_rule.cpp
View File

@@ -504,6 +504,23 @@ int HASH_delete_by_id(MESA_htable_handle hash,int id)
ret=MESA_htable_del(hash,(unsigned char*)&id, sizeof(id), NULL);
return ret;
}
MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc)
{
MAAT_RULE_EX_DATA ad=NULL;
struct Maat_rule_t rule;
fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1);
ex_desc->new_func(ex_desc->idx, &rule, srv_def, &ad, ex_desc->argl,ex_desc->argp);
return ad;
}
void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc)
{
struct Maat_rule_t rule;
fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1);
ex_desc->free_func(ex_desc->idx, &rule, srv_def, ad, ex_desc->argl,ex_desc->argp);
return;
}
_Maat_table_info_t* create_table_info(int max_thread_num)
{
struct _Maat_table_info_t*p=NULL;
@@ -832,7 +849,7 @@ struct _Maat_group_inner_t* create_group_rule(int group_id)
pthread_mutex_init(&(group->mutex), NULL);
return group;
}
void force_destroy_group_rule(struct _Maat_group_inner_t* group)
void _destroy_group_rule(struct _Maat_group_inner_t* group)
{
dynamic_array_destroy(group->regions,free);
group->region_cnt=0;
@@ -854,7 +871,7 @@ void destroy_group_rule(struct _Maat_group_inner_t* group)
{
return;
}
force_destroy_group_rule(group);
_destroy_group_rule(group);
}
void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set)
{
@@ -963,22 +980,39 @@ struct _Maat_compile_inner_t * create_compile_rule(int compile_id)
p=(struct _Maat_compile_inner_t*)calloc(sizeof(struct _Maat_compile_inner_t),1);
p->compile_id=compile_id;
p->group_cnt=0;
p->table_id=0;
p->group_boundary=1;
p->groups=dynamic_array_create(1, 1);
p->ads=ALLOC(MAAT_RULE_EX_DATA, MAX_COMPILE_EX_DATA_NUM);
pthread_rwlock_init(&(p->rwlock), NULL);
return p;
}
void force_destroy_compile_rule(struct _Maat_compile_inner_t * p)
void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule)
{
p->compile_id=-1;
dynamic_array_destroy(p->groups,NULL);
if(p->db_c_rule!=NULL)
const struct _Maat_table_info_t* table=compile_rule->ref_table;
struct db_compile_rule_t* db_compile_rule=compile_rule->db_c_rule;
int i=0;
compile_rule->compile_id=-1;
dynamic_array_destroy(compile_rule->groups,NULL);
pthread_rwlock_wrlock(&(compile_rule->rwlock));
if(db_compile_rule!=NULL)
{
free(p->db_c_rule);
free(db_compile_rule->service_defined);
free(db_compile_rule);
compile_rule->db_c_rule=NULL;
}
pthread_rwlock_destroy(&(p->rwlock));
free(p);
for(i=0; i<table->ex_data_num; i++)
{
rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, table->ex_desc+i);
compile_rule->ads[i]=NULL;
}
free(compile_rule->ads);
pthread_rwlock_unlock(&(compile_rule->rwlock));
pthread_rwlock_destroy(&(compile_rule->rwlock));
free(compile_rule);
}
void destroy_compile_rule(struct _Maat_compile_inner_t * p)
{
@@ -990,7 +1024,7 @@ void destroy_compile_rule(struct _Maat_compile_inner_t * p)
p_group=(struct _Maat_compile_inner_t*)dynamic_array_read(p->groups,i);
assert(p_group==NULL);
}
force_destroy_compile_rule(p);
_destroy_compile_rule(p);
}
scan_rule_t* create_rs_str_rule(unsigned int sub_type,enum MAAT_MATCH_METHOD match_method,int is_case_sensitive,const char* string,int len,int l_offset,int r_offset)
{
@@ -1265,8 +1299,8 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner)
return;
}
rulescan_destroy(scanner->region);
MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))force_destroy_compile_rule);
MESA_htable_destroy(scanner->group_hash, (void (*)(void*))force_destroy_group_rule);
MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))_destroy_compile_rule);
MESA_htable_destroy(scanner->group_hash, (void (*)(void*))_destroy_group_rule);
MESA_htable_destroy(scanner->region_hash, NULL);
map_destroy(scanner->district_map);
scanner->district_map=NULL;
@@ -2254,18 +2288,31 @@ int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t*
{
struct _Maat_compile_inner_t *compile_rule=NULL;
struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head);
int i=0;
compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id);
if(compile_rule==NULL)
{
compile_rule=create_compile_rule(p_maat_rule_head->config_id);
compile_rule->table_id=table->table_id;
HASH_add_by_id(scanner->compile_hash,p_maat_rule_head->config_id,compile_rule);
}
if(compile_rule->db_c_rule!=NULL)//duplicate config
else
{
return -1;
if(compile_rule->db_c_rule!=NULL)//duplicate config
{
return -1;
}
}
pthread_rwlock_wrlock(&(compile_rule->rwlock));
compile_rule->ref_table=table;
compile_rule->db_c_rule=db_compile_rule;
for(i=0; i<table->ex_data_num; i++)
{
compile_rule->ads[i]=rule_ex_data_new(p_maat_rule_head, db_compile_rule->service_defined, table->ex_desc+i);
}
compile_rule->is_valid=1;
pthread_rwlock_unlock(&(compile_rule->rwlock));
return 0;
}
@@ -2282,12 +2329,7 @@ int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t*
return -1;
}
pthread_rwlock_wrlock(&(compile_rule->rwlock));
if(compile_rule->db_c_rule!=NULL)
{
free(compile_rule->db_c_rule->service_defined);
free(compile_rule->db_c_rule);
compile_rule->db_c_rule=NULL;
}
compile_rule->is_valid=0;
pthread_rwlock_unlock(&(compile_rule->rwlock));
if(compile_rule->group_cnt==0)
@@ -2304,7 +2346,14 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s
ret=sscanf(table_line,"%d\t%d\t%d",&(db_group_rule.group_id)
,&(db_group_rule.compile_id)
,&(db_group_rule.is_valid));
assert(ret==3);
if(ret!=3)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"update error,invalid format of group table %s:%s"
,table->table_name[table->updating_name], table_line);
table->udpate_err_cnt++;
return;
}
if(db_group_rule.is_valid==FALSE)
{
del_group_rule(table, &db_group_rule,scanner,logger);

3246
src/entry/cJSON.c
View File

File diff suppressed because it is too large Load Diff

6
src/entry/config_monitor.cpp
View File

@@ -234,10 +234,10 @@ int get_new_idx_path(long long current_version,const char*file_dir,void* logger
{
continue;
}
if(strlen(namelist[i]->d_name)>32)
if(strlen(namelist[i]->d_name)>42)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor
,"config file %s filename too long,should like full_config_index.0000000001"
,"config file %s filename too long,should like full_config_index.00000000000000000001"
,namelist[i]->d_name);
continue;
}
@@ -245,7 +245,7 @@ int get_new_idx_path(long long current_version,const char*file_dir,void* logger
if(sscanf_ret!=2)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor
,"config file %s filename error,should like full_config_index.0000000001"
,"config file %s filename error,should like full_config_index.00000000000000000001"
,namelist[i]->d_name);
continue;
}

103
src/entry/json2iris.cpp
View File

@@ -40,8 +40,8 @@ struct iris_description_t
char tmp_iris_index_dir[MAX_PATH_LINE];
char index_path[MAX_PATH_LINE];
struct iris_table_t group_table;
struct iris_table_t compile_table;
struct iris_table_t* group_table;
struct iris_table_t* compile_table;
MESA_htable_handle group_name_map;
MESA_htable_handle iris_table_map;
MESA_htable_handle str2int_map;
@@ -57,6 +57,21 @@ struct traslate_command_t
const char* default_string;
int default_int;
};
struct iris_table_t* query_table_info(iris_description_t* p_iris,const char* table_name)
{
struct iris_table_t* table_info=NULL;
table_info=(struct iris_table_t*)MESA_htable_search(p_iris->iris_table_map, (const unsigned char*)table_name,strlen(table_name));
if(table_info==NULL)
{
table_info=(struct iris_table_t*)calloc(sizeof(struct iris_table_t),1);
table_info->line_count=0;
memcpy(table_info->table_name,table_name,MIN(sizeof(table_info->table_name)-1, strlen(table_name)));
snprintf(table_info->table_path,sizeof(table_info->table_path),"%s/%s.local",p_iris->tmp_iris_dir,table_info->table_name);
MESA_htable_add(p_iris->iris_table_map,(const unsigned char*)table_info->table_name,strlen(table_info->table_name),table_info);
}
return table_info;
}
static int get_group_seq(struct iris_description_t* iris_cfg)
{
redisReply* data_reply=NULL;
@@ -99,15 +114,6 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,
snprintf(iris_cfg->tmp_iris_index_dir,sizeof(iris_cfg->tmp_iris_index_dir),"%s_iris_tmp/index",json_file);
snprintf(iris_cfg->index_path,sizeof(iris_cfg->index_path),"%s/full_config_index.%010d",iris_cfg->tmp_iris_index_dir,json_version);
memcpy(iris_cfg->compile_table.table_name,compile_tn,strlen(compile_tn));
snprintf(iris_cfg->compile_table.table_path,sizeof(iris_cfg->compile_table.table_path),
"%s/%s.local",iris_cfg->tmp_iris_dir,iris_cfg->compile_table.table_name);
memcpy(iris_cfg->group_table.table_name,group_tn,strlen(group_tn));
snprintf(iris_cfg->group_table.table_path,sizeof(iris_cfg->group_table.table_name),
"%s/%s.local",iris_cfg->tmp_iris_dir,iris_cfg->group_table.table_name);
iris_cfg->redis_write_ctx=redis_write_ctx;
MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs));
@@ -162,6 +168,9 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,
map_register(iris_cfg->str2int_map, "hexbin",1);
map_register(iris_cfg->str2int_map, "case plain",2);
iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn);
iris_cfg->group_table=query_table_info(iris_cfg, group_tn);
return 0;
}
void clear_iris_descriptor(struct iris_description_t *iris_cfg)
@@ -174,6 +183,7 @@ void clear_iris_descriptor(struct iris_description_t *iris_cfg)
{
MESA_htable_destroy(iris_cfg->iris_table_map, NULL);
}
map_destroy(iris_cfg->str2int_map);
return;
}
int create_tmp_dir(struct iris_description_t *p)
@@ -532,20 +542,7 @@ int write_similar_rule(cJSON *region_json,struct iris_description_t *p_iris,cons
return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
}
struct iris_table_t* query_table_info(iris_description_t* p_iris,const char* table_name)
{
struct iris_table_t* table_info=NULL;
table_info=(struct iris_table_t*)MESA_htable_search(p_iris->iris_table_map, (const unsigned char*)table_name,strlen(table_name));
if(table_info==NULL)
{
table_info=(struct iris_table_t*)calloc(sizeof(struct iris_table_t),1);
table_info->line_count=0;
memcpy(table_info->table_name,table_name,MIN(sizeof(table_info->table_name)-1, strlen(table_name)));
snprintf(table_info->table_path,sizeof(table_info->table_path),"%s/%s.local",p_iris->tmp_iris_dir,table_info->table_name);
MESA_htable_add(p_iris->iris_table_map,(const unsigned char*)table_info->table_name,strlen(table_info->table_name),table_info);
}
return table_info;
}
int write_plugin_table(cJSON* plug_table_json,int sequence,iris_description_t* p_iris,void* logger)
{
cJSON* item=NULL,*table_content=NULL,*each_line=NULL;
@@ -695,8 +692,12 @@ int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * l
{
int compile_id=-1,cmd_cnt=0,ret=-1;
cJSON* item=NULL;
struct iris_table_t* table_info=NULL;
struct traslate_command_t compile_cmd[MAX_COLUMN_NUM];
memset(compile_cmd,0,sizeof(compile_cmd));
compile_cmd[cmd_cnt].json_string="compile_id";
compile_cmd[cmd_cnt].json_type=cJSON_Number;
cmd_cnt++;
@@ -733,16 +734,25 @@ int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * l
compile_cmd[cmd_cnt].str2int_flag=1;
cmd_cnt++;
if(p_iris->compile_table.line_count==0)
item=cJSON_GetObjectItem(compile,"table_name");
if(item==NULL||item->type!=cJSON_String)
{
ret=set_file_rulenum(p_iris->compile_table.table_path, 0,logger);
table_info=p_iris->compile_table;
}
else
{
table_info=query_table_info(p_iris,item->valuestring);
}
if(table_info->line_count==0)
{
ret=set_file_rulenum(table_info->table_path, 0,logger);
if(ret<0)
{
return -1;
}
}
ret=direct_write_rule(compile, p_iris->str2int_map,compile_cmd,cmd_cnt, p_iris->compile_table.table_path,logger);
ret=direct_write_rule(compile, p_iris->str2int_map,compile_cmd,cmd_cnt, table_info->table_path,logger);
if(ret<0)
{
return -1;
@@ -755,8 +765,8 @@ int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * l
return -1;
}
compile_id=item->valueint;
p_iris->compile_table.line_count++;
set_file_rulenum(p_iris->compile_table.table_path,p_iris->compile_table.line_count,logger);
table_info->line_count++;
set_file_rulenum(table_info->table_path,table_info->line_count,logger);
return compile_id;
}
int write_group_rule(int compile_id ,int group_id,struct iris_description_t *p_iris,void * logger)
@@ -764,25 +774,25 @@ int write_group_rule(int compile_id ,int group_id,struct iris_description_t *p_i
FILE*fp=NULL;
int ret=0;
if(p_iris->group_table.line_count==0)
if(p_iris->group_table->line_count==0)
{
ret=set_file_rulenum(p_iris->group_table.table_path,0,logger);
ret=set_file_rulenum(p_iris->group_table->table_path,0,logger);
if(ret<0)
{
return -1;
}
}
fp=fopen(p_iris->group_table.table_path,"a");
fp=fopen(p_iris->group_table->table_path,"a");
if(fp==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"fopen %s error %s.",p_iris->group_table.table_path,strerror(errno));
"fopen %s error %s.",p_iris->group_table->table_path,strerror(errno));
return -1;
}
fprintf(fp,"%d\t%d\t1\n",group_id,compile_id);
fclose(fp);
p_iris->group_table.line_count++;
ret=set_file_rulenum(p_iris->group_table.table_path,p_iris->group_table.line_count,logger);
p_iris->group_table->line_count++;
ret=set_file_rulenum(p_iris->group_table->table_path,p_iris->group_table->line_count,logger);
return 0;
}
void table_idx_write_cb(const uchar * key, uint size, void * data, void * user)
@@ -801,12 +811,6 @@ int write_index_file(struct iris_description_t *p_iris,void* logger)
"index file %s fopen error %s.",p_iris->index_path,strerror(errno));
return -1;
}
fprintf(fp,"%s\t%d\t%s\n",p_iris->compile_table.table_name
,p_iris->compile_table.line_count
,p_iris->compile_table.table_path);
fprintf(fp,"%s\t%d\t%s\n",p_iris->group_table.table_name
,p_iris->group_table.line_count
,p_iris->group_table.table_path);
MESA_htable_iterate(p_iris->iris_table_map, table_idx_write_cb, fp);
fclose(fp);
return 0;
@@ -876,8 +880,6 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
item=cJSON_GetObjectItem(group_rule,"group_name");
if(item==NULL||item->type!=cJSON_String)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_json,
"compile rule %d's group has no name,regard as Untitled.",compile_id);
group_name=untitled_group_name;
}
else
@@ -984,7 +986,7 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,r
json_file_size=fstat_buf.st_size;
json_buff=(char*)calloc(1,json_file_size);
json_buff=(char*)calloc(1,json_file_size+1);
read_size=fread(json_buff,1,json_file_size,json_fp);
if(read_size!=json_file_size)
{
@@ -1000,14 +1002,15 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,r
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,"Error before: %-200.200s",cJSON_GetErrorPtr());
goto error_out;
}
if(compile_tn==NULL)
tmp_obj=cJSON_GetObjectItem(json,"compile_table");
if(tmp_obj)
{
tmp_obj=cJSON_GetObjectItem(json,"compile_table");
compile_tn=tmp_obj->valuestring;
}
if(group_tn==NULL)
tmp_obj=cJSON_GetObjectItem(json,"group_table");
if(tmp_obj)
{
tmp_obj=cJSON_GetObjectItem(json,"group_table");
group_tn=tmp_obj->valuestring;
}
ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,redis_write_ctx,&iris_cfg,logger);

4
src/entry/map_str2int.cpp
View File

@@ -95,7 +95,7 @@ void duplicate_func(const uchar * key, uint size, void * data, void * user)
{
MESA_htable_handle target=(MESA_htable_handle)user;
int value=*(int*)data;
int ret=0;
__attribute__((unused)) int ret=0;
char* string=(char*)calloc(sizeof(char),(size+1));
memcpy(string,key,size);
ret=map_register(target,string,value);
@@ -103,7 +103,7 @@ void duplicate_func(const uchar * key, uint size, void * data, void * user)
}
MESA_htable_handle map_duplicate(MESA_htable_handle origin_map)
{
int ret=0;
__attribute__((unused)) int ret=0;
MESA_htable_handle target=NULL;
target=map_create();
ret=MESA_htable_iterate(origin_map, duplicate_func, target);

117
src/inc_internal/Maat_rule_internal.h
View File

@@ -46,7 +46,7 @@ typedef int atomic_t;
#define MAX_PLUGIN_PER_TABLE 32
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_SCANNER_HIT_NUM 64
#define MAX_COMPILE_EX_DATA_NUM 8
#define MAX_GROUP_CACHE 128
#define MAX_FAILED_NUM 128
@@ -84,6 +84,67 @@ enum MAAT_TABLE_TYPE
TABLE_TYPE_PLUGIN
};
enum USER_REGION_ENCODE
{
USER_REGION_ENCODE_NONE=0,
USER_REGION_ENCODE_ESCAPE,
USER_REGION_ENCODE_BASE64
};
struct compile_ex_data_idx
{
Maat_rule_EX_new_func_t *new_func;
Maat_rule_EX_free_func_t* free_func;
Maat_rule_EX_dup_func_t* dup_func;
long argl;
void *argp;
int idx;
int table_id;
};
struct _Maat_table_info_t
{
unsigned short table_id;
unsigned short conj_cnt;
unsigned short updating_name;
char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
enum MAAT_TABLE_TYPE table_type;
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cfg_num;
int cross_cache_size;
int quick_expr_switch;
union
{
int expr_rule_cnt; //expr_type=0,1,3
int ipv4_rule_cnt;
};
union
{
int regex_rule_cnt; //expr_type=2
int ipv6_rule_cnt;
};
struct _plugin_table_info *cb_info;
int valid_flag_column; //for plugin table
int rule_tag_column; //for plugin table;
int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table;
int n_foreign;
//for compile table
enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num;
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
unsigned long long unmatch_tag_cnt;
unsigned long long iconv_err_cnt;
int stat_line_id;
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
};
struct db_str_rule_t
{
@@ -188,10 +249,12 @@ struct _Maat_compile_inner_t
{
struct db_compile_rule_t *db_c_rule;
dynamic_array_t *groups;
int is_valid;
int compile_id;//equal to db_c_rule->m_rule.config_id
int table_id;
const struct _Maat_table_info_t* ref_table;
int group_boundary;
int group_cnt;
MAAT_RULE_EX_DATA* ads;
pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read
};
struct _compile_result_t
@@ -231,51 +294,8 @@ struct _region_stat_t
int ipv6_rule_cnt;
};
};
#define USER_REGION_ENCODE_NONE 0
#define USER_REGION_ENCODE_ESCAPE 1
#define USER_REGION_ENCODE_BASE64 2
struct _Maat_table_info_t
{
unsigned short table_id;
unsigned short conj_cnt;
unsigned short updating_name;
char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
enum MAAT_TABLE_TYPE table_type;
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cfg_num;
int cross_cache_size;
int quick_expr_switch;
union
{
int expr_rule_cnt; //expr_type=0,1,3
int ipv4_rule_cnt;
};
union
{
int regex_rule_cnt; //expr_type=2
int ipv6_rule_cnt;
};
struct _plugin_table_info *cb_info;
int valid_flag_column; //for plugin table
int rule_tag_column; //for plugin table;
int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table;
int n_foreign;
int user_region_encoding; //for compile table, USER_REGION_ENCODE_xx
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
unsigned long long unmatch_tag_cnt;
unsigned long long iconv_err_cnt;
int stat_line_id;
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
};
struct _INNER_scan_status_t
{
@@ -472,6 +492,7 @@ struct serial_rule_t //rm= Redis Maat
enum MAAT_OPERATION op;//0: delete, 1: add.
long rule_id;
int label_id;
char with_error;
long long timeout; // absolute unix time.
char table_name[256];
char* table_line;
@@ -505,6 +526,10 @@ int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_lis
void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, int print_fn, void *logger);
void rewrite_table_line_with_foreign(struct serial_rule_t*p);
void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len);
MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc);
void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc);
void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout);
void empty_serial_rules(struct serial_rule_t* rule);

278
src/inc_internal/cJSON.h
View File

@@ -1,19 +1,19 @@
/*
Copyright (c) 2009 Dave Gamble
Copyright (c) 2009-2017 Dave Gamble and cJSON contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE EXPR_TYPE_AND NONINFRINGEMENT. IN NO EVENT SHALL THE
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
@@ -28,113 +28,247 @@ extern "C"
{
#endif
/* project version */
#define CJSON_VERSION_MAJOR 1
#define CJSON_VERSION_MINOR 7
#define CJSON_VERSION_PATCH 7
#include <stddef.h>
/* cJSON Types: */
#define cJSON_False 0
#define cJSON_True 1
#define cJSON_NULL 2
#define cJSON_Number 3
#define cJSON_String 4
#define cJSON_Array 5
#define cJSON_Object 6
#define cJSON_Invalid (0)
#define cJSON_False (1 << 0)
#define cJSON_True (1 << 1)
#define cJSON_NULL (1 << 2)
#define cJSON_Number (1 << 3)
#define cJSON_String (1 << 4)
#define cJSON_Array (1 << 5)
#define cJSON_Object (1 << 6)
#define cJSON_Raw (1 << 7) /* raw json */
#define cJSON_IsReference 256
#define cJSON_StringIsConst 512
/* The cJSON structure: */
typedef struct cJSON {
struct cJSON *next,*prev; /* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */
struct cJSON *child; /* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */
typedef struct cJSON
{
/* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */
struct cJSON *next;
struct cJSON *prev;
/* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */
struct cJSON *child;
int type; /* The type of the item, as above. */
/* The type of the item, as above. */
int type;
char *valuestring; /* The item's string, if type==cJSON_String */
int valueint; /* The item's number, if type==cJSON_Number */
double valuedouble; /* The item's number, if type==cJSON_Number */
/* The item's string, if type==cJSON_String and type == cJSON_Raw */
char *valuestring;
/* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead */
int valueint;
/* The item's number, if type==cJSON_Number */
double valuedouble;
char *string; /* The item's name string, if this item is the child of, or is in the list of subitems of an object. */
/* The item's name string, if this item is the child of, or is in the list of subitems of an object. */
char *string;
} cJSON;
typedef struct cJSON_Hooks {
typedef struct cJSON_Hooks
{
void *(*malloc_fn)(size_t sz);
void (*free_fn)(void *ptr);
} cJSON_Hooks;
typedef int cJSON_bool;
#if !defined(__WINDOWS__) && (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32))
#define __WINDOWS__
#endif
#ifdef __WINDOWS__
/* When compiling for windows, we specify a specific calling convention to avoid issues where we are being called from a project with a different default calling convention. For windows you have 2 define options:
CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever dllexport symbols
CJSON_EXPORT_SYMBOLS - Define this on library build when you want to dllexport symbols (default)
CJSON_IMPORT_SYMBOLS - Define this if you want to dllimport symbol
For *nix builds that support visibility attribute, you can define similar behavior by
setting default visibility to hidden by adding
-fvisibility=hidden (for gcc)
or
-xldscope=hidden (for sun cc)
to CFLAGS
then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJSON_EXPORT_SYMBOLS does
*/
/* export symbols by default, this is necessary for copy pasting the C and header file */
#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && !defined(CJSON_EXPORT_SYMBOLS)
#define CJSON_EXPORT_SYMBOLS
#endif
#if defined(CJSON_HIDE_SYMBOLS)
#define CJSON_PUBLIC(type) type __stdcall
#elif defined(CJSON_EXPORT_SYMBOLS)
#define CJSON_PUBLIC(type) __declspec(dllexport) type __stdcall
#elif defined(CJSON_IMPORT_SYMBOLS)
#define CJSON_PUBLIC(type) __declspec(dllimport) type __stdcall
#endif
#else /* !WIN32 */
#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined (__SUNPRO_C)) && defined(CJSON_API_VISIBILITY)
#define CJSON_PUBLIC(type) __attribute__((visibility("default"))) type
#else
#define CJSON_PUBLIC(type) type
#endif
#endif
/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse them.
* This is to prevent stack overflows. */
#ifndef CJSON_NESTING_LIMIT
#define CJSON_NESTING_LIMIT 1000
#endif
/* returns the version of cJSON as a string */
CJSON_PUBLIC(const char*) cJSON_Version(void);
/* Supply malloc, realloc and free functions to cJSON */
extern void cJSON_InitHooks(cJSON_Hooks* hooks);
CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks);
/* Memory Management: the caller is always responsible to free the results from all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is cJSON_PrintPreallocated, where the caller has full responsibility of the buffer. */
/* Supply a block of JSON, and this returns a cJSON object you can interrogate. */
CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value);
/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */
/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error so will match cJSON_GetErrorPtr(). */
CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated);
/* Supply a block of JSON, and this returns a cJSON object you can interrogate. Call cJSON_Delete when finished. */
extern cJSON *cJSON_Parse(const char *value);
/* Render a cJSON entity to text for transfer/storage. Free the char* when finished. */
extern char *cJSON_Print(cJSON *item);
/* Render a cJSON entity to text for transfer/storage without any formatting. Free the char* when finished. */
extern char *cJSON_PrintUnformatted(cJSON *item);
/* Render a cJSON entity to text for transfer/storage. */
CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item);
/* Render a cJSON entity to text for transfer/storage without any formatting. */
CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item);
/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */
CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt);
/* Render a cJSON entity to text using a buffer already allocated in memory with given length. Returns 1 on success and 0 on failure. */
/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will use, so to be safe allocate 5 bytes more than you actually need */
CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format);
/* Delete a cJSON entity and all subentities. */
extern void cJSON_Delete(cJSON *c);
CJSON_PUBLIC(void) cJSON_Delete(cJSON *c);
/* Returns the number of items in an array (or object). */
extern int cJSON_GetArraySize(cJSON *array);
/* Retrieve item number "item" from array "array". Returns NULL if unsuccessful. */
extern cJSON *cJSON_GetArrayItem(cJSON *array,int item);
CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array);
/* Retrieve item number "index" from array "array". Returns NULL if unsuccessful. */
CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index);
/* Get item "string" from object. Case insensitive. */
extern cJSON *cJSON_GetObjectItem(cJSON *object,const char *string);
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string);
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string);
CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string);
/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */
extern const char *cJSON_GetErrorPtr(void);
CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void);
/* Check if the item is a string and return its valuestring */
CJSON_PUBLIC(char *) cJSON_GetStringValue(cJSON *item);
/* These functions check the type of an item */
CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item);
/* These calls create a cJSON item of the appropriate type. */
extern cJSON *cJSON_CreateNull(void);
extern cJSON *cJSON_CreateTrue(void);
extern cJSON *cJSON_CreateFalse(void);
extern cJSON *cJSON_CreateBool(int b);
extern cJSON *cJSON_CreateNumber(double num);
extern cJSON *cJSON_CreateString(const char *string);
extern cJSON *cJSON_CreateArray(void);
extern cJSON *cJSON_CreateObject(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean);
CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num);
CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string);
/* raw json */
CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw);
CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void);
/* Create a string where valuestring references a string so
* it will not be freed by cJSON_Delete */
CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string);
/* Create an object/arrray that only references it's elements so
* they will not be freed by cJSON_Delete */
CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child);
CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child);
/* These utilities create an Array of count items. */
extern cJSON *cJSON_CreateIntArray(const int *numbers,int count);
extern cJSON *cJSON_CreateFloatArray(const float *numbers,int count);
extern cJSON *cJSON_CreateDoubleArray(const double *numbers,int count);
extern cJSON *cJSON_CreateStringArray(const char **strings,int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count);
/* Append item to the specified array/object. */
extern void cJSON_AddItemToArray(cJSON *array, cJSON *item);
extern void cJSON_AddItemToObject(cJSON *object,const char *string,cJSON *item);
CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item);
CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item);
/* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object.
* WARNING: When this function was used, make sure to always check that (item->type & cJSON_StringIsConst) is zero before
* writing to `item->string` */
CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item);
/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */
extern void cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item);
extern void cJSON_AddItemReferenceToObject(cJSON *object,const char *string,cJSON *item);
CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item);
CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item);
/* Remove/Detatch items from Arrays/Objects. */
extern cJSON *cJSON_DetachItemFromArray(cJSON *array,int which);
extern void cJSON_DeleteItemFromArray(cJSON *array,int which);
extern cJSON *cJSON_DetachItemFromObject(cJSON *object,const char *string);
extern void cJSON_DeleteItemFromObject(cJSON *object,const char *string);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which);
CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string);
CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string);
CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string);
/* Update array items. */
extern void cJSON_ReplaceItemInArray(cJSON *array,int which,cJSON *newitem);
extern void cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem);
CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */
CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement);
CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem);
CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem);
CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object,const char *string,cJSON *newitem);
/* Duplicate a cJSON item */
extern cJSON *cJSON_Duplicate(cJSON *item,int recurse);
CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse);
/* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will
need to be released. With recurse!=0, it will duplicate any children connected to the item.
The item->next and ->prev pointers are always zero on return from Duplicate. */
/* Recursively compare two cJSON items for equality. If either a or b is NULL or invalid, they will be considered unequal.
* case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */
CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive);
/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */
extern cJSON *cJSON_ParseWithOpts(const char *value,const char **return_parse_end,int require_null_terminated);
extern void cJSON_Minify(char *json);
CJSON_PUBLIC(void) cJSON_Minify(char *json);
/* Macros for creating things quickly. */
#define cJSON_AddNullToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateNull())
#define cJSON_AddTrueToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateTrue())
#define cJSON_AddFalseToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateFalse())
#define cJSON_AddBoolToObject(object,name,b) cJSON_AddItemToObject(object, name, cJSON_CreateBool(b))
#define cJSON_AddNumberToObject(object,name,n) cJSON_AddItemToObject(object, name, cJSON_CreateNumber(n))
#define cJSON_AddStringToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateString(s))
/* Helper functions for creating and adding items to an object at the same time.
* They return the added item or NULL on failure. */
CJSON_PUBLIC(cJSON*) cJSON_AddNullToObject(cJSON * const object, const char * const name);
CJSON_PUBLIC(cJSON*) cJSON_AddTrueToObject(cJSON * const object, const char * const name);
CJSON_PUBLIC(cJSON*) cJSON_AddFalseToObject(cJSON * const object, const char * const name);
CJSON_PUBLIC(cJSON*) cJSON_AddBoolToObject(cJSON * const object, const char * const name, const cJSON_bool boolean);
CJSON_PUBLIC(cJSON*) cJSON_AddNumberToObject(cJSON * const object, const char * const name, const double number);
CJSON_PUBLIC(cJSON*) cJSON_AddStringToObject(cJSON * const object, const char * const name, const char * const string);
CJSON_PUBLIC(cJSON*) cJSON_AddRawToObject(cJSON * const object, const char * const name, const char * const raw);
CJSON_PUBLIC(cJSON*) cJSON_AddObjectToObject(cJSON * const object, const char * const name);
CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * const name);
/* When assigning an integer value, it needs to be propagated to valuedouble too. */
#define cJSON_SetIntValue(object,val) ((object)?(object)->valueint=(object)->valuedouble=(val):(val))
#define cJSON_SetIntValue(object, number) ((object) ? (object)->valueint = (object)->valuedouble = (number) : (number))
/* helper for the cJSON_SetNumberValue macro */
CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number);
#define cJSON_SetNumberValue(object, number) ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) : (number))
/* Macro for iterating over an array or object */
#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next)
/* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */
CJSON_PUBLIC(void *) cJSON_malloc(size_t size);
CJSON_PUBLIC(void) cJSON_free(void *object);
#ifdef __cplusplus
}

23
test/maat_demo.cpp
View File

@@ -58,9 +58,16 @@ void Maat_read_entry_finish_cb(void* u_para)
long long version=0;
int ret=0,is_last_updating_table=0;
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version));
assert(ret==0);
if(ret!=0)
{
assert(0);
}
ret=Maat_read_state(feather,MAAT_STATE_LAST_UPDATING_TABLE, &is_last_updating_table, sizeof(is_last_updating_table));
assert(ret==0);
if(ret!=0)
{
assert(0);
}
//printf("Maat Version %lld at plugin finish callback, is_last_update=%d.\n",version,is_last_updating_table);
return;
@@ -674,7 +681,7 @@ void test_set_cmd_line(Maat_feather_t feather)
const struct Maat_line_t *p_line[TEST_CMD_LINE_NUM];
struct Maat_line_t line_rule[TEST_CMD_LINE_NUM];
char table_line[TEST_CMD_LINE_NUM][128];
int ret=0,i=0;
int i=0;
memset(&line_rule,0,sizeof(line_rule));
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
@@ -688,16 +695,14 @@ void test_set_cmd_line(Maat_feather_t feather)
}
ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
assert(ret>0);
Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
usleep(WAIT_FOR_EFFECTIVE_US);
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
line_rule[i].table_line=NULL;
}
ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
assert(ret>0);
Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
return;
@@ -707,12 +712,10 @@ void history_matter_cb(int table_id,const char* table_line,void* u_para)
}
void test_load_specific_version(Maat_feather_t feather, const char* redis_ip, unsigned short redis_port, int redis_db,void*logger)
{
int ret=0;
long long version=0;
const char *table_name="HISTORY_MATTER_CB";
test_plugin_table(feather,table_name,NULL,history_matter_cb,NULL,NULL,logger);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version));
assert(ret==0);
Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version));
printf("Maat Version %lld.\n",version);
}
int test_add_expr_command(Maat_feather_t feather,const char* region_table,int config_id, int timeout,int label_id, const char* keywords)

27
test/maat_json.json
View File

@@ -521,6 +521,33 @@
]
}
]
},
{
"compile_id": 141,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "Something:I\\bhave\\ba\\bname,7799",
"table_name":"COMPILE_ALIAS",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "i.ytimg.com",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
],
"plugin_table": [

35
test/right_result.txt
View File

@@ -1,35 +0,0 @@
Load entry id 101 SUCCESS.
hit 1 rules
ipv4 scan hit compile rule id 123.
hit current region,but not hit compile rule.
ipv6 scan hit region.
digest scan hit 127.
digest scan hit 127.
digest scan hit 127.
digest scan hit 127.
digest scan hit 127.
digest scan hit 127.
Hit expr_plus rule 128.
URL encode scan utf8 url hit 1 rules, hit ruleid=129
URL encode scan gb2312 url hit 1 rules, hit ruleid=129
test_unicode_esc processing ./testdata_uni2ascii/original_Uygur_webpage.html
UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 2 rules, hit ruleid=130 131
UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131
UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131
UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131
UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131
UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131
UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131
UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131
UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131
UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131
test_unicode_esc processing ./testdata_uni2ascii/original_uy.txt
UNI2ASCII file ./testdata_uni2ascii/original_uy.txt,hit 2 rules, hit ruleid=130 131
test_unicode_esc processing ./testdata_uni2ascii/qq_mail_https.txt
UNI2ASCII file ./testdata_uni2ascii/qq_mail_https.txt,hit 2 rules, hit ruleid=130 131
UNI2ASCII file ./testdata_uni2ascii/qq_mail_https.txt,hit 1 rules, hit ruleid=130
test_unicode_esc processing ./testdata_uni2ascii/sina_read_mail.txt
UNI2ASCII file ./testdata_uni2ascii/sina_read_mail.txt,hit 2 rules, hit ruleid=130 131
test_unicode_esc processing ./testdata_uni2ascii/sohu_mail_unicode.txt
UNI2ASCII file ./testdata_uni2ascii/sohu_mail_unicode.txt,hit 2 rules, hit ruleid=130 131
UNI2ASCII file ./testdata_uni2ascii/sohu_mail_unicode.txt,hit 2 rules, hit ruleid=130 131

3
test/table_info.conf
View File

@@ -28,4 +28,5 @@
9 SIM_URL similar --
10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff
11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} --
12 TEST_FOREIGN_KEY plugin {"valid":4,"foreign":"6,8","tag":3} --
12 TEST_FOREIGN_KEY plugin {"valid":4,"foreign":"6,8","tag":3} --
13 COMPILE_ALIAS compile escape --

92
test/test_maatframe.cpp
View File

@@ -18,6 +18,7 @@
#include <unistd.h>
#include <dirent.h>
#include <openssl/md5.h>
#include <pthread.h>
#include <gtest/gtest.h>
const char* test_maat_redis_ip="127.0.0.1";
@@ -540,6 +541,96 @@ TEST(RuleTags, Compile)
return;
}
struct rule_ex_param
{
int ref_cnt;
char name[128];
int id;
pthread_mutex_t lock;
};
void ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
MAAT_RULE_EX_DATA* ad, long argl, void *argp)
{
int *counter=(int*)argp;
*ad=NULL;
ASSERT_GT(rule->serv_def_len, 4);
struct rule_ex_param* param=(struct rule_ex_param*)calloc(sizeof(struct rule_ex_param), 1);
param->ref_cnt=1;
pthread_mutex_init(&(param->lock), NULL);
sscanf(srv_def_large,"%*[^:]:%[^,],%d",param->name,&(param->id));
(*counter)++;
*ad=param;
return;
}
void ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp)
{
if(*ad==NULL)
{
return;
}
struct rule_ex_param* param=(struct rule_ex_param*)*ad;
pthread_mutex_lock(&(param->lock));
param->ref_cnt--;
if(param->ref_cnt>0)
{
pthread_mutex_unlock(&(param->lock));
return;
}
free(param);
return;
}
void ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp)
{
struct rule_ex_param* from_param=*((struct rule_ex_param**)from);
pthread_mutex_lock(&(from_param->lock));
from_param->ref_cnt++;
pthread_mutex_unlock(&(from_param->lock));
*((struct rule_ex_param**)to)=from_param;
return;
}
TEST(EX_DATA_INDEX, all)
{
#define EX_data_index
int ret=0;
int table_id=0, ex_data_counter=0;
scan_status_t mid=NULL;
struct Maat_rule_t result[4];
const char* url="i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
const char* table_name="HTTP_URL";
const char* expect_name="I have a name";
table_id=Maat_table_register(g_feather,table_name);
ASSERT_GT(table_id, 0);
int ex_param_idx=Maat_rule_get_ex_new_index(g_feather, "COMPILE_ALIAS",
ex_param_new, ex_param_free, ex_param_dup,
0, &ex_data_counter);
ASSERT_TRUE(ex_param_idx>=0);
EXPECT_EQ(ex_data_counter, 1);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url, strlen(url),
result,NULL, 4,
&mid, 0);
EXPECT_EQ(ret, 1);
void *ex_data=Maat_rule_get_ex_data(g_feather, result, ex_param_idx);
ASSERT_TRUE(ex_data!=NULL);
struct rule_ex_param* param=(struct rule_ex_param*)ex_data;
EXPECT_EQ(param->id, 7799);
EXPECT_EQ(strcmp(param->name, expect_name),0);
ex_param_free(0, NULL, NULL, &ex_data, 0, NULL);
Maat_clean_status(&mid);
return;
}
TEST(StreamFuzzyHash, Pure)
{
@@ -736,7 +827,6 @@ TEST_F(MaatFileTest, StreamFiles)
{
hit_cnt++;
}
printf("Stream Scan %s, ret=%d.\n",file_path,ret);
fclose(fp);
free(buff);
buff=NULL;