gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor table schema and runtime to support boolean expression plugin.

Browse Source
This commit is contained in:
zhengchao
2022-08-23 15:04:55 +08:00
parent 47a71011db
commit ddd36c2969
16 changed files with 646 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/CMakeLists.txt
View File

@@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp entry/FQDN_engine.cpp)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table_schema.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp entry/FQDN_engine.cpp)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
include_directories(/opt/MESA/include/MESA/)

214
src/entry/Maat_api.cpp
View File

@@ -577,9 +577,9 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
{
goto failed;
}
Maat_table_get_compile_table_name(feather->table_mgr, feather->compile_tn, sizeof(feather->compile_tn));
Maat_table_get_group2compile_table_name(feather->table_mgr, feather->group2compile_tn, sizeof(feather->group2compile_tn));
Maat_table_get_group2group_table_name(feather->table_mgr, feather->group2group_tn, sizeof(feather->group2group_tn));
Maat_table_manager_get_compile_table_name(feather->table_mgr, feather->compile_tn, sizeof(feather->compile_tn));
Maat_table_manager_get_group2compile_table_name(feather->table_mgr, feather->group2compile_tn, sizeof(feather->group2compile_tn));
Maat_table_manager_get_group2group_table_name(feather->table_mgr, feather->group2group_tn, sizeof(feather->group2group_tn));
feather->logger=logger;
feather->scan_thread_num=max_thread_num;
@@ -1035,7 +1035,7 @@ void Maat_burn_feather(Maat_feather_t feather)
int Maat_table_register(Maat_feather_t feather,const char* table_name)
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
return Maat_table_get_id_by_name(_feather->table_mgr, table_name);
return Maat_table_manager_get_id_by_name(_feather->table_mgr, table_name);
}
int Maat_table_callback_register(Maat_feather_t feather,short table_id,
Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
@@ -1047,7 +1047,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
int i=0,ret=0;
pthread_mutex_lock(&(_feather->background_update_mutex));
ret=Maat_table_add_callback_func(_feather->table_mgr, table_id, start, update, finish, u_para);
ret=Maat_table_manager_add_callback_func(_feather->table_mgr, table_id, start, update, finish, u_para);
if(ret<0)
{
pthread_mutex_unlock(&(_feather->background_update_mutex));
@@ -1117,7 +1117,7 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table
return -1;
}
pthread_mutex_lock(&(_feather->background_update_mutex));
idx=Maat_table_new_compile_rule_ex_index(_feather->table_mgr, compile_table_name,
idx=Maat_table_manager_new_compile_rule_ex_index(_feather->table_mgr, compile_table_name,
new_func,
free_func,
dup_func,
@@ -1128,7 +1128,7 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table
pthread_mutex_unlock(&(_feather->background_update_mutex));
return -1;
}
struct compile_ex_data_idx* compile_ex_desc=Maat_table_get_compile_rule_ex_desc(_feather->table_mgr, compile_table_name, idx);
struct compile_ex_data_idx* compile_ex_desc=Maat_table_manager_get_compile_rule_ex_desc(_feather->table_mgr, compile_table_name, idx);
if(_feather->scanner!=NULL)
{
@@ -1162,35 +1162,7 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa
return ad;
}
int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int ret=0;
struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id);
pthread_mutex_lock(&(_feather->background_update_mutex));
ret=Maat_table_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, key2index_func,argl, argp, _feather->logger);
if(ret<0)
{
pthread_mutex_unlock(&(_feather->background_update_mutex));
return -1;
}
struct Maat_table_runtime* table_rt=NULL;
if(_feather->scanner!=NULL)
{
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
Maat_table_runtime_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger);
}
pthread_mutex_unlock(&(_feather->background_update_mutex));
return 0;
}
MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
@@ -1207,7 +1179,7 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id
{
return NULL;
}
table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL);
table_desc=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key);
@@ -1223,18 +1195,17 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id
return exdata;
}
int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
int Maat_generic_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t *new_func,
Maat_plugin_EX_free_func_t *free_func,
Maat_plugin_EX_dup_func_t *dup_func,
long argl, void *argp)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int ret=-1;
struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id);
struct Maat_table_schema *table_schema=Maat_table_manager_get_by_id_raw(_feather->table_mgr, table_id);
pthread_mutex_lock(&(_feather->background_update_mutex));
ret=Maat_table_ip_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger);
ret=Maat_table_schema_set_EX_data_schema(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger);
if(ret<0)
{
@@ -1246,50 +1217,76 @@ int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id,
if(_feather->scanner!=NULL)
{
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
Maat_table_runtime_ip_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger);
Maat_table_runtime_commit_EX_data_schema(table_rt, table_schema, _feather->logger);
}
pthread_mutex_unlock(&(_feather->background_update_mutex));
return 0;
}
int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp)
{
int ret=-1;
ret=Maat_generic_plugin_EX_register(feather, table_id,
new_func, free_func, dup_func,
argl, argp);
return ret;
}
int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t *new_func,
Maat_plugin_EX_free_func_t *free_func,
Maat_plugin_EX_dup_func_t *dup_func,
long argl, void *argp)
{
int ret=-1;
ret=Maat_generic_plugin_EX_register(feather, table_id,
new_func, free_func, dup_func,
argl, argp);
return ret;
}
int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_new_func_t *new_func,
Maat_plugin_EX_free_func_t *free_func,
Maat_plugin_EX_dup_func_t *dup_func,
long argl, void *argp)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int ret=-1;
struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id);
pthread_mutex_lock(&(_feather->background_update_mutex));
ret=Maat_table_fqdn_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger);
if(ret<0)
{
pthread_mutex_unlock(&(_feather->background_update_mutex));
return -1;
}
struct Maat_table_runtime* table_rt=NULL;
if(_feather->scanner!=NULL)
{
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
Maat_table_runtime_fqdn_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger);
}
pthread_mutex_unlock(&(_feather->background_update_mutex));
return 0;
ret=Maat_generic_plugin_EX_register(feather, table_id,
new_func, free_func, dup_func,
argl, argp);
return ret;
}
int Maat_bool_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t *new_func,
Maat_plugin_EX_free_func_t *free_func,
Maat_plugin_EX_dup_func_t *dup_func,
long argl, void *argp)
{
int ret=-1;
ret=Maat_generic_plugin_EX_register(feather, table_id,
new_func, free_func, dup_func,
argl, argp);
return ret;
}
int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_feather_t *_feather=(_Maat_feather_t*)feather;
struct Maat_table_schema *table_schema=NULL;
struct Maat_table_runtime *table_rt=NULL;
int n_get=0;
struct timespec start,end;
struct timespec start, end;
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
@@ -1299,7 +1296,7 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc
return 0;
}
table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_IP_PLUGIN, NULL);
table_schema=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_IP_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt);
if(table_type!=TABLE_TYPE_IP_PLUGIN)
@@ -1332,12 +1329,12 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc
}
int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_feather_t *_feather=(_Maat_feather_t*)feather;
struct Maat_table_schema *table_schema=NULL;
struct Maat_table_runtime *table_rt=NULL;
int n_get=0;
struct timespec start,end;
struct timespec start, end;
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
@@ -1346,7 +1343,7 @@ int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const cha
{
return 0;
}
table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_FQDN_PLUGIN, NULL);
table_schema=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_FQDN_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt);
if(table_type!=TABLE_TYPE_FQDN_PLUGIN)
@@ -1368,11 +1365,48 @@ int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const cha
return n_get;
}
int Maat_bool_plugin_get_EX_data(Maat_feather_t feather, int table_id, unsigned long long item_id[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
{
struct _Maat_feather_t *_feather=(_Maat_feather_t*)feather;
struct Maat_table_schema *table_schema=NULL;
struct Maat_table_runtime *table_rt=NULL;
int n_get=0;
int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
,enum MAAT_CHARSET charset,const char* data,int data_len
,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num
,int* detail_ret,scan_status_t* mid,int thread_num)
struct timespec start, end;
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
if(_feather->scanner==NULL)
{
return 0;
}
table_schema=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_BOOL_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt);
if(table_type!=TABLE_TYPE_BOOL_PLUGIN)
{
return -1;
}
n_get=Maat_table_runtime_bool_plugin_get_N_ex_data(table_rt, table_schema, item_id, n_item, ex_data_array, n_ex_data);
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, 0);
}
else
{
Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, 0);
}
return n_get;
}
int Maat_full_scan_string_detail(Maat_feather_t feather, int table_id,
enum MAAT_CHARSET charset, const char* data, int data_len,
struct Maat_rule_t*result, int rule_num, struct Maat_hit_detail_t *hit_detail, int detail_num,
int* detail_ret, scan_status_t* mid, int thread_num)
{
int region_ret=0, compile_ret=0;
int district_id=DISTRICT_ANY;
@@ -1404,7 +1438,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
return 0;
}
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -1515,10 +1549,10 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
}
return compile_ret;
}
int Maat_full_scan_string(Maat_feather_t feather,int table_id
,enum MAAT_CHARSET charset,const char* data,int data_len
,struct Maat_rule_t*result,int* found_pos,int rule_num
,scan_status_t* mid,int thread_num)
int Maat_full_scan_string(Maat_feather_t feather, int table_id,
enum MAAT_CHARSET charset, const char* data, int data_len,
struct Maat_rule_t*result, int* found_pos, int rule_num,
scan_status_t* mid, int thread_num)
{
int detail_ret=0,compile_ret=0;
compile_ret=Maat_full_scan_string_detail(feather,table_id,
@@ -1550,7 +1584,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
scan_staus_count_inc(_mid);
int virtual_table_id=0;
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_INTERVAL, &virtual_table_id);
p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_INTERVAL, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -1656,7 +1690,7 @@ int Maat_similar_scan_string(Maat_feather_t feather, int table_id,
scan_staus_count_inc(_mid);
int virtual_table_id=0;
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -1809,14 +1843,14 @@ static int IP_composition_scan(const struct ipaddr* addr, unsigned short int pro
}
else
{
child_table_id=Maat_table_get_child_id(table_mgr, parent_table_id, child_type);
child_table_id=Maat_table_manager_get_child_id(table_mgr, parent_table_id, child_type);
}
if(child_table_id<0)
{
return 0;
}
Maat_table_schema* real_table=Maat_table_get_scan_by_id(table_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id);
Maat_table_schema* real_table=Maat_table_manager_get_scan_by_id(table_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id);
if(real_table==NULL)
{
return 0;
@@ -1868,7 +1902,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
scan_staus_count_inc(_mid);
int virtual_table_id=0;
enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_INVALID;
table_type=Maat_table_get_type_by_id(_feather->table_mgr, table_id);
table_type=Maat_table_manager_get_type_by_id(_feather->table_mgr, table_id);
if(table_type==TABLE_TYPE_INVALID)
{
_feather->scan_err_cnt++;
@@ -1981,7 +2015,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
struct Maat_table_schema *p_table=NULL;
int virtual_table_id=0;
assert(thread_num<_feather->scan_thread_num);
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -2278,7 +2312,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
sfh_instance_t * tmp_fuzzy_handle=NULL;
struct Maat_table_schema *p_table=NULL;
int virtual_table_id=0;
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;

20
src/entry/Maat_command.cpp
View File

@@ -732,7 +732,7 @@ FULL_UPDATE:
}
if(table_mgr)
{
table_id=Maat_table_get_id_by_name(table_mgr, s_rule_array[full_idx].table_name);
table_id=Maat_table_manager_get_id_by_name(table_mgr, s_rule_array[full_idx].table_name);
if(table_id<0)//Unrecognized table.
{
continue;
@@ -1536,7 +1536,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list,
{
continue;
}
p_table=Maat_table_get_desc_by_name(feather->table_mgr, rule_list[i].table_name);
p_table=Maat_table_manager_get_desc_by_name(feather->table_mgr, rule_list[i].table_name);
if(!p_table||p_table->table_type!=TABLE_TYPE_PLUGIN)
{
continue;
@@ -1808,19 +1808,19 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
{
continue;
}
table_id=Maat_table_get_id_by_name(feather->table_mgr, rule_list[i].table_name);
table_id=Maat_table_manager_get_id_by_name(feather->table_mgr, rule_list[i].table_name);
if(table_id<0)//Unrecognized table.
{
no_table_num++;
continue;
}
table_type=Maat_table_get_type_by_id(feather->table_mgr, table_id);
table_type=Maat_table_manager_get_type_by_id(feather->table_mgr, table_id);
if(rule_list[i].op==MAAT_OP_DEL)
{
scan_type=Maat_table_get_scan_type(table_type);
table_schema=Maat_table_get_scan_by_id(feather->table_mgr, table_id, scan_type, NULL);
valid_column=Maat_table_xx_plugin_table_get_valid_flag_column(table_schema);
table_schema=Maat_table_manager_get_scan_by_id(feather->table_mgr, table_id, scan_type, NULL);
valid_column=Maat_table_schema_get_valid_flag_column(table_schema);
ret=invalidate_line(rule_list[i].table_line, table_type, valid_column);
if(ret<0)
{
@@ -1935,7 +1935,7 @@ int Maat_command_raw_set_lines(Maat_feather_t feather,const struct Maat_cmd_line
s_rule=ALLOC(struct serial_rule_t, n_line);
for(i=0;i<n_line;i++)
{
table_id=Maat_table_get_id_by_name(_feather->table_mgr, line_rule[i]->table_name);
table_id=Maat_table_manager_get_id_by_name(_feather->table_mgr, line_rule[i]->table_name);
if(table_id<0)
{
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command,
@@ -1998,7 +1998,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_
s_rule=ALLOC(struct serial_rule_t, line_num);
for(i=0;i<line_num;i++)
{
table_id=Maat_table_get_id_by_name(_feather->table_mgr, line_rule[i]->table_name);
table_id=Maat_table_manager_get_id_by_name(_feather->table_mgr, line_rule[i]->table_name);
if(table_id<0)
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command
@@ -2008,7 +2008,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_
ret=-1;
goto error_out;
}
p_table=Maat_table_get_by_id_raw(_feather->table_mgr, table_id);
p_table=Maat_table_manager_get_by_id_raw(_feather->table_mgr, table_id);
if(!p_table)
{
ret=-1;
@@ -2016,7 +2016,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_
}
int valid_flag_column=0;
valid_flag_column=Maat_table_xx_plugin_table_get_valid_flag_column(p_table);
valid_flag_column=Maat_table_schema_get_valid_flag_column(p_table);
if(valid_flag_column<0)
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command

2
src/entry/Maat_ex_data.cpp
View File

@@ -1,5 +1,5 @@
#include "Maat_ex_data.h"
#include "Maat_table.h"
#include "Maat_table_schema.h"
#include "Maat_utils.h"
#include "Maat_garbage_collection.h"

6
src/entry/Maat_hierarchy.cpp
View File

@@ -998,7 +998,7 @@ static struct bool_matcher* Maat_hierarchy_build_bool_matcher(struct Maat_hierar
"No bool expression to build.");
goto error_out;
}
bm=bool_matcher_new(bool_expr_array, expr_cnt, hier->thread_num, &mem_size);
bm=bool_matcher_new(bool_expr_array, expr_cnt, &mem_size);
if(bm!=NULL)
{
MESA_handle_runtime_log(hier->logger, RLOG_LV_INFO, module_maat_hierarchy,
@@ -1365,7 +1365,7 @@ size_t Maat_hierarchy_get_hit_paths(struct Maat_hierarchy* hier, struct Maat_hie
}
}
bool_match_ret=bool_matcher_match(hier->bm, mid->thread_num,
bool_match_ret=bool_matcher_match(hier->bm,
(unsigned long long*)utarray_eltptr(mid->_all_hit_clause_array, 0), utarray_len(mid->_all_hit_clause_array),
expr_match, MAX_SCANNER_HIT_NUM);
for(i=0; i<bool_match_ret; i++)
@@ -1516,7 +1516,7 @@ int Maat_hierarchy_region_compile(struct Maat_hierarchy* hier, struct Maat_hiera
mid->this_scan_region_hit_cnt=0;
return 0;
}
bool_match_ret=bool_matcher_match(hier->bm, mid->thread_num,
bool_match_ret=bool_matcher_match(hier->bm,
(unsigned long long*)utarray_eltptr(mid->_all_hit_clause_array, 0), utarray_len(mid->_all_hit_clause_array),
expr_match, MAX_SCANNER_HIT_NUM);
for(i=0; i<bool_match_ret && ud_result_cnt<ud_array_sz; i++)

42
src/entry/Maat_rule.cpp
View File

@@ -57,7 +57,7 @@ extern "C"
}
#endif
int MAAT_FRAME_VERSION_3_6_11_20220713=1;
int MAAT_FRAME_VERSION_3_7_0_20220823=1;
int is_valid_table_name(const char* str)
{
@@ -1495,7 +1495,7 @@ void update_group2compile_rule(struct Maat_table_schema* table, const char* tabl
}
if(is_valid_table_name(virtual_table_name))
{
db_g2c_rule.virtual_table_id=Maat_table_get_id_by_name(table_mgr, virtual_table_name);
db_g2c_rule.virtual_table_id=Maat_table_manager_get_id_by_name(table_mgr, virtual_table_name);
if(db_g2c_rule.virtual_table_id<0)
{
//This happens when one data source (e.g. redis) is consumed by multiple Maat instance.
@@ -2301,13 +2301,20 @@ void update_xx_plugin_table(struct Maat_table_schema* table_schema, const char*
return;
}
}
if(table_schema->table_type==TABLE_TYPE_IP_PLUGIN)
switch(table_schema->table_type)
{
Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger);
}
else //TABLE_TYPE_FQDN_PLUGIN
{
Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger);
case TABLE_TYPE_IP_PLUGIN:
Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger);
break;
case TABLE_TYPE_FQDN_PLUGIN:
Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger);
break;
case TABLE_TYPE_BOOL_PLUGIN:
Maat_table_runtime_bool_plugin_new_row(table_rt, table_schema, table_row, logger);
break;
default:
assert(0);
break;
}
scanner->xx_plugin_rule_to_update_cnt++;
return;
@@ -2361,10 +2368,10 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
}
break;
case TABLE_TYPE_PLUGIN:
Maat_table_runtime_plugin_update_commit(table_rt);
Maat_table_runtime_plugin_commit_update(table_rt);
break;
case TABLE_TYPE_IP_PLUGIN:
ret=Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt);
ret=Maat_table_runtime_ip_plugin_commit_update(table_rt);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
@@ -2374,12 +2381,21 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
break;
case TABLE_TYPE_FQDN_PLUGIN:
ret=Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(table_rt);
ret=Maat_table_runtime_fqdn_plugin_commit_update(table_rt);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"FQDN plugin table_id %d build failed.", i);
}
break;
case TABLE_TYPE_BOOL_PLUGIN:
ret=Maat_table_runtime_bool_plugin_commit_update(table_rt);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Boolean expression plugin table_id %d build failed.", i);
}
break;
default:
@@ -2501,13 +2517,13 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
scanner=feather->scanner;
}
// MESA_handle_runtime_log(feather->logger, RLOG_LV_DEBUG, maat_module, "Maat table %s input: %s", table_name, line);
p_table=Maat_table_get_desc_by_name(feather->table_mgr, table_name);
p_table=Maat_table_manager_get_desc_by_name(feather->table_mgr, table_name);
if(!p_table)
{
MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s", table_name);
return -1;
}
Maat_table_set_updating_name(p_table, table_name);
Maat_table_schema_set_updating_name(p_table, table_name);
switch(p_table->table_type)
{

6
src/entry/Maat_stat.cpp
View File

@@ -1,5 +1,5 @@
#include "Maat_rule_internal.h"
#include "Maat_table.h"
#include "Maat_table_schema.h"
#include "Maat_garbage_collection.h"
#include "alignment_int64.h"
#include <time.h>
@@ -128,7 +128,7 @@ void maat_stat_init(struct _Maat_feather_t* feather)
size_t max_table_num=Maat_table_manager_get_size(feather->table_mgr);
for(i=0; i<(int)max_table_num; i++)
{
p_table=Maat_table_get_by_id_raw(feather->table_mgr, i);
p_table=Maat_table_manager_get_by_id_raw(feather->table_mgr, i);
if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN
||p_table->table_type==TABLE_TYPE_GROUP
||p_table->table_type==TABLE_TYPE_COMPILE)
@@ -203,7 +203,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
table_hit_cnt=0;
table_regex_ipv6_num=0;
p_table=Maat_table_get_by_id_raw(feather->table_mgr, i);
p_table=Maat_table_manager_get_by_id_raw(feather->table_mgr, i);
if(p_table==NULL)
{
continue;

304
src/entry/Maat_table_runtime.cpp
View File

@@ -99,9 +99,9 @@ static void destroy_digest_rule(GIE_digest_t*rule)
rule=NULL;
return;
}
struct FQDN_rule* fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_len, int is_suffix_match)
struct FQDN_rule *fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_len, int is_suffix_match)
{
struct FQDN_rule* fqdn_rule=ALLOC(struct FQDN_rule, 1);
struct FQDN_rule *fqdn_rule=ALLOC(struct FQDN_rule, 1);
//Todo: check FQDN format with regex ^([a-zA-Z0-9._-])+$
if(fqdn[0]=='.')
{
@@ -119,7 +119,7 @@ struct FQDN_rule* fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_l
fqdn_rule->id=id;
return fqdn_rule;
}
void fqdn_rule_free(struct FQDN_rule* fqdn_rule)
void fqdn_rule_free(struct FQDN_rule *fqdn_rule)
{
assert(fqdn_rule->user_tag==NULL);
free(fqdn_rule->FQDN);
@@ -127,6 +127,23 @@ void fqdn_rule_free(struct FQDN_rule* fqdn_rule)
free(fqdn_rule);
return;
}
struct bool_expr *bool_expr_new(unsigned int id, unsigned long long item_id[], size_t n_item)
{
struct bool_expr *expr=ALLOC(struct bool_expr, 1);
expr->expr_id=id;
size_t i=0;
for(i=0; i<n_item; i++)
{
expr->items[i].item_id=item_id[i];
expr->items[i].not_flag=0;
}
return expr;
}
void bool_expr_free(struct bool_expr *expr)
{
free(expr);
return;
}
void _notype_fqdn_rule_free(void* p)
{
fqdn_rule_free((struct FQDN_rule*)p);
@@ -147,7 +164,7 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->plugin.ex_schema.key2index_func,
NULL);
if(table_schema->plugin.have_exdata)
if(table_schema->plugin.ex_schema.set_flag)
{
EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema);
}
@@ -156,7 +173,7 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->ip_plugin.ex_schema.key2index_func,
free);
if(table_schema->ip_plugin.have_exdata)
if(table_schema->ip_plugin.ex_schema.set_flag)
{
EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema);
}
@@ -165,11 +182,20 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
table_rt->fqdn_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->fqdn_plugin.ex_schema.key2index_func,
_notype_fqdn_rule_free);
if(table_schema->fqdn_plugin.have_exdata)
if(table_schema->fqdn_plugin.ex_schema.set_flag)
{
EX_data_rt_set_schema(table_rt->fqdn_plugin.ex_data_rt, &table_schema->fqdn_plugin.ex_schema);
}
break;
case TABLE_TYPE_BOOL_PLUGIN:
table_rt->bool_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->fqdn_plugin.ex_schema.key2index_func,
(void (*)(void*))bool_expr_free);
if(table_schema->bool_plugin.ex_schema.set_flag)
{
EX_data_rt_set_schema(table_rt->bool_plugin.ex_data_rt, &table_schema->bool_plugin.ex_schema);
}
break;
default:
break;
}
@@ -249,7 +275,7 @@ struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat
table_rt_mgr->table_rt=ALLOC(struct Maat_table_runtime*, table_rt_mgr->n_table_rt);
for(i=0; i<table_rt_mgr->n_table_rt; i++)
{
table_desc=Maat_table_get_by_id_raw(table_manager, i);
table_desc=Maat_table_manager_get_by_id_raw(table_manager, i);
if(!table_desc)
{
continue;
@@ -305,25 +331,10 @@ const char* Maat_table_runtime_plugin_get_cached_row(struct Maat_table_runtime*
line=EX_data_rt_cached_row_get(plugin_rt->ex_data_rt, Nth_row);
return line;
}
int Maat_table_runtime_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger)
{
size_t i=0;
const char* row=NULL;
EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema);
for(i=0; i<EX_data_rt_get_cached_row_num(table_rt->plugin.ex_data_rt); i++)
{
row=EX_data_rt_cached_row_get(table_rt->plugin.ex_data_rt, i);
Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger);
}
EX_data_rt_update_commit(table_rt->plugin.ex_data_rt);
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->plugin.ex_data_rt);
return 0;
}
MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* key)
{
MAAT_RULE_EX_DATA ex_data=NULL;
if(!table_schema->plugin.have_exdata)
if(!table_schema->plugin.ex_schema.set_flag)
{
assert(0);
return NULL;
@@ -342,7 +353,7 @@ void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, stru
struct plugin_runtime* plugin_rt=&table_rt->plugin;
ret=Maat_helper_read_column(row, plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
plugin_rt->acc_line_num++;
if(plugin_schema->have_exdata)
if(plugin_schema->ex_schema.set_flag)
{
ret=get_column_pos(row, plugin_schema->key_column, &key_offset, &key_len);
if(ret<0)
@@ -368,14 +379,14 @@ void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, stru
plugin_schema->cb_plug[i].update(table_schema->table_id, row, plugin_schema->cb_plug[i].u_para);
}
}
if(!plugin_schema->have_exdata && !plugin_schema->cb_plug_cnt)
if(!plugin_schema->ex_schema.set_flag && !plugin_schema->cb_plug_cnt)
{
EX_data_rt_cache_row_put(plugin_rt->ex_data_rt, row);
}
return;
}
void Maat_table_runtime_plugin_update_commit(struct Maat_table_runtime* table_rt)
void Maat_table_runtime_plugin_commit_update(struct Maat_table_runtime* table_rt)
{
EX_data_rt_update_commit(table_rt->plugin.ex_data_rt);
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->plugin.ex_data_rt);
@@ -420,7 +431,7 @@ void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt,
size_t fqdn_offset=0, fqdn_len=0;
struct FQDN_rule* fqdn_rule=NULL;
int ret=0;
if(fqdn_plugin_schema->have_exdata)
if(fqdn_plugin_schema->ex_schema.set_flag)
{
ret=Maat_helper_read_column(row, fqdn_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
if(ret<0)
@@ -479,7 +490,7 @@ void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt,
return;
}
int Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(struct Maat_table_runtime* table_rt)
int Maat_table_runtime_fqdn_plugin_commit_update(struct Maat_table_runtime* table_rt)
{
struct FQDN_engine* new_fqdn_engine=NULL, *old_fqdn_engine=NULL;
struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin;
@@ -518,23 +529,80 @@ int Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(struct Maat_table_runtime
table_rt->fqdn_plugin.changed_flag=0;
return ret;
}
int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger)
struct EX_data_rt *Maat_table_runtime_get_EX_data_rt(struct Maat_table_runtime* table_rt)
{
struct EX_data_rt *rt=NULL;
switch(table_rt->table_type)
{
case TABLE_TYPE_PLUGIN:
rt=table_rt->plugin.ex_data_rt;
break;
case TABLE_TYPE_IP_PLUGIN:
rt=table_rt->ip_plugin.ex_data_rt;
break;
case TABLE_TYPE_FQDN_PLUGIN:
rt=table_rt->fqdn_plugin.ex_data_rt;
break;
case TABLE_TYPE_BOOL_PLUGIN:
rt=table_rt->bool_plugin.ex_data_rt;
break;
default:
break;
}
return rt;
}
int Maat_table_runtime_commit_EX_data_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger)
{
size_t i=0;
const char* row=NULL;
struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin);
EX_data_rt_set_schema(fqdn_plugin_rt->ex_data_rt, &table_schema->fqdn_plugin.ex_schema);
for(i=0; i<EX_data_rt_get_cached_row_num(fqdn_plugin_rt->ex_data_rt); i++)
assert(table_rt->table_type==table_schema->table_type);
struct EX_data_rt *ex_data_rt=NULL;
struct EX_data_schema *ex_data_schema=Maat_table_schema_get_EX_data_schema(table_schema);
ex_data_rt=Maat_table_runtime_get_EX_data_rt(table_rt);
EX_data_rt_set_schema(ex_data_rt, ex_data_schema);
for(i=0; i<EX_data_rt_get_cached_row_num(ex_data_rt); i++)
{
row=EX_data_rt_cached_row_get(fqdn_plugin_rt->ex_data_rt, i);
Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, row, logger);
row=EX_data_rt_cached_row_get(ex_data_rt, i);
switch(table_rt->table_type)
{
case TABLE_TYPE_PLUGIN:
Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger);
break;
case TABLE_TYPE_IP_PLUGIN:
Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, row, logger);
break;
case TABLE_TYPE_FQDN_PLUGIN:
Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, row, logger);
break;
case TABLE_TYPE_BOOL_PLUGIN:
Maat_table_runtime_bool_plugin_new_row(table_rt, table_schema, row, logger);
break;
default:
break;
}
}
EX_data_rt_clear_row_cache(ex_data_rt);
switch(table_rt->table_type)
{
case TABLE_TYPE_PLUGIN:
Maat_table_runtime_plugin_commit_update(table_rt);
break;
case TABLE_TYPE_IP_PLUGIN:
Maat_table_runtime_ip_plugin_commit_update(table_rt);
break;
case TABLE_TYPE_FQDN_PLUGIN:
Maat_table_runtime_fqdn_plugin_commit_update(table_rt);
break;
case TABLE_TYPE_BOOL_PLUGIN:
Maat_table_runtime_bool_plugin_commit_update(table_rt);
break;
default:
break;
}
EX_data_rt_clear_row_cache(fqdn_plugin_rt->ex_data_rt);
Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(table_rt);
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(fqdn_plugin_rt->ex_data_rt);
return 0;
}
int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size)
{
struct FQDN_match results[size];
@@ -555,6 +623,145 @@ int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* tabl
return n_result;
}
void Maat_table_runtime_bool_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
{
struct bool_plugin_table_schema *bool_plugin_schema=&(table_schema->bool_plugin);
struct bool_plugin_runtime *bool_plugin_rt=&(table_rt->bool_plugin);
size_t is_valid_offset=0, valid_len=0;
size_t row_id_offset=0, row_id_len=0;
size_t bool_expr_offset=0, bool_expr_len=0;
struct bool_expr *expr=NULL;
int ret=0;
unsigned long long item_id[MAX_ITEMS_PER_BOOL_EXPR];
size_t n_item=0;
if(bool_plugin_schema->ex_schema.set_flag)
{
ret=Maat_helper_read_column(row, bool_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"bool_plugin EX data process error: cannot find is_valid column %d of %s",
bool_plugin_schema->valid_flag_column, row);
return;
}
ret=Maat_helper_read_column(row, bool_plugin_schema->row_id_column, &row_id_offset, &row_id_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"bool_plugin EX data process error: cannot find row id column %d of %s",
bool_plugin_schema->row_id_column, row);
return;
}
ret=Maat_helper_read_column(row, bool_plugin_schema->bool_expr_column, &bool_expr_offset, &bool_expr_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"fqdn_plugin EX data process error: cannot find bool_expr column %d of %s",
bool_plugin_schema->bool_expr_column, row);
return;
}
char *token=NULL, *sub_token=NULL, *saveptr;
char expr_buffer[256];
memset(expr_buffer, 0, sizeof(expr_buffer));
memcpy(expr_buffer, row+bool_expr_offset, bool_expr_len);
for (token = expr_buffer; ; token= NULL)
{
sub_token= strtok_r(token, "&", &saveptr);
if (sub_token == NULL)
break;
ret=sscanf(sub_token, "%llu", item_id+n_item);
n_item++;
if(ret!=1||n_item==MAX_ITEMS_PER_BOOL_EXPR)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"fqdn_plugin EX data process error: invalid format of bool_expr column %d of %s",
bool_plugin_schema->bool_expr_column, row);
return;
}
}
if(atoi(row+is_valid_offset)==1)//add
{
expr=bool_expr_new((unsigned int)atoi(row+row_id_offset), item_id, n_item);
if(expr==NULL)
ret=EX_data_rt_row2EX_data(bool_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, expr, logger);
if(ret<0)
{
bool_expr_free(expr);
expr=NULL;
}
}
else
{
EX_data_rt_delete_by_row(bool_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, logger);
}
}
else
{
EX_data_rt_cache_row_put(bool_plugin_rt->ex_data_rt, row);
table_rt->origin_rule_num=EX_data_rt_get_cached_row_num(bool_plugin_rt->ex_data_rt);
}
bool_plugin_rt->changed_flag=1;
return;
}
int Maat_table_runtime_bool_plugin_commit_update(struct Maat_table_runtime* table_rt)
{
struct bool_matcher* new_bool_matcher=NULL, *old_bool_matcher=NULL;
struct bool_plugin_runtime* bool_plugin_rt=&table_rt->bool_plugin;
assert(table_rt->table_type==TABLE_TYPE_BOOL_PLUGIN);
struct EX_data_container **exc_array=NULL;
struct bool_expr* exprs=NULL;
size_t expr_cnt=0, i=0, ret=0, mem_usage=0;
if(!bool_plugin_rt->changed_flag)
{
return ret;
}
expr_cnt=EX_data_rt_list_updating_ex_containers(bool_plugin_rt->ex_data_rt, &exc_array);
exprs=ALLOC(struct bool_expr, expr_cnt);
for(i=0; i<expr_cnt; i++)
{
exprs[i]=*((struct bool_expr *)(EX_data_container_get_user_data(exc_array[i])));
assert(exprs[i].user_tag==exc_array[i]||exprs[i].user_tag==NULL);
exprs[i].user_tag=exc_array[i];
}
if(expr_cnt>0)
{
new_bool_matcher=bool_matcher_new(exprs, expr_cnt, &mem_usage);
if(!new_bool_matcher)
{
ret=-1;
}
}
old_bool_matcher=bool_plugin_rt->matcher;
bool_plugin_rt->matcher=new_bool_matcher;
Maat_garbage_bagging(table_rt->ref_garbage_bin, old_bool_matcher, (void (*)(void*))bool_matcher_free);
EX_data_rt_update_commit(bool_plugin_rt->ex_data_rt);
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(bool_plugin_rt->ex_data_rt);
free(exprs);
free(exc_array);
table_rt->fqdn_plugin.changed_flag=0;
return ret;
}
int Maat_table_runtime_bool_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, unsigned long long item_id[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size)
{
struct bool_expr_match results[size];
int n_result=0, i=0;
if(table_rt->table_type!=TABLE_TYPE_BOOL_PLUGIN)
{
return -1;
}
if(!table_rt->bool_plugin.matcher)
{
return 0;
}
n_result=bool_matcher_match(table_rt->bool_plugin.matcher, item_id, n_item, results, size);
for(i=0; i<n_result; i++)
{
ex_data_array[i]=EX_data_rt_get_EX_data_by_container(table_rt->bool_plugin.ex_data_rt, (struct EX_data_container *)results[i].user_tag);
}
return n_result;
}
int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt)
{
long i=0,data_size=0;
@@ -621,7 +828,7 @@ int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt)
return q_cnt;
}
int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt)
int Maat_table_runtime_ip_plugin_commit_update(struct Maat_table_runtime* table_rt)
{
struct ip_matcher* new_ip_matcher=NULL, *old_ip_matcher=NULL;
size_t rule_cnt=0;
@@ -673,7 +880,7 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s
size_t row_id_offset=0, row_id_len=0;
struct ip_rule* ip_rule=NULL;
int ret=0;
if(ip_plugin_schema->have_exdata)
if(ip_plugin_schema->ex_schema.set_flag)
{
ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
if(ret<0)
@@ -724,23 +931,6 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s
ip_plugin_rt->changed_flag=1;
return;
}
int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger)
{
size_t i=0;
const char* row=NULL;
struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin);
EX_data_rt_set_schema(ip_plugin_rt->ex_data_rt, &table_schema->ip_plugin.ex_schema);
for(i=0; i<EX_data_rt_get_cached_row_num(ip_plugin_rt->ex_data_rt); i++)
{
row=EX_data_rt_cached_row_get(ip_plugin_rt->ex_data_rt, i);
Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, row, logger);
}
EX_data_rt_clear_row_cache(ip_plugin_rt->ex_data_rt);
Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt);
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(ip_plugin_rt->ex_data_rt);
return 0;
}
int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size)
{
struct scan_result results[size];

315
src/entry/Maat_table.cpp → src/entry/Maat_table_schema.cpp
View File

@@ -1,4 +1,4 @@
#include "Maat_table.h"
#include "Maat_table_schema.h"
#include "map_str2int.h"
#include "Maat_utils.h"
#include "cJSON.h"
@@ -49,6 +49,9 @@ enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type)
case TABLE_TYPE_FQDN_PLUGIN:
ret=SCAN_TYPE_FQDN_PLUGIN;
break;
case TABLE_TYPE_BOOL_PLUGIN:
ret=SCAN_TYPE_BOOL_PLUGIN;
break;
case TABLE_TYPE_COMPILE:
ret=SCAN_TYPE_NONE;
break;
@@ -140,7 +143,8 @@ int _read_integer_arrary(char* string, int *array, int size)
}
#define COLUMN_PLUGIN_SCHEMA_JSON 4
#define COLUMN_IP_PLUGIN_SCHEMA_JSON 4
#define COLUMN_FQDN_PLUGIN_SHCEMA_JSON 4
#define COLUMN_FQDN_PLUGIN_SCHEMA_JSON 4
#define COLUMN_BOOL_PLUGIN_SCHEMA_JSON 4
#define COLUMN_COMPOSITION_SCHEMA_JSON 4
#define COLUMN_VIRUTAL_SCHEMA_JSON 4
int read_virtual_table_schema(struct Maat_table_manager* table_mgr, const char* line, struct Maat_table_schema* table, MESA_htable_handle reserved_word_map)
@@ -387,7 +391,7 @@ int read_fqdn_plugin_table_schema(const char* line, struct Maat_table_schema* p)
struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(p->fqdn_plugin);
copy_line=_maat_strdup(line);
ret=get_column_pos(copy_line, COLUMN_FQDN_PLUGIN_SHCEMA_JSON, &offset, &len);
ret=get_column_pos(copy_line, COLUMN_FQDN_PLUGIN_SCHEMA_JSON, &offset, &len);
if(ret<0)
{
goto error_out;
@@ -454,6 +458,76 @@ error_out:
free(copy_line);
return -1;
}
int read_bool_plugin_table_schema(const char* line, struct Maat_table_schema* p)
{
int ret=0, read_cnt=0;
size_t offset=0, len=0;
cJSON* json=NULL, *tmp=NULL;
char* copy_line=NULL, *schema_json=NULL;
struct bool_plugin_table_schema* bool_plugin_schema=&(p->bool_plugin);
copy_line=_maat_strdup(line);
ret=get_column_pos(copy_line, COLUMN_BOOL_PLUGIN_SCHEMA_JSON, &offset, &len);
if(ret<0)
{
goto error_out;
}
if(offset+len<strlen(copy_line))
{
copy_line[offset+len]='\0';
}
schema_json=copy_line+offset;
json=cJSON_Parse(schema_json);
if(!json)
{
goto error_out;
}
tmp=cJSON_GetObjectItem(json, "row_id");
if(tmp!=NULL && tmp->type==cJSON_Number)
{
bool_plugin_schema->row_id_column=tmp->valueint;
read_cnt++;
}
tmp=cJSON_GetObjectItem(json, "bool_expr");
if(tmp!=NULL && tmp->type==cJSON_Number)
{
bool_plugin_schema->bool_expr_column=tmp->valueint;
read_cnt++;
}
tmp=cJSON_GetObjectItem(json, "valid");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
bool_plugin_schema->valid_flag_column=tmp->valueint;
read_cnt++;
}
bool_plugin_schema->rule_tag_column=-1;
tmp=cJSON_GetObjectItem(json, "tag");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
bool_plugin_schema->rule_tag_column=tmp->valueint;
//read_cnt++; Tag is optional, so NOT ++ intentionally.
}
cJSON_Delete(json);
free(copy_line);
if(read_cnt<4)
{
return -1;
}
else
{
return 0;
}
error_out:
free(copy_line);
return -1;
}
int read_composition_table_schema(struct Maat_table_manager* table_mgr, const char* line, struct Maat_table_schema* p, MESA_htable_handle string2int_map)
@@ -582,6 +656,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN);
maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN);
maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN);
maat_kv_register(reserved_word_map, "bool_plugin", TABLE_TYPE_BOOL_PLUGIN);
maat_kv_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL);
maat_kv_register(reserved_word_map, "interval", TABLE_TYPE_INTERVAL);
maat_kv_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS);
@@ -687,6 +762,16 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
goto invalid_table;
}
break;
case TABLE_TYPE_BOOL_PLUGIN:
ret=read_bool_plugin_table_schema(line, p);
if(ret<0)
{
fprintf(stderr, "Maat read table info %s line %d error: illegal bool_plugin table schema.\n", table_info_path, i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Maat read table info %s line %d error: illegal bool_plugin table schema.", table_info_path, i);
goto invalid_table;
}
break;
case TABLE_TYPE_COMPOSITION:
ret=read_composition_table_schema(table_mgr, line, p, reserved_word_map);
if(ret<0)
@@ -778,7 +863,7 @@ size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr)
{
return table_mgr->table_cnt;
}
int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
int Maat_table_manager_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
{
int i=0;
for(i=0; i< MAX_TABLE_NUM; i++)
@@ -791,7 +876,7 @@ int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char
}
return 0;
}
int Maat_table_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
int Maat_table_manager_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
{
int i=0;
for(i=0; i< MAX_TABLE_NUM; i++)
@@ -804,7 +889,7 @@ int Maat_table_get_group2compile_table_name(struct Maat_table_manager* table_mgr
}
return 0;
}
int Maat_table_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
int Maat_table_manager_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
{
int i=0;
for(i=0; i< MAX_TABLE_NUM; i++)
@@ -817,20 +902,7 @@ int Maat_table_get_group2group_table_name(struct Maat_table_manager* table_mgr,
}
return 0;
}
const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id)
{
if(table_id>MAX_TABLE_NUM)
{
return NULL;
}
if(table_mgr->p_table_info[table_id])
{
return table_mgr->p_table_info[table_id]->table_name[0];
}
return NULL;
}
enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id)
enum MAAT_TABLE_TYPE Maat_table_manager_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id)
{
if(table_id>MAX_TABLE_NUM)
{
@@ -843,17 +915,8 @@ enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_
return TABLE_TYPE_INVALID;
}
struct Maat_table_schema * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id)
{
if(table_id>MAX_TABLE_NUM||table_id<0)
{
return NULL;
}
return table_mgr->p_table_info[table_id];
}
struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id)
struct Maat_table_schema * Maat_table_manager_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id)
{
enum MAAT_SCAN_TYPE tab_scan_type;
struct Maat_table_schema **p_table_info=table_mgr->p_table_info;
@@ -890,7 +953,7 @@ struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager*
}
return p_physical_table;
}
int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name)
int Maat_table_manager_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name)
{
int table_id=-1,ret=0;
ret=maat_kv_read(table_mgr->map_tablename2id, table_name, &table_id);
@@ -903,7 +966,7 @@ int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char*
return -1;
}
}
int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
int Maat_table_manager_add_callback_func(struct Maat_table_manager* table_mgr,
int table_id,
Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
Maat_update_callback_t *update,//table line ,u_para
@@ -911,7 +974,7 @@ int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
void* u_para)
{
int idx=0;
struct Maat_table_schema *p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL);
struct Maat_table_schema *p_table=Maat_table_manager_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL);
struct plugin_table_schema *plugin_desc=&(p_table->plugin);
if(p_table==NULL)
{
@@ -931,17 +994,17 @@ int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
return 1;
}
struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx)
struct compile_ex_data_idx* Maat_table_manager_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx)
{
int table_id=-1;
struct Maat_table_schema *p_table=NULL;
table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name);
table_id=Maat_table_manager_get_id_by_name(table_mgr, compile_table_name);
if(table_id<0)
{
return NULL;
}
p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL);
p_table=Maat_table_manager_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL);
if(!p_table)
{
return NULL;
@@ -953,7 +1016,7 @@ struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_tabl
return NULL;
}
int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name,
int Maat_table_manager_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name,
Maat_rule_EX_new_func_t *new_func,
Maat_rule_EX_free_func_t* free_func,
Maat_rule_EX_dup_func_t* dup_func,
@@ -961,12 +1024,12 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c
{
int table_id=-1;
struct Maat_table_schema *p_table=NULL;
table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name);
table_id=Maat_table_manager_get_id_by_name(table_mgr, compile_table_name);
if(table_id<0)
{
return -1;
}
p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL);
p_table=Maat_table_manager_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL);
if(!p_table)
{
return -1;
@@ -991,122 +1054,66 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c
return idx;
}
void Maat_table_EX_data_schema_set(struct EX_data_schema* ex_schema,
struct EX_data_schema *Maat_table_schema_get_EX_data_schema(struct Maat_table_schema *table_schema)
{
struct EX_data_schema *ex_schema=NULL;
switch(table_schema->table_type)
{
case TABLE_TYPE_PLUGIN:
ex_schema=&table_schema->plugin.ex_schema;
break;
case TABLE_TYPE_IP_PLUGIN:
ex_schema=&table_schema->ip_plugin.ex_schema;
break;
case TABLE_TYPE_FQDN_PLUGIN:
ex_schema=&table_schema->fqdn_plugin.ex_schema;
break;
case TABLE_TYPE_BOOL_PLUGIN:
ex_schema=&table_schema->bool_plugin.ex_schema;
break;
default:
break;
}
return ex_schema;
}
int Maat_table_schema_set_EX_data_schema(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp)
long argl, void *argp,
void* logger)
{
struct EX_data_schema *ex_schema=NULL;
if(new_func==NULL || free_func==NULL || dup_func==NULL )
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
return -1;
}
ex_schema=Maat_table_schema_get_EX_data_schema(table_schema);
if(ex_schema==NULL)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a valid plugin table.", __FUNCTION__);
return -1;
}
if(ex_schema->set_flag)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data schema already registed.", __FUNCTION__);
return -1;
}
ex_schema->new_func=new_func;
ex_schema->free_func=free_func;
ex_schema->dup_func=dup_func;
ex_schema->key2index_func=key2index_func;//Set but not used.
ex_schema->argl=argl;
ex_schema->argp=argp;
return;
}
int Maat_table_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger)
{
if(new_func==NULL || free_func==NULL || dup_func==NULL )
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
return -1;
}
if(table_schema->table_type!=TABLE_TYPE_PLUGIN)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, Regist target is not a plugin table.", __FUNCTION__);
return -1;
}
if(table_schema->plugin.have_exdata)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__);
return -1;
}
if(table_schema->plugin.key_column==0 || table_schema->plugin.valid_flag_column==0)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, not enough schema information.", __FUNCTION__);
return -1;
}
Maat_table_EX_data_schema_set(&table_schema->plugin.ex_schema,
new_func, free_func, dup_func, key2index_func, argl, argp);
table_schema->plugin.have_exdata=1;
ex_schema->set_flag=1;
return 0;
}
int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger)
{
if(new_func==NULL || free_func==NULL || dup_func==NULL )
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
return -1;
}
if(table_schema->table_type!=TABLE_TYPE_IP_PLUGIN)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a ip_plugin table.", __FUNCTION__);
return -1;
}
if(table_schema->ip_plugin.have_exdata)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__);
return -1;
}
Maat_table_EX_data_schema_set(&table_schema->ip_plugin.ex_schema,
new_func, free_func, dup_func, key2index_func, argl, argp);
table_schema->ip_plugin.have_exdata=1;
return 0;
}
int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger)
{
if(new_func==NULL || free_func==NULL || dup_func==NULL )
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
return -1;
}
if(table_schema->table_type!=TABLE_TYPE_FQDN_PLUGIN)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a fqdn_plugin table.", __FUNCTION__);
return -1;
}
if(table_schema->fqdn_plugin.have_exdata)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__);
return -1;
}
Maat_table_EX_data_schema_set(&table_schema->fqdn_plugin.ex_schema,
new_func, free_func, dup_func, key2index_func, argl, argp);
table_schema->fqdn_plugin.have_exdata=1;
return 0;
}
}
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type)
{
table_mgr->active_plugin_table_num=0;
@@ -1178,11 +1185,11 @@ int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager*
{
return table_mgr->is_last_plugin_table_updating;
}
struct Maat_table_schema* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name)
struct Maat_table_schema* Maat_table_manager_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name)
{
struct Maat_table_schema * p_table=NULL;
int table_id=0;
table_id=Maat_table_get_id_by_name(table_mgr, table_name);
table_id=Maat_table_manager_get_id_by_name(table_mgr, table_name);
if(table_id<0)
{
return NULL;
@@ -1190,7 +1197,7 @@ struct Maat_table_schema* Maat_table_get_desc_by_name(struct Maat_table_manager*
p_table=table_mgr->p_table_info[table_id];
return p_table;
}
void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char* table_name)
void Maat_table_schema_set_updating_name(struct Maat_table_schema* p_table, const char* table_name)
{
int i=0;
for(i=0; i<p_table->conj_cnt; i++)
@@ -1202,11 +1209,20 @@ void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char*
}
assert(i<=p_table->conj_cnt);
}
struct Maat_table_schema * Maat_table_manager_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id)
{
if(table_id>MAX_TABLE_NUM||table_id<0)
{
return NULL;
}
int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type)
return table_mgr->p_table_info[table_id];
}
int Maat_table_manager_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type)
{
int ret=-1;
struct Maat_table_schema* p_table=Maat_table_get_by_id_raw(table_mgr, parent_table_id);
struct Maat_table_schema* p_table=Maat_table_manager_get_by_id_raw(table_mgr, parent_table_id);
if(p_table->table_type!=TABLE_TYPE_COMPOSITION)
{
return -1;
@@ -1214,7 +1230,7 @@ int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_tab
ret=p_table->composition.component_table_id[type];
return ret;
}
int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table)
int Maat_table_schema_get_valid_flag_column(struct Maat_table_schema* p_table)
{
int valid_flag_column=-1;
switch(p_table->table_type)
@@ -1228,6 +1244,9 @@ int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p
case TABLE_TYPE_FQDN_PLUGIN:
valid_flag_column=p_table->fqdn_plugin.valid_flag_column;
break;
case TABLE_TYPE_BOOL_PLUGIN:
valid_flag_column=p_table->bool_plugin.valid_flag_column;
break;
default:
valid_flag_column=-1;
break;

6
src/entry/bool_matcher.cpp
View File

@@ -31,9 +31,9 @@ bool operator<(const struct bool_item & lhs, const struct bool_item & rhs)
return lhs.item_id<rhs.item_id;
}
struct bool_matcher * bool_matcher_new(struct bool_expr * exprs, size_t expr_num, unsigned int max_thread_num, size_t * mem_size)
struct bool_matcher * bool_matcher_new(struct bool_expr * exprs, size_t expr_num, size_t * mem_size)
{
if(exprs==NULL || expr_num==0 || max_thread_num==0) return NULL;
if(exprs==NULL || expr_num==0) return NULL;
unsigned int mem_bytes=0;
@@ -145,7 +145,7 @@ int do_match(struct bool_expr_item * expr, unsigned long long * item_ids, size_t
return 1;
}
int bool_matcher_match(struct bool_matcher * matcher, unsigned int thread_id, unsigned long long * item_ids, size_t item_num, struct bool_expr_match * results, size_t n_result)
int bool_matcher_match(struct bool_matcher * matcher, unsigned long long * item_ids, size_t item_num, struct bool_expr_match * results, size_t n_result)
{
if(matcher==NULL) return -1;
if(item_num==0) return 0;

2
src/entry/json2iris.cpp
View File

@@ -12,7 +12,7 @@
#include "cJSON.h"
#include "hiredis.h"
#include "map_str2int.h"
#include "Maat_table.h"
#include "Maat_table_schema.h"
#include "Maat_rule_internal.h"
#include "Maat_utils.h"

2
src/inc_internal/Maat_rule_internal.h
View File

@@ -3,7 +3,7 @@
#include "Maat_rule.h"
#include "Maat_command.h"
#include "Maat_limits.h"
#include "Maat_table.h"
#include "Maat_table_schema.h"
#include "Maat_table_runtime.h"
#include <MESA/MESA_htable.h>

27
src/inc_internal/Maat_table_runtime.h
View File

@@ -1,8 +1,9 @@
#include "Maat_table.h"
#include "Maat_table_schema.h"
#include "Maat_ex_data.h"
#include "IPMatcher.h"
#include "gram_index_engine.h"
#include "bool_matcher.h"
#include "FQDN_engine.h"
#include "alignment_int64.h"
#include <MESA/MESA_list_queue.h>
@@ -18,6 +19,12 @@ struct fqdn_plugin_runtime
struct EX_data_rt* ex_data_rt;
int changed_flag;
};
struct bool_plugin_runtime
{
struct bool_matcher *matcher;
struct EX_data_rt* ex_data_rt;
int changed_flag;
};
struct plugin_runtime
{
struct EX_data_rt* ex_data_rt;
@@ -54,6 +61,7 @@ struct Maat_table_runtime
{
struct similar_runtime similar; //for digest and similarity
struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin
struct bool_plugin_runtime bool_plugin;
struct plugin_runtime plugin;
struct ip_plugin_runtime ip_plugin;
struct expr_runtime expr;
@@ -80,24 +88,25 @@ enum MAAT_TABLE_TYPE Maat_table_runtime_get_type(struct Maat_table_runtime* tabl
size_t Maat_table_runtime_plugin_cached_row_count(struct Maat_table_runtime* table_rt);
const char* Maat_table_runtime_plugin_get_cached_row(struct Maat_table_runtime* table_rt, size_t Nth_row);
int Maat_table_runtime_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, void* logger);
MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, const char* key);
int Maat_table_runtime_commit_EX_data_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);
void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag);
void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id);
int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
void Maat_table_runtime_plugin_update_commit(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_plugin_commit_update(struct Maat_table_runtime* table_rt);
MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, const char* key);
void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);
int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size);
int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_ip_plugin_commit_update(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
int Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);
int Maat_table_runtime_fqdn_plugin_commit_update(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size);
void Maat_table_runtime_bool_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
int Maat_table_runtime_bool_plugin_commit_update(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_bool_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, unsigned long long item_id[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size);

65
src/inc_internal/Maat_table.h → src/inc_internal/Maat_table_schema.h
View File

@@ -24,6 +24,7 @@ enum MAAT_SCAN_TYPE
SCAN_TYPE_PLUGIN,
SCAN_TYPE_IP_PLUGIN,
SCAN_TYPE_FQDN_PLUGIN,
SCAN_TYPE_BOOL_PLUGIN,
SCAN_TYPE_IP,
SCAN_TYPE_INTERVAL,
SCAN_TYPE_STRING,
@@ -60,7 +61,8 @@ enum MAAT_TABLE_TYPE
TABLE_TYPE_COMPILE,
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN,
TABLE_TYPE_FQDN_PLUGIN
TABLE_TYPE_FQDN_PLUGIN,
TABLE_TYPE_BOOL_PLUGIN
};
struct compile_ex_data_idx
@@ -113,6 +115,7 @@ struct EX_data_schema
Maat_plugin_EX_key2index_func_t* key2index_func;
long argl;
void *argp;
int set_flag;
};
struct plugin_table_schema
{
@@ -122,7 +125,6 @@ struct plugin_table_schema
int n_foreign;
int foreign_columns[MAX_FOREIGN_CLMN_NUM];
int cb_plug_cnt;
int have_exdata;
struct plugin_table_callback_schema cb_plug[MAX_PLUGIN_PER_TABLE];
struct EX_data_schema ex_schema;
};
@@ -148,6 +150,16 @@ struct fqdn_plugin_table_schema
int have_exdata;
struct EX_data_schema ex_schema;
};
struct bool_plugin_table_schema
{
int row_id_column;
int bool_expr_column;
int rule_tag_column;
int valid_flag_column;
int have_exdata;
struct EX_data_schema ex_schema;
};
struct Maat_table_schema
{
int table_id;
@@ -162,6 +174,7 @@ struct Maat_table_schema
struct plugin_table_schema plugin;
struct ip_plugin_table_schema ip_plugin;
struct fqdn_plugin_table_schema fqdn_plugin;
struct bool_plugin_table_schema bool_plugin;
struct virtual_table_schema virtual_table;
struct composition_table_schema composition;
void* others;//group, ip, interval and digest don't have any special schema.
@@ -177,59 +190,47 @@ void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr);
size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr);
size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr);
struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id);
struct Maat_table_schema * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id);
struct Maat_table_schema * Maat_table_manager_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id);
struct Maat_table_schema * Maat_table_manager_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id);
enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type);
int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
int Maat_table_manager_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
int Maat_table_manager_add_callback_func(struct Maat_table_manager* table_mgr,
int table_id,
Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
Maat_update_callback_t *update,//table line ,u_para
Maat_finish_callback_t *finish,//u_para
void* u_para);
int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_manager_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_manager_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_manager_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id);
enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id);
const char* Maat_table_manager_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id);
enum MAAT_TABLE_TYPE Maat_table_manager_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id);
int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name,
int Maat_table_manager_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name,
Maat_rule_EX_new_func_t *new_func,
Maat_rule_EX_free_func_t* free_func,
Maat_rule_EX_dup_func_t* dup_func,
long argl, void *argp);
struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx);
int Maat_table_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger);
int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger);
int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
struct compile_ex_data_idx* Maat_table_manager_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx);
int Maat_table_schema_set_EX_data_schema(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger);
struct EX_data_schema *Maat_table_schema_get_EX_data_schema(struct Maat_table_schema *table_schema);
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type);
void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr);
int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr);
struct Maat_table_schema* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char* table_name);
struct Maat_table_schema* Maat_table_manager_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
void Maat_table_schema_set_updating_name(struct Maat_table_schema* p_table, const char* table_name);
int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type);
int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table);
int Maat_table_manager_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type);
int Maat_table_schema_get_valid_flag_column(struct Maat_table_schema* p_table);