gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

将scanner中的运行态数据拆分到Maat_table_runtime.cpp中。

Browse Source
This commit is contained in:
zhengchao
2019-07-28 11:45:57 +06:00
parent 2909cb1997
commit 4c4222a302
13 changed files with 1166 additions and 804 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/CMakeLists.txt
View File

@@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
include_directories(/opt/MESA/include/MESA/)

381
src/entry/Maat_api.cpp
View File

@@ -16,42 +16,7 @@
#include "rulescan.h"
#include "json2iris.h"
struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id)
{
struct Maat_table_desc *p_table=NULL, *p_real_table=NULL;
if(table_id>MAX_TABLE_NUM)
{
return NULL;
}
if(_feather->p_table_info[table_id]==NULL)
{
return NULL;
}
p_table=_feather->p_table_info[table_id];
if(p_table==NULL)
{
return NULL;
}
if(p_table->table_type==TABLE_TYPE_VIRTUAL)
{
p_real_table=_feather->p_table_info[p_table->virtual_table.real_table_id];
*virutal_table_id=table_id;
}
else
{
p_real_table=p_table;
*virutal_table_id=0;
}
if(p_real_table->table_type!=expect_type)
{
if((expect_type==TABLE_TYPE_EXPR && p_table->table_type!=TABLE_TYPE_EXPR_PLUS)||
(expect_type==TABLE_TYPE_IP && p_table->table_type!=TABLE_TYPE_IP_PLUS))
{
return NULL;
}
}
return p_real_table;
}
inline void INC_SCANNER_REF(Maat_scanner*scanner,int thread_num)
{
alignment_int64_array_add(scanner->ref_cnt, thread_num, 1);
@@ -547,19 +512,14 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
return NULL;
}
_Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1);
feather->table_cnt=Maat_table_read_table_info(feather->p_table_info, MAX_TABLE_NUM, table_info_path, logger);
if(feather->table_cnt==0)
{
goto failed;
}
feather->map_tablename2id=Maat_table_build(feather->p_table_info, MAX_TABLE_NUM,
feather->compile_tn, sizeof(feather->compile_tn),
feather->group_tn, sizeof(feather->group_tn),
logger);
if(feather->map_tablename2id==NULL)
feather->table_mgr=Maat_table_manager_create(table_info_path, logger);
if(feather->table_mgr==NULL)
{
goto failed;
}
Maat_table_get_compile_table_name(feather->table_mgr, feather->compile_tn, sizeof(feather->compile_tn));
Maat_table_get_group_table_name(feather->table_mgr, feather->group_tn, sizeof(feather->group_tn));
feather->logger=logger;
feather->scan_thread_num=max_thread_num;
feather->garbage_q=MESA_lqueue_create(0,0);
@@ -994,16 +954,7 @@ void Maat_burn_feather(Maat_feather_t feather)
int Maat_table_register(Maat_feather_t feather,const char* table_name)
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
int table_id=-1,ret=0;
ret=map_str2int(_feather->map_tablename2id, table_name, &table_id);
if(ret>0)
{
return table_id;
}
else
{
return -1;
}
return Maat_table_get_id_by_name(_feather->table_mgr, table_name);
}
int Maat_table_callback_register(Maat_feather_t feather,short table_id,
Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
@@ -1012,52 +963,35 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
void* u_para)
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
int idx=0,i=0;
struct Maat_table_desc *p_table=_feather->p_table_info[table_id];
struct plugin_table_desc *plugin_desc=&(p_table->plugin);
struct plugin_runtime* plugin_aux=NULL;
const char* lines=NULL;
if(p_table==NULL)
{
return -1;
}
if(p_table->table_type!=TABLE_TYPE_PLUGIN)
{
return -1;
}
//plugin table register blocks background update.
int i=0,ret=0;
pthread_mutex_lock(&(_feather->background_update_mutex));
idx=plugin_desc->cb_plug_cnt;
if(idx==MAX_PLUGIN_PER_TABLE)
ret=Maat_table_add_callback_func(_feather->table_mgr, table_id, start, update, finish, u_para);
if(ret<0)
{
pthread_mutex_unlock(&(_feather->background_update_mutex));
return -1;
}
plugin_desc->cb_plug_cnt++;
plugin_desc->cb_plug[idx].start=start;
plugin_desc->cb_plug[idx].update=update;
plugin_desc->cb_plug[idx].finish=finish;
plugin_desc->cb_plug[idx].u_para=u_para;
if(_feather->scanner==NULL)
{
pthread_mutex_unlock(&(_feather->background_update_mutex));
return 1;
}
plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin);
if(plugin_aux->cache_line_num>0)
const char* line=NULL;
struct Maat_table_runtime* table_rt=NULL;
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
long long line_cnt=Maat_table_runtime_plugin_cached_line_count(table_rt);
if(line_cnt>0)
{
if(start!=NULL)
{
start(MAAT_RULE_UPDATE_TYPE_FULL,u_para);
}
for(i=0;i<plugin_aux->cache_line_num;i++)
for(i=0; i<line_cnt; i++)
{
lines=(const char*)dynamic_array_read(plugin_aux->cache_lines,i);
if(lines==NULL)
line=Maat_table_runtime_plugin_get_cached_line(table_rt, i);
if(line==NULL)
{
break;
}
update(table_id,lines,u_para);
update(table_id,line,u_para);
}
if(finish!=NULL)
{
@@ -1095,39 +1029,31 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table
long argl, void *argp)
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
int table_id=-1,ret=0, idx=-1;
ret=map_str2int(_feather->map_tablename2id, compile_table_name, &table_id);
if(ret<0)
{
return -1;
}
struct Maat_table_desc *p_table=_feather->p_table_info[table_id];
if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL || dup_func==NULL)
{
return -1;
}
struct compile_table_desc* compile_desc=&(p_table->compile);
pthread_mutex_lock(&(_feather->background_update_mutex));
if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM)
{
ret=-1;
goto failed;
}
idx=compile_desc->ex_data_num;
compile_desc->ex_desc[idx].idx=idx;
compile_desc->ex_desc[idx].table_id=table_id;
compile_desc->ex_desc[idx].argl=argl;
compile_desc->ex_desc[idx].argp=argp;
compile_desc->ex_desc[idx].new_func=new_func;
compile_desc->ex_desc[idx].free_func=free_func;
compile_desc->ex_desc[idx].dup_func=dup_func;
int idx=-1;
if(new_func==NULL || free_func==NULL || dup_func==NULL)
{
return -1;
}
pthread_mutex_lock(&(_feather->background_update_mutex));
idx=Maat_table_new_compile_rule_ex_index(_feather->table_mgr, compile_table_name,
new_func,
free_func,
dup_func,
argl, argp);
if(idx<0)
{
pthread_mutex_unlock(&(_feather->background_update_mutex));
return -1;
}
struct compile_ex_data_idx* compile_ex_desc=Maat_table_get_compile_rule_ex_desc(_feather->table_mgr, compile_table_name, idx);
compile_desc->ex_data_num++;
if(_feather->scanner!=NULL)
{
MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_desc->ex_desc+idx);
MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_ex_desc);
}
failed:
pthread_mutex_unlock(&(_feather->background_update_mutex));
return idx;
@@ -1155,129 +1081,7 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa
pthread_rwlock_unlock(&(relation->rwlock));
return ad;
}
struct wrap_plugin_EX_data
{
MAAT_RULE_EX_DATA exdata;
const struct Maat_table_desc* ref_plugin_table;
};
void wrap_plugin_EX_data_free(void *data)
{
struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data;
const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc);
ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp);
wrap_data->ref_plugin_table=NULL;
free(wrap_data);
return;
}
MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index)
{
MESA_htable_handle key2ex_hash=NULL;
unsigned int slot_size=1;
while(estimate_size!=0)
{
estimate_size=estimate_size>>1;
slot_size*=2;
}
if(slot_size==1)
{
slot_size=4096;
}
MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs));
hargs.thread_safe=8;
hargs.hash_slot_size = slot_size;
hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
hargs.expire_time = 0;
hargs.key_comp = NULL;
hargs.key2index = NULL; //Not supported yet.
hargs.recursive = 1;
hargs.data_free = wrap_plugin_EX_data_free;
hargs.data_expire_with_condition = NULL;
key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(key2ex_hash, 0);
return key2ex_hash;
}
int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line,
MESA_htable_handle key2ex_hash, void *logger)
{
size_t key_offset=0, key_len=0;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
int ret=0;
ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data del error: cannot find column %d of %s",
plugin_desc->key_column, line);
return -1;
}
ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data del error: no such key %.*s of %s",
key_len, line+key_offset, line);
return -1;
}
return 0;
}
int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line,
MESA_htable_handle key2ex_hash, void *logger)
{
char* key=NULL;
size_t key_offset=0, key_len=0;
MAAT_RULE_EX_DATA exdata=NULL;
struct wrap_plugin_EX_data* wrap_data=NULL;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
int ret=0;
ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data add error: cannot find column %d of %s",
plugin_desc->key_column, line);
return -1;
}
key=ALLOC(char, key_len+1);
memcpy(key, line+key_offset, key_len);
plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata,
plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
wrap_data=ALLOC(struct wrap_plugin_EX_data, 1);
wrap_data->exdata=exdata;
wrap_data->ref_plugin_table=plugin_table;
ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data);
free(key);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data add error: duplicated key %.*s of %s",
key_len, line+key_offset, line);
wrap_plugin_EX_data_free(wrap_data);
return -1;
}
return 0;
}
MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table,
struct dynamic_array_t* lines, size_t line_cnt, void* logger)
{
MESA_htable_handle key2ex_hash=NULL;
size_t i=0;
const char* line=NULL;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func);
for(i=0; i< line_cnt; i++)
{
line=(const char*)dynamic_array_read(lines, i);
plugin_EX_data_new(plugin_table, line, key2ex_hash, logger);
}
return key2ex_hash;
}
int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
@@ -1287,39 +1091,35 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_table_desc *table_desc=_feather->p_table_info[table_id];
struct plugin_table_desc* plugin_desc=&(table_desc->plugin);
struct Maat_table_runtime* table_rt=NULL;
int idx=-1;
if(new_func==NULL || free_func==NULL || dup_func==NULL )
{
assert(0);
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
return -1;
}
if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata
|| plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0)
{
assert(0);
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module,
"%s failed: key or valid flag column are not specified", __FUNCTION__);
return -1;
}
pthread_mutex_lock(&(_feather->background_update_mutex));
plugin_desc->ex_desc.new_func=new_func;
plugin_desc->ex_desc.free_func=free_func;
plugin_desc->ex_desc.dup_func=dup_func;
plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used.
plugin_desc->ex_desc.argl=argl;
plugin_desc->ex_desc.argp=argp;
plugin_desc->have_exdata=1;
idx=Maat_table_plugin_new_ex_index(_feather->table_mgr, table_id,
new_func,
free_func,
dup_func,
key2index_func,
argl, argp);
if(idx<0)
{
pthread_mutex_unlock(&(_feather->background_update_mutex));
return -1;
}
struct Maat_table_desc *table_desc=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
struct Maat_table_runtime* table_rt=NULL;
if(_feather->scanner!=NULL)
{
table_rt=_feather->scanner->table_rt[table_id];
assert(table_rt->plugin.key2ex_hash==NULL);
table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines,
table_rt->plugin.cache_line_num, _feather->logger);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
Maat_table_runtime_plugin_new_ex_idx(table_rt, table_desc, _feather->logger);
}
pthread_mutex_unlock(&(_feather->background_update_mutex));
@@ -1328,28 +1128,16 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_table_desc *table_desc=_feather->p_table_info[table_id];
struct Maat_table_runtime *table_rt= NULL;
struct plugin_table_desc* plugin_desc=&(table_desc->plugin);
struct wrap_plugin_EX_data* wrap_data=NULL;
struct Maat_table_desc *table_desc=NULL;
struct Maat_table_runtime *table_rt=NULL;
MAAT_RULE_EX_DATA exdata=NULL;
if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata==0)
{
assert(0);
return NULL;
}
if(_feather->scanner==NULL)
{
return NULL;
}
table_rt= _feather->scanner->table_rt[table_id];
wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash,
(const unsigned char*)key, strlen(key));
if(wrap_data!=NULL)
{
plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata),
plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
}
table_desc=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key);
return exdata;
}
@@ -1384,7 +1172,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
clock_gettime(CLOCK_MONOTONIC,&start);
}
_mid=grab_mid(mid,_feather, thread_num, 0);
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR, &virtual_table_id);
p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -1410,7 +1198,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
scan_data.text_data.tlen=data_len;
scan_data.text_data.toffset=0;
struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id);
if(table_rt->origin_rule_num==0)
{
return 0;
@@ -1518,7 +1306,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
clock_gettime(CLOCK_MONOTONIC,&start);
}
int virutal_table_id=0;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_INTERVAL, &virutal_table_id);
p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_INTERVAL, &virutal_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -1529,7 +1317,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
{
return 0;
}
struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id);
if(table_rt->origin_rule_num==0)
{
return 0;
@@ -1609,7 +1397,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
clock_gettime(CLOCK_MONOTONIC,&start);
}
int virtual_table_id=0;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_IP, &virtual_table_id);
p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_IP, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -1620,7 +1408,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
{
return 0;
}
struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id);
if(table_rt->origin_rule_num==0)
{
return 0;
@@ -1728,7 +1516,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
struct Maat_table_desc *p_table=NULL;
int virtual_table_id=0;
assert(thread_num<_feather->scan_thread_num);
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR, &virtual_table_id);
p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -1748,7 +1536,8 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
{
return sp;
}
struct Maat_table_runtime* table_rt=scanner->table_rt[sp->p_real_table->table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id);
if(table_rt->origin_rule_num==0)
{
return sp;
@@ -1796,7 +1585,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
{
return 0;
}
struct Maat_table_runtime* table_rt=scanner->table_rt[sp->p_real_table->table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id);
if(sp->feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC, &start);
@@ -1969,7 +1758,7 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para)
struct Maat_table_runtime* table_rt=NULL;
if(scanner!=NULL)
{
table_rt=scanner->table_rt[sp->p_real_table->table_id];
table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id);
alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1);
}
@@ -2011,7 +1800,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
sfh_instance_t * tmp_fuzzy_handle=NULL;
struct Maat_table_desc *p_table=NULL;
int virtual_table_id=0;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST, &virtual_table_id);
p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_DIGEST, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -2028,7 +1817,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
{
return sp;
}
struct Maat_table_runtime* table_rt=scanner->table_rt[table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_id);
tmp_fuzzy_handle=SFH_instance(total_len);
if(tmp_fuzzy_handle==NULL)
{
@@ -2086,7 +1875,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{
return 0;
}
struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->p_real_table->table_id];
struct Maat_table_runtime *table_rt=Maat_table_runtime_get(sp->feather->scanner->table_rt_mgr, sp->p_real_table->table_id);
GIE_handle_t* GIE_handle=table_rt->similar.gie_handle;
unsigned long long digest_len=0;
char* digest_buff=NULL;
@@ -2169,7 +1958,7 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para)
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct Maat_scanner* scanner=sp->feather->scanner;
struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->p_real_table->table_id];
struct Maat_table_runtime *table_rt=Maat_table_runtime_get(sp->feather->scanner->table_rt_mgr, sp->p_real_table->table_id);
alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1);
if(scanner!=NULL)
{
@@ -2273,7 +2062,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
clock_gettime(CLOCK_MONOTONIC,&start);
}
int virtual_table_id=0;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id);
p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -2284,7 +2073,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
{
return 0;
}
struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id);
if(table_rt->origin_rule_num==0)
{
return 0;
@@ -2356,7 +2145,7 @@ int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* valu
}
break;
case MAAT_STATE_LAST_UPDATING_TABLE:
*int_val=_feather->is_last_plugin_table_updating;
*int_val=Maat_table_manager_is_last_plugin_table_updating(_feather->table_mgr);
break;
case MAAT_STATE_IN_UPDATING:
if(size!=sizeof(int))

41
src/entry/Maat_command.cpp
View File

@@ -915,7 +915,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
continue;
}
region_cmd=&(group_cmd->regions[j]);
region_cmd->table_name=_maat_strdup(feather->p_table_info[region_inner->table_id]->table_name[0]);
region_cmd->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, region_inner->table_id));
region_cmd->region_id=region_inner->region_id;
//NOTICE: region_type only avilable when OP_ADD,
region_cmd->region_type=REGION_EXPR;
@@ -1374,7 +1374,7 @@ error_out:
int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
{
int i=0,j=0,ret=0;
int i=0, j=0;
const char *table_name=NULL;
int table_id=0;
struct Maat_group_t* p_group=NULL;
@@ -1399,8 +1399,8 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
{
p_region=&(p_group->regions[j]);
table_name=p_region->table_name;
ret=map_str2int(feather->map_tablename2id, table_name, &table_id);
if(ret<0)
table_id=Maat_table_get_id_by_name(feather->table_mgr, table_name);
if(table_id<0)
{
MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module
,"Unknown table %s of Maat_cmd_t[%d]->group[%d]->region[%d]."
@@ -1409,7 +1409,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
return -1;
}
table_type=type_region2table(p_region);
if(table_type!=feather->p_table_info[table_id]->table_type)
if(table_type!=Maat_table_get_type_by_id(feather->table_mgr, table_id))
{
MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module
,"Table %s not support region type %d of Maat_cmd_t[%d]->group[%d]->region[%d]."
@@ -1419,7 +1419,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
return -1;
}
free((char*)p_region->table_name);
p_region->table_name=_maat_strdup(feather->p_table_info[table_id]->table_name[0]);
p_region->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, table_id));
}
}
return 0;
@@ -1651,7 +1651,7 @@ void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n
}
int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger)
{
int ret=0, table_id=0, i=0;
int i=0;
int rule_with_foreign_key=0;
struct Maat_table_desc* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
@@ -1661,14 +1661,13 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list,
{
continue;
}
ret=map_str2int(feather->map_tablename2id, rule_list[i].table_name, &table_id);
if(ret<0)
p_table=Maat_table_get_desc_by_name(feather->table_mgr, rule_list[i].table_name);
if(!p_table||p_table->table_type!=TABLE_TYPE_PLUGIN)
{
continue;
}
p_table=feather->p_table_info[table_id];
plugin_desc= &(p_table->plugin);
if(p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->n_foreign==0)
if(plugin_desc->n_foreign==0)
{
continue;
}
@@ -1808,6 +1807,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
int update_type=CM_UPDATE_TYPE_INC;
long long new_version=0;
enum MAAT_TABLE_TYPE table_type;
struct Maat_table_desc* table_desc=NULL;
const struct plugin_table_desc* plugin_desc=NULL;
void* logger=feather->logger;
@@ -1889,17 +1889,18 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
if(rule_list[i].table_line==NULL||rule_list[i].with_error==1)
{
continue;
}
ret=map_str2int(feather->map_tablename2id,rule_list[i].table_name,&table_id);
}
table_id=Maat_table_get_id_by_name(feather->table_mgr, rule_list[i].table_name);
if(table_id<0)//Unrecognized table.
{
continue;
}
}
table_type=Maat_table_get_type_by_id(feather->table_mgr, table_id);
if(rule_list[i].op==MAAT_OP_DEL)
{
if(table_type==TABLE_TYPE_PLUGIN)
{
{
table_desc=Maat_table_get_by_id(feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
plugin_desc=&(table_desc->plugin);
valid_column=plugin_desc->valid_flag_column;
}
@@ -2105,8 +2106,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
server_time=redis_server_time(write_ctx);
s_rule=(struct serial_rule_t *)calloc(sizeof(struct serial_rule_t),line_num);
for(i=0;i<line_num;i++)
{
ret=map_str2int(_feather->map_tablename2id, line_rule[i]->table_name, &table_id);
{
table_id=Maat_table_get_id_by_name(_feather->table_mgr, line_rule[i]->table_name);
if(table_id<0)
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command
@@ -2115,9 +2116,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
, line_rule[i]->table_name);
ret=-1;
goto error_out;
}
p_table=_feather->p_table_info[table_id];
plugin_desc=&(p_table->plugin);
}
p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
if(!p_table)
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command
@@ -2126,6 +2126,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
, line_rule[i]->table_name);
ret=-1;
goto error_out;
}
plugin_desc=&(p_table->plugin);
if(op==MAAT_OP_ADD)
{

402
src/entry/Maat_rule.cpp
View File

@@ -11,6 +11,7 @@
#include <time.h>
#include <sys/prctl.h>
#include <sys/syscall.h>
#include <strings.h>
#include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h>
@@ -479,11 +480,7 @@ error_out:
cJSON_Delete(json);
return ret;
}
int lqueue_destroy_cb(void *data, long data_len, void *arg)
{
assert(0);
return 0;
}
void * HASH_fetch_by_id(MESA_htable_handle hash,int id)
{
return MESA_htable_search(hash,(unsigned char*)&(id),sizeof(id));
@@ -938,127 +935,10 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule)
op_expr->rule_type=p_rule->rule_type;
return;
}
GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest,
short cfds_lvl,struct Maat_group_inner* tag)
{
GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1);
int digest_len=0;
rule->id=id;
rule->operation=op;
if(digest!=NULL)
{
digest_len=strlen(digest);
rule->sfh=(char*)calloc(sizeof(char),digest_len+1);
memcpy(rule->sfh,digest,digest_len);
}
rule->sfh_length=digest_len;
rule->cfds_lvl=cfds_lvl;
rule->tag=(void*)tag;
return rule;
}
void destroy_digest_rule(GIE_digest_t*rule)
{
if(rule->sfh!=NULL)
{
free(rule->sfh);
rule->sfh=NULL;
}
free(rule);
rule=NULL;
return;
}
struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num)
{
struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1);
table_rt->table_type=table_desc->table_type;
switch(table_desc->table_type)
{
case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY:
table_rt->similar.update_q=MESA_lqueue_create(0,0);
break;
case TABLE_TYPE_PLUGIN:
table_rt->plugin.cache_lines=dynamic_array_create(1, 1024);
if(table_desc->plugin.have_exdata)
{
table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size,
table_desc->plugin.ex_desc.key2index_func);
}
break;
default:
break;
}
table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num);
table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num);
table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num);
table_rt->stream_num=alignment_int64_array_alloc(max_thread_num);
table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num);
return table_rt;
}
void table_runtime_free(struct Maat_table_runtime* p)
{
long q_cnt=0,data_size=0;
int i=0;
UNUSED int q_ret=0;
GIE_digest_t* digest_rule=NULL;
if(p==NULL)
{
return;
}
switch(p->table_type)
{
case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY:
if(p->similar.gie_handle!=NULL)
{
GIE_destory(p->similar.gie_handle);
}
if(p->similar.update_q!=NULL)
{
q_cnt=MESA_lqueue_get_count(p->similar.update_q);
for(i=0;i<q_cnt;i++)
{
data_size=sizeof(GIE_digest_t*);
q_ret=(MESA_queue_errno_t)MESA_lqueue_get_head(p->similar.update_q,&digest_rule,&data_size);
assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK);
destroy_digest_rule(digest_rule);
}
MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL);
}
break;
case TABLE_TYPE_PLUGIN:
dynamic_array_destroy(p->plugin.cache_lines, free);
p->plugin.cache_lines=NULL;
if(p->plugin.key2ex_hash!=NULL)
{
MESA_htable_destroy(p->plugin.key2ex_hash, NULL);
}
default:
break;
}
alignment_int64_array_free(p->scan_cnt);
alignment_int64_array_free(p->scan_cpu_time);
alignment_int64_array_free(p->input_bytes);
alignment_int64_array_free(p->stream_num);
alignment_int64_array_free(p->hit_cnt);
free(p);
return;
}
struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *feather)
{
int scan_thread_num=feather->scan_thread_num;
// int rs_scan_type=feather->rule_scan_type;
struct Maat_table_desc ** pp_table_desc=feather->p_table_info;
struct Maat_table_runtime* table_rt=NULL;
int i=0;
UNUSED int ret=0;
MESA_htable_create_args_t hargs;
@@ -1115,8 +995,8 @@ struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *f
scanner->region_update_q=MESA_lqueue_create(0, 0);
scanner->region=rulescan_initialize(scan_thread_num);
//For best performance test:
//1.Do NOT set this option,rulescan return no hit detail as default;
//For best scan performance:
//1.Do NOT set this option, rulescan return no hit detail as default;
//2.Set necessary STR rule to QUICK;
if(feather->rule_scan_type==1)
{
@@ -1130,16 +1010,8 @@ struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *f
scanner->tomb_ref=feather->garbage_q;
scanner->logger_ref=feather->logger;
scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num);
for(i=0;i<MAX_TABLE_NUM;i++)
{
if(pp_table_desc[i]==NULL)
{
continue;
}
table_rt=table_runtime_new(pp_table_desc[i], feather->scan_thread_num);
scanner->table_rt[i]=table_rt;
}
scanner->table_rt_mgr=Maat_table_runtime_manager_create(feather->table_mgr, feather->scan_thread_num);
scanner->max_table_num=Maat_table_manager_get_size(feather->table_mgr);
return scanner;
}
@@ -1189,10 +1061,9 @@ void destroy_maat_scanner(struct Maat_scanner*scanner)
}
}
}
for(i=0;i<MAX_TABLE_NUM;i++)
{
table_runtime_free(scanner->table_rt[i]);
}
Maat_table_rt_manager_destroy(scanner->table_rt_mgr);
scanner->table_rt_mgr=NULL;
igraph_destroy(&scanner->group_graph);
free(scanner);
return;
@@ -1235,9 +1106,8 @@ struct _region_stat_t
int ipv6_rule_cnt;
};
};
void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_stat, int size)
void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_stat, size_t size)
{
assert(op_expr->table_id<size);
int op=0;
if(op_expr->p_expr->operation==0)//add
{
@@ -1277,7 +1147,7 @@ void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_sta
void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,void*logger,struct Maat_scanner* maat_scanner)
{
long i=0,data_size=0;
long data_size=0, i=0;
unsigned int j=0;
int ret=0;
unsigned int failed_ids[MAX_FAILED_NUM];
@@ -1288,7 +1158,8 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,
const long q_cnt=MESA_lqueue_get_count(expr_queue);
struct timespec start,end;
unsigned long long update_interval=0;
struct _region_stat_t region_counter[MAX_TABLE_NUM];
size_t max_table_num=maat_scanner->max_table_num;
struct _region_stat_t region_counter[max_table_num];
memset(region_counter, 0, sizeof(region_counter));
struct Maat_table_runtime* table_rt=NULL;
if(q_cnt==0)
@@ -1317,7 +1188,7 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,
}
}
count_rs_region(op_expr,region_counter,MAX_TABLE_NUM);
count_rs_region(op_expr, region_counter, max_table_num);
destroy_op_expr(op_expr);
op_expr=NULL;
}
@@ -1344,9 +1215,9 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,
,(double)update_interval
,update_interval);
//update scanner's region cnt;
for(i=0;i<MAX_TABLE_NUM;i++)
for(i=0; (size_t)i<max_table_num; i++)
{
table_rt=maat_scanner->table_rt[i];
table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, i);
if(table_rt==NULL)
{
continue;
@@ -1384,52 +1255,6 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,
free(to_update_expr);
}
void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*logger,struct Maat_scanner* maat_scanner,int table_id)
{
long i=0,data_size=0;
int ret=0;
GIE_digest_t* digest_rule=NULL;
GIE_digest_t** update_array=NULL;
UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK;
const long q_cnt=MESA_lqueue_get_count(update_q);
if(q_cnt==0)
{
return;
}
struct Maat_table_runtime* table_rt=maat_scanner->table_rt[table_id];
update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt);
for(i=0;i<q_cnt;i++)
{
data_size=sizeof(void*);
q_ret=(MESA_queue_errno_t)MESA_lqueue_get_head(update_q,&digest_rule,&data_size);
assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK);
update_array[i]=digest_rule;
digest_rule=NULL;
}
ret=GIE_update(handle,update_array, (int)q_cnt);
if(ret!=(int)q_cnt)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"GIE_update error,when batch update %d/%ld rules.",ret,q_cnt);
}
for(i=0;i<q_cnt;i++)
{
if(update_array[i]->operation==GIE_INSERT_OPT)
{
table_rt->origin_rule_num++;
}
else
{
table_rt->origin_rule_num--;
}
destroy_digest_rule(update_array[i]);
update_array[i]=NULL;
}
free(update_array);
update_array=NULL;
return;
}
struct region_group_relation
{
int region_id;
@@ -2049,34 +1874,25 @@ int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule* intval_
MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*));
return 0;
}
int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule* db_digest_rule,struct Maat_scanner *scanner,void* logger)
int add_digest_rule(struct Maat_table_desc* table, struct db_digest_rule* db_rule, struct Maat_scanner *scanner,void* logger)
{
struct Maat_group_inner* group_rule=NULL;
GIE_digest_t* digest_rule=NULL;
struct Maat_group_inner* u_para=NULL;
struct Maat_table_runtime * table_rt=scanner->table_rt[table->table_id];
struct Maat_table_runtime * table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
int expr_id=0,district_id=-1;
group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id);
group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id);
if(group_rule==NULL)
{
group_rule=create_group_rule(db_digest_rule->group_id, 0, scanner);
group_rule=create_group_rule(db_rule->group_id, 0, scanner);
}
expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule, table->table_id, db_digest_rule->region_id, district_id, expr_id, TABLE_TYPE_DIGEST, scanner);
u_para=add_region_to_group(group_rule, table->table_id, db_rule->region_id, district_id, expr_id, TABLE_TYPE_DIGEST, scanner);
if(u_para==NULL)
{
return -1;
}
if(table->table_type==TABLE_TYPE_SIMILARITY)
{
db_digest_rule->digest_string=str_unescape(db_digest_rule->digest_string);
}
digest_rule=create_digest_rule(expr_id, GIE_INSERT_OPT
,db_digest_rule->digest_string
,db_digest_rule->confidence_degree
,group_rule);
MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*));
Maat_table_runtime_digest_add(table_rt, expr_id, db_rule->digest_string, db_rule->confidence_degree, group_rule);
scanner->gie_update_q_size++;
return 0;
}
@@ -2086,8 +1902,8 @@ int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int
unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0};
int expr_num=0;
struct Maat_group_inner* group_rule=NULL;
struct Maat_table_runtime* table_rt=NULL;
struct op_expr_t* op_expr=NULL;
GIE_digest_t* digest_rule=NULL;
group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(maat_scanner->group_hash, group_id);
if(group_rule==NULL)
{
@@ -2126,11 +1942,8 @@ int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int
case TABLE_TYPE_SIMILARITY:
case TABLE_TYPE_DIGEST:
assert(expr_num==1);
digest_rule=create_digest_rule(expr_id[0], GIE_DELETE_OPT //del digest
,NULL
,0
,NULL);
MESA_lqueue_join_tail(maat_scanner->table_rt[table->table_id]->similar.update_q,&digest_rule, sizeof(void*));
table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, table->table_id);
Maat_table_runtime_digest_del(table_rt, expr_id[0]);
maat_scanner->gie_update_q_size++;
break;
default:
@@ -2331,10 +2144,10 @@ int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_
scanner->to_update_compile_cnt++;
return 1;
}
void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner, MESA_htable_handle map_tablename2id, void* logger)
void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner, struct Maat_table_manager* table_mgr, void* logger)
{
struct db_group_rule_t db_group_rule;
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
int ret=0;
char virtual_table_name[MAX_TABLE_NAME_LEN]={0};
memset(&db_group_rule, 0, sizeof(db_group_rule));
@@ -2364,10 +2177,10 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru
table->udpate_err_cnt++;
return;
}
if(strlen(virtual_table_name)>0)
if(strlen(virtual_table_name)>0&&!strcasecmp(virtual_table_name, "null"))
{
ret=map_str2int(map_tablename2id, virtual_table_name, &(db_group_rule.virtual_table_id));
if(ret<0)
db_group_rule.virtual_table_id=Maat_table_get_id_by_name(table_mgr, virtual_table_name);
if(db_group_rule.virtual_table_id<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"update error, unknown virutal table name: %s of group table %s:%s.",
@@ -2381,7 +2194,8 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru
{
ret=del_group_rule(table, &db_group_rule, scanner, logger);
//leave no trace when compatible_group_update calling
if(table->table_type==TABLE_TYPE_GROUP&&ret==1)
assert(table->table_type==TABLE_TYPE_GROUP);
if(ret==1)
{
table_rt->origin_rule_num--;
assert(table_rt->origin_rule_num>=0);
@@ -2422,7 +2236,7 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc
{
struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1);
int ret=0,db_hexbin=0,rule_type=0;
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
switch(table->table_type)
{
case TABLE_TYPE_EXPR:
@@ -2711,7 +2525,7 @@ void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struc
struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1);
char src_ip1[40]={0}, src_ip2[40]={0}, dst_ip1[40]={0}, dst_ip2[40]={0};
char saddr_format[16]={0}, sport_format[16]={0}, daddr_format[16]={0}, dport_format[16]={0};
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
unsigned short src_port1=0, src_port2=0, dst_port1=0, dst_port2=0;
int protocol=0,direction=0;
int ret=0;
@@ -2918,7 +2732,7 @@ error_out:
void update_intval_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger)
{
struct db_intval_rule* intval_rule=ALLOC(struct db_intval_rule, 1);
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
int ret=0;
ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id)
,&(intval_rule->group_id)
@@ -2984,7 +2798,7 @@ error_out:
void update_compile_rule(struct Maat_table_desc* table,const char* table_line ,struct Maat_scanner *scanner, const struct rule_tag* tags, int n_tags,void* logger)
{
struct compile_table_desc* compile_desc=&(table->compile);
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
struct Maat_compile_rule *p_compile=NULL;
struct Maat_rule_head m_rule_tmp;
@@ -3069,7 +2883,7 @@ error_out:
void update_digest_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger)
{
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
struct db_digest_rule* digest_rule=ALLOC(struct db_digest_rule, 1);
int ret=0;
char digest_buff[MAX_TABLE_LINE_SIZE]={'\0'};
@@ -3270,7 +3084,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Ma
int i=0, ret=1, matched_tag=1;
unsigned int len=strlen(table_line)+1;
struct plugin_table_desc* plugin_desc=&(table->plugin);
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
char *p=NULL;
char* copy=NULL;
size_t is_valid_offset=0, valid_len=0;
@@ -3413,11 +3227,7 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q,
struct bool_matcher *tmp1=NULL,*tmp2=NULL;
MESA_htable_handle tmp_map=NULL;
struct Maat_table_runtime* table_rt=NULL;
int i=0;
long q_cnt;
GIE_create_para_t para;
para.gram_value=7;
para.position_accuracy=10;
int i=0, ret=0;
igraph_bool_t is_dag;
igraph_is_dag(&(scanner->group_graph), &is_dag);
if(!is_dag)
@@ -3451,9 +3261,9 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q,
scanner->region_update_q,
logger,
scanner);
for(i=0;i<MAX_TABLE_NUM;i++)
for(i=0; (size_t)i<scanner->max_table_num; i++)
{
table_rt=scanner->table_rt[i];
table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, i);
if(table_rt==NULL)
{
continue;
@@ -3462,30 +3272,13 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q,
{
case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY:
q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q);
if(q_cnt==0)
ret=Maat_table_runtime_digest_batch_udpate(table_rt);
if(ret<0)
{
continue;
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"GIE_update error.");
}
if(table_rt->similar.gie_handle==NULL)
{
if(table_rt->table_type==TABLE_TYPE_SIMILARITY)
{
para.ED_reexamine=1;
para.format=GIE_INPUT_FORMAT_PLAIN;
}
else
{
para.ED_reexamine=0;
para.format=GIE_INPUT_FORMAT_SFH;
}
table_rt->similar.gie_handle=GIE_create(&para);
}
digest_batch_update(table_rt->similar.gie_handle,
table_rt->similar.update_q,
logger,
scanner,
i);
break;
case TABLE_TYPE_PLUGIN:
break;
@@ -3509,12 +3302,9 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q,
}
void maat_start_cb(long long new_version,int update_type,void*u_para)
void maat_start_cb(long long new_version, int update_type, void*u_para)
{
struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para;
struct Maat_table_desc* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
int i=0,j=0;
feather->new_version=new_version;
if(update_type==CM_UPDATE_TYPE_FULL)
@@ -3532,26 +3322,7 @@ void maat_start_cb(long long new_version,int update_type,void*u_para)
feather->maat_version,new_version);
feather->maat_version=new_version;
}
feather->active_plugin_table_num=0;
for(i=0;i<MAX_TABLE_NUM;i++)
{
p_table=feather->p_table_info[i];
plugin_desc=&(p_table->plugin);
if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->cb_plug_cnt==0)
{
continue;
}
feather->active_plugin_table_num++;
for(j=0;j<plugin_desc->cb_plug_cnt;j++)
{
if(plugin_desc->cb_plug[j].start!=NULL)
{
plugin_desc->cb_plug[j].start(update_type,plugin_desc->cb_plug[j].u_para);
}
}
}
Maat_table_manager_all_plugin_cb_start(feather->table_mgr, update_type);
return;
}
long long scanner_rule_num(struct Maat_scanner *scanner)
@@ -3559,9 +3330,9 @@ long long scanner_rule_num(struct Maat_scanner *scanner)
long long total=0;
struct Maat_table_runtime* table_rt=NULL;
int i=0;
for(i=0;i<MAX_TABLE_NUM;i++)
for(i=0; (size_t)i<scanner->max_table_num; i++)
{
table_rt=scanner->table_rt[i];
table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, i);
if(table_rt!=NULL)
{
total+=table_rt->origin_rule_num;
@@ -3572,42 +3343,9 @@ long long scanner_rule_num(struct Maat_scanner *scanner)
void maat_finish_cb(void* u_para)
{
struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para;
struct Maat_table_desc* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
long expr_wait_q_cnt=0;
int i=0, j=0;
int call_plugin_table_cnt=0;
for(i=0;i<MAX_TABLE_NUM;i++)
{
p_table=feather->p_table_info[i];
if(p_table==NULL)
{
continue;
}
switch(p_table->table_type)
{
case TABLE_TYPE_PLUGIN:
plugin_desc=&(p_table->plugin);
call_plugin_table_cnt++;
if(call_plugin_table_cnt==feather->active_plugin_table_num)
{
feather->is_last_plugin_table_updating=1;
}
for(j=0;j<plugin_desc->cb_plug_cnt;j++)
{
if(plugin_desc->cb_plug[j].finish!=NULL)
{
plugin_desc->cb_plug[j].finish(plugin_desc->cb_plug[j].u_para);
}
}
feather->is_last_plugin_table_updating=0;
break;
default:
break;
}
}
Maat_table_manager_all_plugin_cb_finish(feather->table_mgr);
if(feather->update_tmp_scanner!=NULL)
{
@@ -3651,14 +3389,11 @@ void maat_finish_cb(void* u_para)
feather->maat_version);
}
feather->new_version=-1;
feather->active_plugin_table_num=0;
return;
}
int maat_update_cb(const char* table_name,const char* line,void *u_para)
{
struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para;
int ret=-1,i=0;
int table_id=-1;
Maat_scanner* scanner=NULL;
struct Maat_table_desc* p_table=NULL;
if(feather->update_tmp_scanner!=NULL)
@@ -3670,47 +3405,39 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
scanner=feather->scanner;
}
// MESA_handle_runtime_log(feather->logger, RLOG_LV_DEBUG, maat_module, "Maat table %s input: %s", table_name, line);
ret=map_str2int(feather->map_tablename2id,table_name,&table_id);
if(ret<0)
p_table=Maat_table_get_desc_by_name(feather->table_mgr, table_name);
if(!p_table)
{
MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s",table_name);
MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s", table_name);
return -1;
}
p_table=feather->p_table_info[table_id];
for(i=0;i<p_table->conj_cnt;i++)
{
if(0==memcmp(p_table->table_name[i],table_name,strlen(table_name)))
{
p_table->updating_name=i;
}
}
assert(i<=p_table->conj_cnt);
Maat_table_set_updating_name(p_table, table_name);
switch(feather->p_table_info[table_id]->table_type)
switch(p_table->table_type)
{
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
update_expr_rule(feather->p_table_info[table_id], line, scanner, feather->logger);
update_expr_rule(p_table, line, scanner, feather->logger);
break;
case TABLE_TYPE_IP:
case TABLE_TYPE_IP_PLUS:
update_ip_rule(feather->p_table_info[table_id], line, scanner, feather->logger);
update_ip_rule(p_table, line, scanner, feather->logger);
break;
case TABLE_TYPE_INTERVAL:
update_intval_rule(feather->p_table_info[table_id], line, scanner,feather->logger);
update_intval_rule(p_table, line, scanner,feather->logger);
break;
case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY:
update_digest_rule(feather->p_table_info[table_id], line, scanner,feather->logger);
update_digest_rule(p_table, line, scanner,feather->logger);
break;
case TABLE_TYPE_COMPILE:
update_compile_rule(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
update_compile_rule(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
break;
case TABLE_TYPE_GROUP:
update_group_rule(feather->p_table_info[table_id], line, scanner, feather->map_tablename2id, feather->logger);
update_group_rule(p_table, line, scanner, feather->table_mgr, feather->logger);
break;
case TABLE_TYPE_PLUGIN:
update_plugin_table(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
update_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
default:
break;
@@ -3884,12 +3611,11 @@ void *thread_rule_monitor(void *arg)
}
}
MESA_htable_destroy(feather->map_tablename2id,free);
Maat_table_manager_destroy(feather->table_mgr);
destroy_maat_scanner(feather->scanner);
garbage_bury(feather->garbage_q,0,feather->logger);
assert(0==MESA_lqueue_get_count(feather->garbage_q));
MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL);
Maat_table_clear(feather->p_table_info, MAX_TABLE_NUM);
alignment_int64_array_free(feather->thread_call_cnt);
alignment_int64_array_free(feather->inner_mid_cnt);

19
src/entry/Maat_stat.cpp
View File

@@ -1,4 +1,5 @@
#include "Maat_rule_internal.h"
#include "Maat_table.h"
#include "alignment_int64.h"
#include <time.h>
#include <MESA/field_stat.h>
@@ -121,10 +122,11 @@ void maat_stat_init(struct _Maat_feather_t* feather)
FS_STYLE_COLUMN,
FS_CALC_SPEED,
"hit_rate");
feather->total_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT,"Sum");
for(i=0;i<MAX_TABLE_NUM;i++)
feather->total_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT, "Sum");
size_t max_table_num=Maat_table_manager_get_size(feather->table_mgr);
for(i=0; i<(int)max_table_num; i++)
{
p_table=feather->p_table_info[i];
p_table=Maat_table_get_by_id_raw(feather->table_mgr, i);
if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN
||p_table->table_type==TABLE_TYPE_GROUP
||p_table->table_type==TABLE_TYPE_COMPILE)
@@ -185,7 +187,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,Maat_table_manager_get_count(feather->table_mgr));
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0,FS_OP_SET,not_grp_hit_cnt);
@@ -197,7 +200,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value);
feather->update_err_cnt=0;
feather->iconv_err_cnt=0;
for(i=0;i<MAX_TABLE_NUM;i++)
size_t max_table_num=Maat_table_manager_get_size(feather->table_mgr);
for(i=0; i<(int)max_table_num; i++)
{
table_stream_num=0;
table_scan_cnt=0;
@@ -205,12 +209,13 @@ void maat_stat_output(struct _Maat_feather_t* feather)
table_scan_cpu_time=0;
table_hit_cnt=0;
table_regex_ipv6_num=0;
p_table=feather->p_table_info[i];
p_table=Maat_table_get_by_id_raw(feather->table_mgr, i);
if(p_table==NULL)
{
continue;
}
table_rt=feather->scanner->table_rt[i];
table_rt=Maat_table_runtime_get(feather->scanner->table_rt_mgr, i);
switch(p_table->table_type)
{
case TABLE_TYPE_PLUGIN:

478
src/entry/Maat_table.cpp
View File

@@ -9,6 +9,16 @@
#include <assert.h>
#include <MESA/MESA_handle_logger.h>
#define MAX_TABLE_NUM 256
struct Maat_table_manager
{
struct Maat_table_desc* p_table_info[MAX_TABLE_NUM];
size_t table_cnt;
MESA_htable_handle map_tablename2id;
int active_plugin_table_num;
int is_last_plugin_table_updating;
};
int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map)
{
@@ -192,11 +202,85 @@ error_out:
free(copy_line);
return -1;
}
int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_table, const char* table_info_path, void* logger)
static int Maat_table_build_map(struct Maat_table_manager* table_mgr, void* logger)
{
struct Maat_table_desc** p_table_info=table_mgr->p_table_info;
size_t n_table=MAX_TABLE_NUM;
MESA_htable_handle map_tablename2id=map_create();
size_t i=0;
int j=0, ret=0;
for(i=0;i<n_table;i++)
{
if(p_table_info[i]==NULL)
{
continue;
}
switch(p_table_info[i]->table_type)
{
case TABLE_TYPE_VIRTUAL:
ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id));
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Undefined real table %s, virtual table %s of table id %d.",
p_table_info[i]->virtual_table.real_table_name,
p_table_info[i]->table_name[j],
p_table_info[i]->table_id);
goto failed;
}
break;
default:
break;
}
for(j=0; j<p_table_info[i]->conj_cnt; j++)
{
ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Duplicate table %s of table id %d",
p_table_info[i]->table_name[j],
p_table_info[i]->table_id);
continue;
}
}
}
table_mgr->map_tablename2id=map_tablename2id;
return 0;
failed:
map_destroy(map_tablename2id);
return -1;
}
void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr)
{
size_t i=0;
for(i=0;i<MAX_TABLE_NUM;i++)
{
if(table_mgr->p_table_info[i]==NULL)
{
continue;
}
table_info_free(table_mgr->p_table_info[i]);
table_mgr->p_table_info[i]=NULL;
}
MESA_htable_destroy(table_mgr->map_tablename2id, free);
free(table_mgr);
return;
}
struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger)
{
struct Maat_table_manager* table_mgr=NULL;
FILE*fp=NULL;
char line[MAX_TABLE_LINE_SIZE];
int i=0,ret=0,table_cnt=0;
int i=0, ret=0;
char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0};
MESA_htable_handle string2int_map=NULL;;
struct Maat_table_desc*p=NULL;
@@ -207,8 +291,11 @@ int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_t
fprintf(stderr,"Maat read table info %s error.\n",table_info_path);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s failed: %s.\n", table_info_path, strerror(errno));
return 0;
return NULL;
}
table_mgr=ALLOC(struct Maat_table_manager, 1);
struct Maat_table_desc** p_table_info=table_mgr->p_table_info;
size_t n_table=MAX_TABLE_NUM;
string2int_map=map_create();
map_register(string2int_map,"expr", TABLE_TYPE_EXPR);
@@ -348,83 +435,350 @@ int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_t
}
p_table_info[p->table_id]=p;
table_cnt++;
table_mgr->table_cnt++;
continue;
invalid_table:
table_info_free(p);
p=NULL;
}
fclose(fp);
map_destroy(string2int_map);
return table_cnt;
}
MESA_htable_handle Maat_table_build(struct Maat_table_desc** p_table_info, size_t n_table, char* compile_tn, size_t n_ctn, char* group_tn, size_t n_gtn, void* logger)
{
MESA_htable_handle map_tablename2id=map_create();
size_t i=0;
int j=0, ret=0;
for(i=0;i<n_table;i++)
ret=Maat_table_build_map(table_mgr, logger);
if(ret<0)
{
if(p_table_info[i]==NULL)
return NULL;
}
map_destroy(string2int_map);
return table_mgr;
}
size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr)
{
return MAX_TABLE_NUM;
}
size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr)
{
return table_mgr->table_cnt;
}
int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
{
int i=0;
for(i=0; i< MAX_TABLE_NUM; i++)
{
if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_COMPILE)
{
strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz);
return 1;
}
}
return 0;
}
int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
{
int i=0;
for(i=0; i< MAX_TABLE_NUM; i++)
{
if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP)
{
strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz);
return 1;
}
}
return 0;
}
const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id)
{
if(table_id>MAX_TABLE_NUM)
{
return NULL;
}
if(table_mgr->p_table_info[table_id])
{
return table_mgr->p_table_info[table_id]->table_name[0];
}
return NULL;
}
enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id)
{
if(table_id>MAX_TABLE_NUM)
{
return TABLE_TYPE_INVALID;
}
if(table_mgr->p_table_info[table_id])
{
return table_mgr->p_table_info[table_id]->table_type;
}
return TABLE_TYPE_INVALID;
}
struct Maat_table_desc * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id)
{
if(table_id>MAX_TABLE_NUM)
{
return NULL;
}
return table_mgr->p_table_info[table_id];
}
struct Maat_table_desc * Maat_table_get_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id)
{
struct Maat_table_desc **p_table_info=table_mgr->p_table_info;
size_t n_table=MAX_TABLE_NUM;
struct Maat_table_desc *p_table=NULL, *p_real_table=NULL;
if((unsigned int) table_id>n_table)
{
return NULL;
}
if(p_table_info[table_id]==NULL)
{
return NULL;
}
p_table=p_table_info[table_id];
if(p_table==NULL)
{
return NULL;
}
if(p_table->table_type==TABLE_TYPE_VIRTUAL)
{
p_real_table=p_table_info[p_table->virtual_table.real_table_id];
*virutal_table_id=table_id;
}
else
{
p_real_table=p_table;
if(virutal_table_id) *virutal_table_id=0;
}
if(p_real_table->table_type!=expect_type)
{
if((expect_type==TABLE_TYPE_EXPR && p_table->table_type!=TABLE_TYPE_EXPR_PLUS)||
(expect_type==TABLE_TYPE_IP && p_table->table_type!=TABLE_TYPE_IP_PLUS))
{
return NULL;
}
}
return p_real_table;
}
int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name)
{
int table_id=-1,ret=0;
ret=map_str2int(table_mgr->map_tablename2id, table_name, &table_id);
if(ret>0)
{
return table_id;
}
else
{
return -1;
}
}
int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
int table_id,
Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
Maat_update_callback_t *update,//table line ,u_para
Maat_finish_callback_t *finish,//u_para
void* u_para)
{
int idx=0;
struct Maat_table_desc *p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
struct plugin_table_desc *plugin_desc=&(p_table->plugin);
if(p_table==NULL)
{
return -1;
}
idx=plugin_desc->cb_plug_cnt;
if(idx==MAX_PLUGIN_PER_TABLE)
{
return -1;
}
plugin_desc->cb_plug_cnt++;
plugin_desc->cb_plug[idx].start=start;
plugin_desc->cb_plug[idx].update=update;
plugin_desc->cb_plug[idx].finish=finish;
plugin_desc->cb_plug[idx].u_para=u_para;
return 1;
}
struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx)
{
int table_id=-1;
struct Maat_table_desc *p_table=NULL;
table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name);
if(table_id<0)
{
return NULL;
}
p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL);
if(!p_table)
{
return NULL;
}
if(idx<p_table->compile.ex_data_num)
{
return p_table->compile.ex_desc+idx;
}
return NULL;
}
int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name,
Maat_rule_EX_new_func_t *new_func,
Maat_rule_EX_free_func_t* free_func,
Maat_rule_EX_dup_func_t* dup_func,
long argl, void *argp)
{
int table_id=-1;
struct Maat_table_desc *p_table=NULL;
table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name);
if(table_id<0)
{
return -1;
}
p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL);
if(!p_table)
{
return -1;
}
int idx=-1;
struct compile_table_desc* compile_desc=&(p_table->compile);
if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM)
{
return -1;
}
idx=compile_desc->ex_data_num;
compile_desc->ex_desc[idx].idx=idx;
compile_desc->ex_desc[idx].table_id=table_id;
compile_desc->ex_desc[idx].argl=argl;
compile_desc->ex_desc[idx].argp=argp;
compile_desc->ex_desc[idx].new_func=new_func;
compile_desc->ex_desc[idx].free_func=free_func;
compile_desc->ex_desc[idx].dup_func=dup_func;
compile_desc->ex_data_num++;
return idx;
}
int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp)
{
struct Maat_table_desc *table_desc=NULL;;
table_desc=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
struct plugin_table_desc* plugin_desc=&(table_desc->plugin);
if(plugin_desc->have_exdata
|| plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0)
{
return -1;
}
plugin_desc->ex_desc.new_func=new_func;
plugin_desc->ex_desc.free_func=free_func;
plugin_desc->ex_desc.dup_func=dup_func;
plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used.
plugin_desc->ex_desc.argl=argl;
plugin_desc->ex_desc.argp=argp;
plugin_desc->have_exdata=1;
return 0;
}
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type)
{
table_mgr->active_plugin_table_num=0;
int i=0, j=0;
struct Maat_table_desc* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
for(i=0; i<MAX_TABLE_NUM; i++)
{
p_table=table_mgr->p_table_info[i];
plugin_desc=&(p_table->plugin);
if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->cb_plug_cnt==0)
{
continue;
}
switch(p_table_info[i]->table_type)
table_mgr->active_plugin_table_num++;
for(j=0;j<plugin_desc->cb_plug_cnt;j++)
{
case TABLE_TYPE_GROUP:
strncpy(group_tn, p_table_info[i]->table_name[0], n_gtn);
break;
case TABLE_TYPE_COMPILE:
strncpy(compile_tn, p_table_info[i]->table_name[0], n_ctn);
break;
case TABLE_TYPE_VIRTUAL:
ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id));
if(ret<0)
if(plugin_desc->cb_plug[j].start!=NULL)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Undefined real table %s, virtual table %s of table id %d.",
p_table_info[i]->virtual_table.real_table_name,
p_table_info[i]->table_name[j],
p_table_info[i]->table_id);
goto failed;
plugin_desc->cb_plug[j].start(update_type, plugin_desc->cb_plug[j].u_para);
}
}
}
}
void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr)
{
int i=0, j=0;
struct Maat_table_desc* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
int call_plugin_table_cnt=0;
for(i=0;i<MAX_TABLE_NUM;i++)
{
p_table=table_mgr->p_table_info[i];
if(p_table==NULL)
{
continue;
}
switch(p_table->table_type)
{
case TABLE_TYPE_PLUGIN:
plugin_desc=&(p_table->plugin);
call_plugin_table_cnt++;
if(call_plugin_table_cnt==table_mgr->active_plugin_table_num)
{
table_mgr->is_last_plugin_table_updating=1;
}
for(j=0;j<plugin_desc->cb_plug_cnt;j++)
{
if(plugin_desc->cb_plug[j].finish!=NULL)
{
plugin_desc->cb_plug[j].finish(plugin_desc->cb_plug[j].u_para);
}
}
table_mgr->is_last_plugin_table_updating=0;
break;
default:
break;
}
for(j=0; j<p_table_info[i]->conj_cnt; j++)
{
ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Duplicate table %s of table id %d",
p_table_info[i]->table_name[j],
p_table_info[i]->table_id);
continue;
}
}
}
return map_tablename2id;
failed:
map_destroy(map_tablename2id);
return NULL;
}
void Maat_table_clear(struct Maat_table_desc** p_table_info, size_t n_table)
{
size_t i=0;
for(i=0;i<n_table;i++)
{
if(p_table_info[i]==NULL)
{
continue;
}
table_info_free(p_table_info[i]);
p_table_info[i]=NULL;
}
table_mgr->active_plugin_table_num=0;
return;
}
int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr)
{
return table_mgr->is_last_plugin_table_updating;
}
struct Maat_table_desc* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name)
{
struct Maat_table_desc * p_table=NULL;
int table_id=0;
table_id=Maat_table_get_id_by_name(table_mgr, table_name);
if(table_id<0)
{
return NULL;
}
p_table=table_mgr->p_table_info[table_id];
return p_table;
}
void Maat_table_set_updating_name(struct Maat_table_desc* p_table, const char* table_name)
{
int i=0;
for(i=0; i<p_table->conj_cnt; i++)
{
if(0==strcmp(p_table->table_name[i], table_name))
{
p_table->updating_name=i;
}
}
assert(i<=p_table->conj_cnt);
}

427
src/entry/Maat_table_runtime.cpp Normal file
View File

@@ -0,0 +1,427 @@
#include "Maat_table_runtime.h"
#include "Maat_rule.h"
#include "Maat_utils.h"
#include <MESA/MESA_handle_logger.h>
#include <MESA/MESA_list_queue.h>
#include <assert.h>
struct wrap_plugin_EX_data
{
MAAT_RULE_EX_DATA exdata;
const struct Maat_table_desc* ref_plugin_table;
};
void wrap_plugin_EX_data_free(void *data)
{
struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data;
const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc);
ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp);
wrap_data->ref_plugin_table=NULL;
free(wrap_data);
return;
}
MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index)
{
MESA_htable_handle key2ex_hash=NULL;
unsigned int slot_size=1;
while(estimate_size!=0)
{
estimate_size=estimate_size>>1;
slot_size*=2;
}
if(slot_size==1)
{
slot_size=4096;
}
MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs));
hargs.thread_safe=8;
hargs.hash_slot_size = slot_size;
hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
hargs.expire_time = 0;
hargs.key_comp = NULL;
hargs.key2index = NULL; //Not supported yet.
hargs.recursive = 1;
hargs.data_free = wrap_plugin_EX_data_free;
hargs.data_expire_with_condition = NULL;
key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(key2ex_hash, 0);
return key2ex_hash;
}
int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line,
MESA_htable_handle key2ex_hash, void *logger)
{
size_t key_offset=0, key_len=0;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
int ret=0;
ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data del error: cannot find column %d of %s",
plugin_desc->key_column, line);
return -1;
}
ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data del error: no such key %.*s of %s",
key_len, line+key_offset, line);
return -1;
}
return 0;
}
int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line,
MESA_htable_handle key2ex_hash, void *logger)
{
char* key=NULL;
size_t key_offset=0, key_len=0;
MAAT_RULE_EX_DATA exdata=NULL;
struct wrap_plugin_EX_data* wrap_data=NULL;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
int ret=0;
ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data add error: cannot find column %d of %s",
plugin_desc->key_column, line);
return -1;
}
key=ALLOC(char, key_len+1);
memcpy(key, line+key_offset, key_len);
plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata,
plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
wrap_data=ALLOC(struct wrap_plugin_EX_data, 1);
wrap_data->exdata=exdata;
wrap_data->ref_plugin_table=plugin_table;
ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data);
free(key);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data add error: duplicated key %.*s of %s",
key_len, line+key_offset, line);
wrap_plugin_EX_data_free(wrap_data);
return -1;
}
return 0;
}
struct Maat_table_runtime_manager
{
struct Maat_table_runtime** table_rt;
size_t n_table_rt;
};
static GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest,
short cfds_lvl, void* tag)
{
GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1);
int digest_len=0;
rule->id=id;
rule->operation=op;
if(digest!=NULL)
{
digest_len=strlen(digest);
rule->sfh=(char*)calloc(sizeof(char),digest_len+1);
memcpy(rule->sfh,digest,digest_len);
}
rule->sfh_length=digest_len;
rule->cfds_lvl=cfds_lvl;
rule->tag=tag;
return rule;
}
static void destroy_digest_rule(GIE_digest_t*rule)
{
if(rule->sfh!=NULL)
{
free(rule->sfh);
rule->sfh=NULL;
}
free(rule);
rule=NULL;
return;
}
static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num)
{
struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1);
table_rt->table_type=table_desc->table_type;
switch(table_desc->table_type)
{
case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY:
table_rt->similar.update_q=MESA_lqueue_create(0,0);
break;
case TABLE_TYPE_PLUGIN:
table_rt->plugin.cache_lines=dynamic_array_create(1, 1024);
if(table_desc->plugin.have_exdata)
{
table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size,
table_desc->plugin.ex_desc.key2index_func);
}
break;
default:
break;
}
table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num);
table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num);
table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num);
table_rt->stream_num=alignment_int64_array_alloc(max_thread_num);
table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num);
return table_rt;
}
static void table_runtime_free(struct Maat_table_runtime* p)
{
long q_cnt=0,data_size=0;
int i=0;
UNUSED int q_ret=0;
GIE_digest_t* digest_rule=NULL;
if(p==NULL)
{
return;
}
switch(p->table_type)
{
case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY:
if(p->similar.gie_handle!=NULL)
{
GIE_destory(p->similar.gie_handle);
}
if(p->similar.update_q!=NULL)
{
q_cnt=MESA_lqueue_get_count(p->similar.update_q);
for(i=0;i<q_cnt;i++)
{
data_size=sizeof(GIE_digest_t*);
q_ret=(MESA_queue_errno_t)MESA_lqueue_get_head(p->similar.update_q,&digest_rule,&data_size);
assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK);
destroy_digest_rule(digest_rule);
}
MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL);
}
break;
case TABLE_TYPE_PLUGIN:
dynamic_array_destroy(p->plugin.cache_lines, free);
p->plugin.cache_lines=NULL;
if(p->plugin.key2ex_hash!=NULL)
{
MESA_htable_destroy(p->plugin.key2ex_hash, NULL);
}
default:
break;
}
alignment_int64_array_free(p->scan_cnt);
alignment_int64_array_free(p->scan_cpu_time);
alignment_int64_array_free(p->input_bytes);
alignment_int64_array_free(p->stream_num);
alignment_int64_array_free(p->hit_cnt);
free(p);
return;
}
struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num)
{
const struct Maat_table_desc* table_desc=NULL;
struct Maat_table_runtime* table_rt=NULL;
struct Maat_table_runtime_manager* table_rt_mgr=ALLOC(struct Maat_table_runtime_manager, 1);
size_t i=0;
table_rt_mgr->n_table_rt=Maat_table_manager_get_size(table_manager);
table_rt_mgr->table_rt=ALLOC(struct Maat_table_runtime*, table_rt_mgr->n_table_rt);
for(i=0; i<table_rt_mgr->n_table_rt; i++)
{
table_desc=Maat_table_get_by_id_raw(table_manager, i);
if(!table_desc)
{
continue;
}
table_rt=table_runtime_new(table_desc, max_thread_num);
table_rt_mgr->table_rt[i]=table_rt;
}
return table_rt_mgr;
}
void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr)
{
size_t i=0;
for(i=0; i<table_rt_mgr->n_table_rt; i++)
{
table_runtime_free(table_rt_mgr->table_rt[i]);
table_rt_mgr->table_rt[i]=NULL;
}
free(table_rt_mgr->table_rt);
table_rt_mgr->table_rt=NULL;
free(table_rt_mgr);
}
struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id)
{
assert(table_id<(int)table_rt_mgr->n_table_rt);
return table_rt_mgr->table_rt[table_id];
}
long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt)
{
struct plugin_runtime* plugin_rt=&(table_rt->plugin);
return plugin_rt->cache_line_num;
}
const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line)
{
const char* line=NULL;
struct plugin_runtime* plugin_rt=&(table_rt->plugin);
line=(const char*)dynamic_array_read(plugin_rt->cache_lines, Nth_line);
return line;
}
MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table,
struct dynamic_array_t* lines, size_t line_cnt, void* logger)
{
MESA_htable_handle key2ex_hash=NULL;
size_t i=0;
const char* line=NULL;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func);
for(i=0; i< line_cnt; i++)
{
line=(const char*)dynamic_array_read(lines, i);
plugin_EX_data_new(plugin_table, line, key2ex_hash, logger);
}
return key2ex_hash;
}
int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, void* logger)
{
assert(table_rt->plugin.key2ex_hash==NULL);
if(table_rt->plugin.key2ex_hash)
{
return -1;
}
table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines,
table_rt->plugin.cache_line_num, logger);
return 0;
}
MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, const char* key)
{
struct plugin_table_desc* plugin_desc=NULL;
struct wrap_plugin_EX_data* wrap_data=NULL;
MAAT_RULE_EX_DATA exdata=NULL;
plugin_desc=&(table_desc->plugin);
if(!plugin_desc->have_exdata)
{
assert(0);
return NULL;
}
wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash,
(const unsigned char*)key, strlen(key));
if(wrap_data!=NULL)
{
plugin_desc->ex_desc.dup_func(table_desc->table_id, &(exdata), &(wrap_data->exdata),
plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
}
return exdata;
}
void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag)
{
GIE_digest_t* digest_rule=NULL;
char *dup_digest=_maat_strdup(digest);
if(table_rt->table_type==TABLE_TYPE_SIMILARITY)
{
dup_digest=str_unescape(dup_digest);
}
digest_rule=create_digest_rule(expr_id, GIE_INSERT_OPT,
dup_digest,
confidence_degree,
tag);
MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*));
return;
}
void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id)
{
GIE_digest_t* digest_rule=NULL;
digest_rule=create_digest_rule(expr_id, GIE_DELETE_OPT //del digest
,NULL
,0
,NULL);
MESA_lqueue_join_tail(table_rt->similar.update_q,&digest_rule, sizeof(void*));
return;
}
int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt)
{
long i=0,data_size=0;
int ret=0;
GIE_digest_t* digest_rule=NULL;
GIE_digest_t** update_array=NULL;
UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK;
GIE_create_para_t para;
para.gram_value=7;
para.position_accuracy=10;
const long q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q);
if(q_cnt==0)
{
return 0;
}
if(table_rt->similar.gie_handle==NULL)
{
if(table_rt->table_type==TABLE_TYPE_SIMILARITY)
{
para.ED_reexamine=1;
para.format=GIE_INPUT_FORMAT_PLAIN;
}
else
{
para.ED_reexamine=0;
para.format=GIE_INPUT_FORMAT_SFH;
}
table_rt->similar.gie_handle=GIE_create(&para);
}
update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt);
for(i=0;i<q_cnt;i++)
{
data_size=sizeof(void*);
q_ret=(MESA_queue_errno_t)MESA_lqueue_get_head(table_rt->similar.update_q, &digest_rule, &data_size);
assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK);
update_array[i]=digest_rule;
digest_rule=NULL;
}
ret=GIE_update(table_rt->similar.gie_handle, update_array, (int)q_cnt);
for(i=0;i<q_cnt;i++)
{
if(update_array[i]->operation==GIE_INSERT_OPT)
{
table_rt->origin_rule_num++;
}
else
{
table_rt->origin_rule_num--;
}
destroy_digest_rule(update_array[i]);
update_array[i]=NULL;
}
free(update_array);
update_array=NULL;
if(ret!=(int)q_cnt)
{
return -1;
}
return q_cnt;
}

8
src/entry/Maat_utils.cpp
View File

@@ -3,7 +3,7 @@
#include <ctype.h>
#include <stdlib.h>
#include <openssl/md5.h>
#include <assert.h>
#include "Maat_utils.h"
pid_t gettid()
{
@@ -252,3 +252,9 @@ const char* charset_get_name(enum MAAT_CHARSET charset)
{
return CHARSET_STRING[charset];
}
int lqueue_destroy_cb(void *data, long data_len, void *arg)
{
assert(0);
return 0;
}

1
src/inc_internal/Maat_limits.h
View File

@@ -1,4 +1,3 @@
#define MAX_TABLE_NUM 256
#define MAX_TABLE_LINE_SIZE (1024*16)
#define MAX_EXPR_KEYLEN 1024
#define MAX_DISTRICT_LEN 64

59
src/inc_internal/Maat_rule_internal.h
View File

@@ -4,6 +4,7 @@
#include "Maat_command.h"
#include "Maat_limits.h"
#include "Maat_table.h"
#include "Maat_table_runtime.h"
#include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h>
@@ -218,54 +219,7 @@ struct _stream_para_t
pthread_mutex_t fuzzy_mutex;
unsigned char query_point[8];
};
struct similar_runtime
{
GIE_handle_t* gie_handle;
MESA_lqueue_head update_q;
};
struct plugin_runtime
{
dynamic_array_t *cache_lines;
long long cache_line_num;
long long acc_line_num;
long long cache_size;
MESA_htable_handle key2ex_hash;
};
struct expr_runtime
{
long long expr_rule_cnt; //expr_type=0,1,3
long long regex_rule_cnt; //expr_type=2
};
struct ip_runtime
{
long long ipv4_rule_cnt;
long long ipv6_rule_cnt;
};
struct group_runtime
{
long long not_flag_group;
};
struct Maat_table_runtime
{
enum MAAT_TABLE_TYPE table_type;
long origin_rule_num;
union
{
struct similar_runtime similar; //for digest and similarity
struct plugin_runtime plugin;
struct expr_runtime expr;
struct ip_runtime ip;
struct group_runtime group;
void * other;
};
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
};
struct rule_tag
{
char* tag_name;
@@ -281,7 +235,8 @@ struct Maat_scanner
size_t to_update_group_cnt;
size_t to_update_compile_cnt;
struct Maat_table_runtime* table_rt[MAX_TABLE_NUM];
struct Maat_table_runtime_manager* table_rt_mgr;
size_t max_table_num;
MESA_htable_handle region_hash; //key: region_id, value: struct region_group_relation*
MESA_htable_handle exprid_hash; //key: expr_id, value: int array_idx of Maat_group_inner->regions;
@@ -344,7 +299,8 @@ struct _Maat_feather_t
struct Maat_scanner *scanner;
struct Maat_scanner *update_tmp_scanner;
MESA_lqueue_head garbage_q;
int table_cnt;
struct Maat_table_manager* table_mgr;
int DEFERRED_LOAD_ON;
int REDIS_MODE_ON;
enum data_source input_mode;
@@ -360,8 +316,7 @@ struct _Maat_feather_t
int cumulative_update_off;
int stat_on;
int perf_on;
struct Maat_table_desc *p_table_info[MAX_TABLE_NUM];
MESA_htable_handle map_tablename2id;
void* logger;
long long maat_version;
long long last_full_version;
@@ -394,8 +349,6 @@ struct _Maat_feather_t
int foreign_cont_linger;
//internal states
long long new_version;
int active_plugin_table_num;
int is_last_plugin_table_updating;
//for scanner independent stat>>>>
int backgroud_update_enabled;

82
src/inc_internal/Maat_table.h
View File

@@ -20,6 +20,7 @@ enum USER_REGION_ENCODE
enum MAAT_TABLE_TYPE
{
TABLE_TYPE_INVALID=-1,
TABLE_TYPE_EXPR=0,
TABLE_TYPE_IP,
TABLE_TYPE_IP_PLUS,
@@ -50,6 +51,21 @@ struct compile_table_desc
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
};
struct expr_table_desc
{
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cross_cache_size;
int quick_expr_switch;//obsolete since 20190401
long long iconv_err_cnt;
};
struct virtual_table_desc
{
int real_table_id;
char real_table_name[MAX_TABLE_NAME_LEN];
};
struct plugin_table_callback_desc
{
Maat_start_callback_t *start;
@@ -79,26 +95,7 @@ struct plugin_table_desc
struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE];
struct plugin_table_ex_data_desc ex_desc;
};
struct expr_table_desc
{
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cross_cache_size;
int quick_expr_switch;//obsolete since 20190401
long long iconv_err_cnt;
};
struct ip_table_desc
{
int ipv4_rule_cnt;
int ipv6_rule_cnt;
};
struct virtual_table_desc
{
int real_table_id;
char real_table_name[MAX_TABLE_NAME_LEN];
};
struct Maat_table_desc
{
int table_id;
@@ -110,17 +107,52 @@ struct Maat_table_desc
{
struct compile_table_desc compile;
struct expr_table_desc expr;
struct ip_table_desc ip;
struct plugin_table_desc plugin;
struct virtual_table_desc virtual_table;;
void* others;//group, interval and digest don't have sperate description info.
void* others;//group, ip, interval and digest don't have sperate description info.
};
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
unsigned long long unmatch_tag_cnt;
int stat_line_id;
};
int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_table, const char* table_info_path, void* logger);
MESA_htable_handle Maat_table_build(struct Maat_table_desc** p_table_info, size_t n_table, char* compile_tn, size_t n_ctn, char* group_tn, size_t n_gtn, void* logger);
void Maat_table_clear(struct Maat_table_desc** p_table_info, size_t n_table);
struct Maat_table_manager;
struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger);
void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr);
size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr);
size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr);
struct Maat_table_desc * Maat_table_get_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id);
struct Maat_table_desc * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id);
int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
int table_id,
Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
Maat_update_callback_t *update,//table line ,u_para
Maat_finish_callback_t *finish,//u_para
void* u_para);
int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id);
enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id);
int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name,
Maat_rule_EX_new_func_t *new_func,
Maat_rule_EX_free_func_t* free_func,
Maat_rule_EX_dup_func_t* dup_func,
long argl, void *argp);
struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx);
int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp);
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type);
void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr);
int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr);
struct Maat_table_desc* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
void Maat_table_set_updating_name(struct Maat_table_desc* p_table, const char* table_name);

69
src/inc_internal/Maat_table_runtime.h Normal file
View File

@@ -0,0 +1,69 @@
#include "Maat_table.h"
#include "gram_index_engine.h"
#include "alignment_int64.h"
#include "dynamic_array.h"
#include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h>
struct similar_runtime
{
GIE_handle_t* gie_handle;
MESA_lqueue_head update_q;
};
struct plugin_runtime
{
dynamic_array_t *cache_lines;
long long cache_line_num;
long long acc_line_num;
long long cache_size;
MESA_htable_handle key2ex_hash;
};
struct expr_runtime
{
long long expr_rule_cnt; //expr_type=0,1,3
long long regex_rule_cnt; //expr_type=2
};
struct ip_runtime
{
long long ipv4_rule_cnt;
long long ipv6_rule_cnt;
};
struct group_runtime
{
long long not_flag_group;
};
struct Maat_table_runtime
{
enum MAAT_TABLE_TYPE table_type;
long origin_rule_num;
union
{
struct similar_runtime similar; //for digest and similarity
struct plugin_runtime plugin;
struct expr_runtime expr;
struct ip_runtime ip;
struct group_runtime group;
void * other;
};
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
};
struct Maat_table_runtime_manager;
struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num);
void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr);
struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id);
long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt);
const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line);
int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, void* logger);
MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, const char* key);
void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag);
void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id);
int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt);

1
src/inc_internal/Maat_utils.h
View File

@@ -72,5 +72,6 @@ char* md5_file(const char* filename, char* md5string);
int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len);
const char** charset_get_all_name(void);
const char* charset_get_name(enum MAAT_CHARSET charset);
int lqueue_destroy_cb(void *data, long data_len, void *arg);