gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

合并plugin和ip_plugin的处理逻辑，抽象为Maat_ex_data.h/cpp。

Browse Source
This commit is contained in:
zhengchao
2020-05-04 17:46:09 +08:00
parent 9d0d510348
commit 4b4d25b691
15 changed files with 707 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
inc/Maat_rule.h
View File

@@ -293,7 +293,6 @@ typedef void Maat_plugin_EX_free_func_t(int table_id, MAAT_PLUGIN_EX_DATA* ad, l
typedef void Maat_plugin_EX_dup_func_t(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp);
typedef int Maat_plugin_EX_key2index_func_t(const char* key);
//For IP plugin, key2index_func MUST be NULL.
int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
@@ -302,6 +301,14 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
long argl, void *argp);
//Data is duplicated by dup_func of Maat_plugin_EX_register, caller is responsible to FREE the data.
MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key);
int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
long argl, void *argp);
struct ip_address
{
int ip_type; //4: IPv4, 6: IPv6
@@ -312,7 +319,7 @@ struct ip_address
};
};
MAAT_PLUGIN_EX_DATA Maat_IP_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip);
int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data);
enum MAAT_RULE_OPT

2
src/CMakeLists.txt
View File

@@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
include_directories(/opt/MESA/include/MESA/)

72
src/entry/Maat_api.cpp
View File

@@ -1331,38 +1331,24 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int idx=-1;
int ret=0;
struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id);
if(new_func==NULL || free_func==NULL || dup_func==NULL )
{
assert(0);
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
return -1;
}
pthread_mutex_lock(&(_feather->background_update_mutex));
ret=Maat_table_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, key2index_func,argl, argp, _feather->logger);
idx=Maat_table_plugin_new_ex_index(_feather->table_mgr, table_id,
new_func,
free_func,
dup_func,
key2index_func,
argl, argp);
if(idx<0)
if(ret<0)
{
pthread_mutex_unlock(&(_feather->background_update_mutex));
return -1;
}
struct Maat_table_schema *table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
struct Maat_table_runtime* table_rt=NULL;
if(_feather->scanner!=NULL)
{
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
Maat_table_runtime_plugin_new_ex_idx(table_rt, table_desc, _feather->logger);
Maat_table_runtime_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger);
}
pthread_mutex_unlock(&(_feather->background_update_mutex));
return 0;
}
MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key)
@@ -1380,24 +1366,62 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id
exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key);
return exdata;
}
int Maat_IP_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
long argl, void *argp)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_table_schema *table_desc=NULL;
int ret=-1;
struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id);
pthread_mutex_lock(&(_feather->background_update_mutex));
ret=Maat_table_ip_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger);
if(ret<0)
{
pthread_mutex_unlock(&(_feather->background_update_mutex));
return -1;
}
struct Maat_table_runtime* table_rt=NULL;
if(_feather->scanner!=NULL)
{
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
Maat_table_runtime_ip_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger);
}
pthread_mutex_unlock(&(_feather->background_update_mutex));
return 0;
}
int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_table_schema *table_schema=NULL;
struct Maat_table_runtime *table_rt=NULL;
int n_get=0;
if(_feather->scanner==NULL)
{
return 0;
}
table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
if(table_rt->table_type!=TABLE_TYPE_IP_PLUGIN)
{
return -1;
}
n_get=Maat_table_runtime_ip_plugin_get_N_ex_data(table_rt, table_desc, ip, ex_data_array, n_ex_data);
struct ip_data ip_data=*(const struct ip_data*)ip;
if(ip_data.type==IPv4)
{
ip_data.ipv4=ntohl(ip_data.ipv4);
}
else
{
ipv6_ntoh(ip_data.ipv6);
}
n_get=Maat_table_runtime_ip_plugin_get_N_ex_data(table_rt, table_schema, &ip_data, ex_data_array, n_ex_data);
return n_get;
}

193
src/entry/Maat_ex_data.cpp Normal file
View File

@@ -0,0 +1,193 @@
#include "Maat_ex_data.h"
#include "Maat_table.h"
#include "Maat_utils.h"
#include <MESA/MESA_handle_logger.h>
#include <assert.h>
void EX_data_container_free(void *data)
{
struct EX_data_container* wrap_data=(struct EX_data_container*)data;
const struct EX_data_schema* ex_schema=wrap_data->rt->ex_schema;
ex_schema->free_func(wrap_data->rt->table_id, &(wrap_data->ex_data), ex_schema->argl, ex_schema->argp);
if(wrap_data->user_data && wrap_data->rt->user_data_free)
{
wrap_data->rt->user_data_free(wrap_data->user_data);
}
wrap_data->user_data=NULL;
wrap_data->rt=NULL;
free(wrap_data);
return;
}
static MESA_htable_handle EX_data_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index)
{
MESA_htable_handle key2ex_hash=NULL;
unsigned int slot_size=1;
while(estimate_size!=0)
{
estimate_size=estimate_size>>1;
slot_size*=2;
}
if(slot_size==1)
{
slot_size=4096;
}
MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs));
hargs.thread_safe=8;
hargs.hash_slot_size = slot_size;
hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
hargs.expire_time = 0;
hargs.key_comp = NULL;
hargs.key2index = NULL; //Not supported yet.
hargs.recursive = 1;
hargs.data_free = EX_data_container_free;
hargs.data_expire_with_condition = NULL;
key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(key2ex_hash, 0);
return key2ex_hash;
}
struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data))
{
struct EX_data_rt* p=ALLOC(struct EX_data_rt, 1);
p->key2ex_hash=EX_data_hash_new(estimate_size, key2index);
p->cache_rows=dynamic_array_create(4, 1024);
p->table_id=table_id;
p->user_data_free=user_data_free;
return p;
};
void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema)
{
p->ex_schema=schema;
}
void EX_data_rt_free(struct EX_data_rt* p)
{
if(p->cache_rows)
{
dynamic_array_destroy(p->cache_rows, free);
p->cache_rows=NULL;
}
MESA_htable_destroy(p->key2ex_hash, NULL);
return;
}
void EX_data_rt_cache_row(struct EX_data_rt* p, const char* row)
{
size_t len=strlen(row)+1;
char* row_copy=ALLOC(char, len);
memcpy(row_copy, row, len);
p->cache_size+=len;
dynamic_array_write(p->cache_rows, p->cache_row_num, row_copy);
p->cache_row_num++;
return;
}
const char* EX_data_rt_get_cached_row(struct EX_data_rt* p, int i)
{
const char* row=NULL;
row=(const char*)dynamic_array_read(p->cache_rows, i);
return row;
}
void EX_data_rt_clear_row_cache(struct EX_data_rt* p)
{
dynamic_array_destroy(p->cache_rows, free);
p->cache_rows=NULL;
p->cache_row_num=0;
p->cache_size=0;
}
int EX_data_rt_get_row_num(struct EX_data_rt* p)
{
return p->cache_row_num;
}
struct EX_data_container* EX_data_rt_row2EX_data(struct EX_data_rt* ex_rt,
const char* row, const char* key, size_t key_len,
void* user_data, void* logger)
{
MAAT_RULE_EX_DATA ex_data=NULL;
int ret=0;
const struct EX_data_schema* ex_schema=ex_rt->ex_schema;
struct EX_data_container* ex_container=ALLOC(struct EX_data_container, 1);
ex_schema->new_func(ex_rt->table_id, key, row, &ex_data,
ex_schema->argl, ex_schema->argp);
ex_container->ex_data=ex_data;
ex_container->rt=ex_rt;
ex_container->user_data=user_data;
ret=MESA_htable_add(ex_rt->key2ex_hash, (unsigned char*)key, key_len, ex_container);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"EX data add error: duplicated key %.*s of %s",
key_len, key, row);
EX_data_container_free(ex_container);
return NULL;
}
return ex_container;
}
int EX_data_rt_delete_by_row(struct EX_data_rt* ex_rt, const char* row, const char* key, size_t key_len,
void *logger)
{
int ret=0;
ret=MESA_htable_del(ex_rt->key2ex_hash, (const unsigned char*)key, key_len, NULL);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"EX data del error: no such key %.*s of %s",
key_len, key, row);
return -1;
}
return 0;
}
MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const char* key, size_t key_len)
{
struct EX_data_container* container=NULL;
MAAT_RULE_EX_DATA ex_data=NULL;
if(!ex_rt->ex_schema)
{
assert(0);
return NULL;
}
container=(struct EX_data_container*)MESA_htable_search(ex_rt->key2ex_hash,
(const unsigned char*)key, strlen(key));
if(container!=NULL)
{
ex_rt->ex_schema->dup_func(ex_rt->table_id, &(ex_data), &(container->ex_data),
ex_rt->ex_schema->argl, ex_rt->ex_schema->argp);
}
return ex_data;
}
MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container)
{
MAAT_RULE_EX_DATA ex_data=NULL;
ex_rt->ex_schema->dup_func(ex_rt->table_id, &(ex_data), &(container->ex_data),
ex_rt->ex_schema->argl, ex_rt->ex_schema->argp);
return ex_data;
}
struct key2EX_hash_walker
{
EX_data_container_q* listed;
size_t count;
};
void walk_key2EX_hash(const uchar * key, uint size, void * data, void * user)
{
struct key2EX_hash_walker *walker=(struct key2EX_hash_walker*)user;
struct EX_data_container* ex_container=(struct EX_data_container*)data;
TAILQ_INSERT_TAIL(walker->listed, ex_container, entries);
walker->count++;
return;
}
size_t EX_data_rt_list_all(struct EX_data_rt* ex_rt, EX_data_container_q* listed)
{
size_t ex_data_cnt;
struct key2EX_hash_walker walker={listed, 0};
TAILQ_INIT(listed);
MESA_htable_iterate(ex_rt->key2ex_hash, walk_key2EX_hash, &walker);
ex_data_cnt=(size_t)MESA_htable_get_elem_num(ex_rt->key2ex_hash);
assert(walker.count==ex_data_cnt);
return ex_data_cnt;
}

118
src/entry/Maat_rule.cpp
View File

@@ -3000,44 +3000,41 @@ void garbage_bury(MESA_lqueue_head garbage_q, int timeout, void *logger)
q_cnt,bury_cnt);
}
}
void update_plugin_table(struct Maat_table_schema* table, const char* table_line, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
void update_plugin_table(struct Maat_table_schema* table_schema, const char* row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
{
int i=0, ret=1, matched_tag=1;
unsigned int len=strlen(table_line)+1;
struct plugin_table_schema* plugin_desc=&(table->plugin);
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
char *p=NULL;
int ret=1, matched_tag=1;
struct plugin_table_schema* plugin_desc=&(table_schema->plugin);
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_schema->table_id);
char* copy=NULL;
size_t is_valid_offset=0, valid_len=0;
size_t accept_tag_offset=0, accept_tag_len=0;
if(plugin_desc->rule_tag_column>0&&n_tags>0)
{
ret=Maat_helper_read_column(table_line, plugin_desc->rule_tag_column, &accept_tag_offset, &accept_tag_len);
ret=Maat_helper_read_column(row, plugin_desc->rule_tag_column, &accept_tag_offset, &accept_tag_len);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error, could not locate tag in column %d of plugin table %s:%s",
"update error, could not locate tag in column %d of plugin table_schema %s:%s",
plugin_desc->rule_tag_column,
table->table_name[table->updating_name],
table_line);
table->udpate_err_cnt++;
table_schema->table_name[table_schema->updating_name],
row);
table_schema->udpate_err_cnt++;
return;
}
if(accept_tag_len>2)
{
copy=ALLOC(char, accept_tag_len+1);
memcpy(copy, table_line+accept_tag_offset, accept_tag_len);
memcpy(copy, row+accept_tag_offset, accept_tag_len);
matched_tag=compare_accept_tag(copy, tags, n_tags);
if(matched_tag<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid tag format of plugin table %s:%s"
,table->table_name[table->updating_name],table_line);
table->udpate_err_cnt++;
"update error,invalid tag format of plugin table_schema %s:%s"
,table_schema->table_name[table_schema->updating_name],row);
table_schema->udpate_err_cnt++;
}
if(matched_tag==0)
{
table->unmatch_tag_cnt++;
table_schema->unmatch_tag_cnt++;
}
free(copy);
copy=NULL;
@@ -3047,79 +3044,43 @@ void update_plugin_table(struct Maat_table_schema* table, const char* table_line
return;
}
}
table_rt->plugin.acc_line_num++;
if(plugin_desc->have_exdata || plugin_desc->cb_plug_cnt>0)
{
if(plugin_desc->have_exdata)
{
ret=Maat_helper_read_column(table_line, plugin_desc->valid_flag_column, &is_valid_offset, &valid_len);
//thread safe is protected by background_update_mutex
if(atoi(table_line+is_valid_offset)==1)
{
plugin_EX_data_new(table, table_line, table_rt->plugin.key2ex_hash, logger);
}
else
{
plugin_EX_data_free(table_line, plugin_desc->key_column, table_rt->plugin.key2ex_hash, logger);
}
}
if(plugin_desc->cb_plug_cnt>0)
{
for(i=0;i<plugin_desc->cb_plug_cnt;i++)
{
plugin_desc->cb_plug[i].update(table->table_id, table_line, plugin_desc->cb_plug[i].u_para);
}
}
}
else
{
p=ALLOC(char, len);
memcpy(p,table_line,len);
table_rt->plugin.cache_size+=len;
dynamic_array_write(table_rt->plugin.cache_lines, table_rt->plugin.cache_line_num, p);
table_rt->plugin.cache_line_num++;
}
Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger);
}
void update_ip_plugin_table(struct Maat_table_schema* table,const char* table_line,Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
{
int ret=1, matched_tag=1;
struct ip_plugin_table_schema* ip_plugin_schema=&(table->ip_plugin);
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin);
struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin);
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_schema->table_id);
char* copy=NULL;
size_t is_valid_offset=0, valid_len=0;
size_t accept_tag_offset=0, accept_tag_len=0;
if(ip_plugin_schema->rule_tag_column>0&&n_tags>0)
{
ret=Maat_helper_read_column(table_line, ip_plugin_schema->rule_tag_column, &accept_tag_offset, &accept_tag_len);
ret=Maat_helper_read_column(table_row, ip_plugin_schema->rule_tag_column, &accept_tag_offset, &accept_tag_len);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error, could not locate tag in column %d of plugin table %s:%s",
"update error, could not locate tag in column %d of plugin table_schema %s:%s",
ip_plugin_schema->rule_tag_column,
table->table_name[table->updating_name],
table_line);
table->udpate_err_cnt++;
table_schema->table_name[table_schema->updating_name],
table_row);
table_schema->udpate_err_cnt++;
return;
}
if(accept_tag_len>2)
{
copy=ALLOC(char, accept_tag_len+1);
memcpy(copy, table_line+accept_tag_offset, accept_tag_len);
memcpy(copy, table_row+accept_tag_offset, accept_tag_len);
matched_tag=compare_accept_tag(copy, tags, n_tags);
if(matched_tag<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"update error, invalid tag format of ip_plugin table %s:%s",
table->table_name[table->updating_name], table_line);
table->udpate_err_cnt++;
"update error, invalid tag format of ip_plugin table_schema %s:%s",
table_schema->table_name[table_schema->updating_name], table_row);
table_schema->udpate_err_cnt++;
}
if(matched_tag==0)
{
table->unmatch_tag_cnt++;
table_schema->unmatch_tag_cnt++;
}
free(copy);
copy=NULL;
@@ -3129,25 +3090,8 @@ void update_ip_plugin_table(struct Maat_table_schema* table,const char* table_li
return;
}
}
ret=Maat_helper_read_column(table_line, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
//thread safe is protected by background_update_mutex
if(atoi(table_line+is_valid_offset)==1)
{
ret=plugin_EX_data_new(table, table_line, ip_plugin_rt->rowid2ex_hash, logger);
if(ret==0)
{
ip_plugin_rt->row_num++;
}
}
else
{
ret=plugin_EX_data_free(table_line, ip_plugin_schema->row_id_column, ip_plugin_rt->rowid2ex_hash, logger);
if(ret==0)
{
ip_plugin_rt->row_num--;
}
}
Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger);
return;
}
void vector_print(igraph_vector_t *v) {
@@ -3293,7 +3237,7 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q,
}
break;
case TABLE_TYPE_IP_PLUGIN:
Maat_table_runtime_rebuild_ip_matcher(table_rt);
Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt);
old_ip_matcher=Maat_table_runtime_dettach_old_ip_matcher(table_rt);
garbage_bagging(GARBAGE_IP_MATCHER, old_ip_matcher, garbage_q);
break;

2
src/entry/Maat_stat.cpp
View File

@@ -219,7 +219,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
switch(p_table->table_type)
{
case TABLE_TYPE_PLUGIN:
plugin_cache_num+=table_rt->plugin.cache_line_num;
plugin_cache_num+=table_rt->plugin.ex_data_rt->cache_row_num;
plugin_acc_num+=table_rt->plugin.acc_line_num;
break;
case TABLE_TYPE_GROUP:

91
src/entry/Maat_table.cpp
View File

@@ -285,11 +285,11 @@ int read_ip_plugin_table_schema(const char* line, struct Maat_table_schema* p)
free(copy_line);
if(read_cnt<5)
{
return 0;
return -1;
}
else
{
return -1;
return 0;
}
error_out:
free(copy_line);
@@ -869,32 +869,89 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c
return idx;
}
int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id,
void Maat_table_EX_data_schema_set(struct EX_data_schema* ex_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp)
{
struct Maat_table_schema *table_desc=NULL;;
table_desc=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
struct plugin_table_schema* plugin_desc=&(table_desc->plugin);
if(plugin_desc->have_exdata
|| plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0)
ex_schema->new_func=new_func;
ex_schema->free_func=free_func;
ex_schema->dup_func=dup_func;
ex_schema->key2index_func=key2index_func;//Set but not used.
ex_schema->argl=argl;
ex_schema->argp=argp;
return;
}
int Maat_table_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger)
{
if(new_func==NULL || free_func==NULL || dup_func==NULL )
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
return -1;
}
plugin_desc->ex_desc.new_func=new_func;
plugin_desc->ex_desc.free_func=free_func;
plugin_desc->ex_desc.dup_func=dup_func;
plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used.
plugin_desc->ex_desc.argl=argl;
plugin_desc->ex_desc.argp=argp;
plugin_desc->have_exdata=1;
if(table_schema->table_type!=TABLE_TYPE_PLUGIN)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, Regist target is not a plugin table.", __FUNCTION__);
return -1;
}
if(table_schema->plugin.have_exdata)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__);
return -1;
}
if(table_schema->plugin.key_column==0 || table_schema->plugin.valid_flag_column==0)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, not enough schema information.", __FUNCTION__);
return -1;
}
Maat_table_EX_data_schema_set(&table_schema->plugin.ex_schema,
new_func, free_func, dup_func, key2index_func, argl, argp);
table_schema->plugin.have_exdata=1;
return 0;
}
int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger)
{
if(new_func==NULL || free_func==NULL || dup_func==NULL )
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
return -1;
}
if(table_schema->table_type!=TABLE_TYPE_IP_PLUGIN)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a ip_plugin table.", __FUNCTION__);
return -1;
}
if(table_schema->ip_plugin.have_exdata)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__);
return -1;
}
Maat_table_EX_data_schema_set(&table_schema->ip_plugin.ex_schema,
new_func, free_func, dup_func, key2index_func, argl, argp);
table_schema->ip_plugin.have_exdata=1;
return 0;
}
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type)
{
table_mgr->active_plugin_table_num=0;

397
src/entry/Maat_table_runtime.cpp
View File

@@ -7,96 +7,52 @@
#include <assert.h>
#include <sys/queue.h>
struct wrap_plugin_EX_data
{
MAAT_RULE_EX_DATA exdata;
int table_id;
const struct plugin_table_ex_data_schema* ex_desc;
struct ip_rule range_rule;//for ip_plugin
TAILQ_ENTRY(wrap_plugin_EX_data) entries;
};
void wrap_plugin_EX_data_free(void *data)
{
struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data;
wrap_data->ex_desc->free_func(wrap_data->table_id, &(wrap_data->exdata), wrap_data->ex_desc->argl, wrap_data->ex_desc->argp);
wrap_data->ex_desc=NULL;
wrap_data->table_id=-1;
free(wrap_data);
return;
}
MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index)
{
MESA_htable_handle key2ex_hash=NULL;
unsigned int slot_size=1;
while(estimate_size!=0)
{
estimate_size=estimate_size>>1;
slot_size*=2;
}
if(slot_size==1)
{
slot_size=4096;
}
MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs));
hargs.thread_safe=8;
hargs.hash_slot_size = slot_size;
hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
hargs.expire_time = 0;
hargs.key_comp = NULL;
hargs.key2index = NULL; //Not supported yet.
hargs.recursive = 1;
hargs.data_free = wrap_plugin_EX_data_free;
hargs.data_expire_with_condition = NULL;
key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(key2ex_hash, 0);
return key2ex_hash;
}
int plugin_EX_data_free(const char* line, int key_column,
int plugin_EX_data_free(const char* row, int key_column,
MESA_htable_handle key2ex_hash, void *logger)
{
size_t key_offset=0, key_len=0;
int ret=0;
ret=get_column_pos(line, key_column, &key_offset, &key_len);
ret=get_column_pos(row, key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"plugin/ip_plugin EX data del error: cannot find column %d of %s",
key_column, line);
key_column, row);
return -1;
}
ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL);
ret=MESA_htable_del(key2ex_hash, (const unsigned char*)row+key_offset, key_len, NULL);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"plugin/ip_plugin EX data del error: no such key %.*s of %s",
key_len, line+key_offset, line);
key_len, row+key_offset, row);
return -1;
}
return 0;
}
int ip_plugin_line_read(const struct ip_plugin_table_schema* schema, const char* line, struct ip_rule* range_rule)
struct ip_rule* ip_plugin_row2ip_rule(const struct ip_plugin_table_schema* schema, const char* row)
{
struct ip_rule* range_rule=ALLOC(struct ip_rule, 1);
int ret[4]={0};
size_t column_offset=0, column_len=0;
char start_ip[128]={0}, end_ip[128]={0};
ret[0]=get_column_pos(line, schema->row_id_column, &column_offset, &column_len);
range_rule->rule_id=atoi(line+column_offset);
ret[0]=get_column_pos(row, schema->row_id_column, &column_offset, &column_len);
range_rule->rule_id=atoi(row+column_offset);
ret[1]=get_column_pos(line, schema->ip_type_column, &column_offset, &column_len);
int ip_type=atoi(line+column_offset);
ret[1]=get_column_pos(row, schema->ip_type_column, &column_offset, &column_len);
int ip_type=atoi(row+column_offset);
ret[2]=get_column_pos(line, schema->start_ip_column, &column_offset, &column_len);
strncpy(start_ip, line+column_offset, MIN(column_len, sizeof(start_ip)));
ret[2]=get_column_pos(row, schema->start_ip_column, &column_offset, &column_len);
strncpy(start_ip, row+column_offset, MIN(column_len, sizeof(start_ip)));
ret[3]=get_column_pos(line, schema->end_ip_column, &column_offset, &column_len);
strncpy(end_ip, line+column_offset, MIN(column_len, sizeof(end_ip)));
ret[3]=get_column_pos(row, schema->end_ip_column, &column_offset, &column_len);
strncpy(end_ip, row+column_offset, MIN(column_len, sizeof(end_ip)));
if(ret[0]<0||ret[1]<0||ret[2]<0||ret[3]<0)
{
return -1;
free(range_rule);
return NULL;
}
if(ip_type==4)
@@ -111,76 +67,17 @@ int ip_plugin_line_read(const struct ip_plugin_table_schema* schema, const char*
}
else
{
return -1;
free(range_rule);
return NULL;
}
if(ret[0]<0)
{
return -1;
free(range_rule);
return NULL;
}
range_rule->rule_id=(unsigned int)atoi(line+column_offset);
range_rule->user_tag=NULL;
return 0;
return range_rule;
}
int plugin_EX_data_new(const struct Maat_table_schema* table_schema,
const char* line,
MESA_htable_handle key2ex_hash, void *logger)
{
char* key=NULL;
size_t key_offset=0, key_len=0;
MAAT_RULE_EX_DATA exdata=NULL;
const struct plugin_table_ex_data_schema* ex_desc=NULL;
int key_column=-1;
struct wrap_plugin_EX_data* wrap_data=ALLOC(struct wrap_plugin_EX_data, 1);
switch(table_schema->table_type)
{
case TABLE_TYPE_PLUGIN:
ex_desc=&(table_schema->plugin.ex_desc);
key_column=table_schema->plugin.key_column;
break;
case TABLE_TYPE_IP_PLUGIN:
ex_desc=&(table_schema->ip_plugin.ex_desc);
key_column=table_schema->ip_plugin.row_id_column;
ip_plugin_line_read(&table_schema->ip_plugin, line, &(wrap_data->range_rule));
wrap_data->range_rule.user_tag=wrap_data;
break;
default:
assert(0);
break;
}
int ret=0;
ret=get_column_pos(line, key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"plugin/ip_plugin EX data add error: cannot find column %d of %s",
key_column, line);
goto error_out;
}
key=ALLOC(char, key_len+1);
memcpy(key, line+key_offset, key_len);
ex_desc->new_func(table_schema->table_id, key, line, &exdata,
ex_desc->argl, ex_desc->argp);
wrap_data->exdata=exdata;
wrap_data->ex_desc=ex_desc;
wrap_data->table_id=table_schema->table_id;
ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data);
free(key);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"plugin/ip_plugin EX data add error: duplicated key %.*s of %s",
key_len, line+key_offset, line);
wrap_plugin_EX_data_free(wrap_data);
return -1;
}
return 0;
error_out:
free(wrap_data);
free(key);
return -1;
}
struct Maat_table_runtime_manager
{
struct Maat_table_runtime** table_rt;
@@ -217,28 +114,36 @@ static void destroy_digest_rule(GIE_digest_t*rule)
return;
}
static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_desc, int max_thread_num)
static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num)
{
struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1);
table_rt->table_type=table_desc->table_type;
switch(table_desc->table_type)
table_rt->table_type=table_schema->table_type;
switch(table_schema->table_type)
{
case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY:
table_rt->similar.update_q=MESA_lqueue_create(0,0);
break;
case TABLE_TYPE_PLUGIN:
table_rt->plugin.cache_lines=dynamic_array_create(1, 1024);
if(table_desc->plugin.have_exdata)
table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->plugin.estimate_size,
table_schema->plugin.ex_schema.key2index_func,
NULL);
if(table_schema->plugin.have_exdata)
{
table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size,
table_desc->plugin.ex_desc.key2index_func);
EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema);
}
break;
case TABLE_TYPE_IP_PLUGIN:
table_rt->ip_plugin.rowid2ex_hash=wrap_plugin_EX_hash_new(table_desc->ip_plugin.estimate_size,
NULL);
table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->plugin.estimate_size,
table_schema->plugin.ex_schema.key2index_func,
free);
if(table_schema->ip_plugin.have_exdata)
{
EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema);
}
break;
default:
break;
@@ -283,18 +188,14 @@ static void table_runtime_free(struct Maat_table_runtime* p)
MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL);
}
break;
case TABLE_TYPE_PLUGIN:
dynamic_array_destroy(p->plugin.cache_lines, free);
p->plugin.cache_lines=NULL;
if(p->plugin.key2ex_hash!=NULL)
{
MESA_htable_destroy(p->plugin.key2ex_hash, NULL);
}
break;
case TABLE_TYPE_IP_PLUGIN:
MESA_htable_destroy(p->ip_plugin.rowid2ex_hash, NULL);
ip_matcher_free(p->ip_plugin.ip_matcher);
p->ip_plugin.row_num=0;
EX_data_rt_free(p->ip_plugin.ex_data_rt);
assert(p->ip_plugin.old_ip_matcher==NULL);
break;
case TABLE_TYPE_PLUGIN:
EX_data_rt_free(p->plugin.ex_data_rt);
break;
default:
break;
@@ -349,81 +250,83 @@ struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_mana
long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt)
{
struct plugin_runtime* plugin_rt=&(table_rt->plugin);
return plugin_rt->cache_line_num;
return plugin_rt->ex_data_rt->cache_row_num;
}
const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line)
{
const char* line=NULL;
struct plugin_runtime* plugin_rt=&(table_rt->plugin);
line=(const char*)dynamic_array_read(plugin_rt->cache_lines, Nth_line);
line=(const char*)dynamic_array_read(plugin_rt->ex_data_rt->cache_rows, Nth_line);
return line;
}
MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_schema* plugin_table,
struct dynamic_array_t* lines, size_t line_cnt, void* logger)
int Maat_table_runtime_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger)
{
MESA_htable_handle key2ex_hash=NULL;
size_t i=0;
const char* line=NULL;
const struct plugin_table_schema* plugin_desc= &(plugin_table->plugin);
int i=0;
const char* row=NULL;
EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema);
key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func);
for(i=0; i< line_cnt; i++)
for(i=0; i<EX_data_rt_get_row_num(table_rt->plugin.ex_data_rt); i++)
{
line=(const char*)dynamic_array_read(lines, i);
plugin_EX_data_new(plugin_table, line, key2ex_hash, logger);
row=EX_data_rt_get_cached_row(table_rt->plugin.ex_data_rt, i);
Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger);
}
return key2ex_hash;
}
int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, void* logger)
{
assert(table_rt->plugin.key2ex_hash==NULL);
if(table_rt->plugin.key2ex_hash)
{
return -1;
}
table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines,
table_rt->plugin.cache_line_num, logger);
return 0;
}
MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, const char* key)
MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* key)
{
struct plugin_table_schema* plugin_desc=NULL;
struct wrap_plugin_EX_data* wrap_data=NULL;
MAAT_RULE_EX_DATA exdata=NULL;
plugin_desc=&(table_desc->plugin);
if(!plugin_desc->have_exdata)
MAAT_RULE_EX_DATA ex_data=NULL;
if(!table_schema->plugin.have_exdata)
{
assert(0);
return NULL;
}
wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash,
(const unsigned char*)key, strlen(key));
if(wrap_data!=NULL)
{
plugin_desc->ex_desc.dup_func(table_desc->table_id, &(exdata), &(wrap_data->exdata),
plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
}
return exdata;
ex_data=EX_data_rt_get_EX_data_by_key(table_rt->plugin.ex_data_rt, key, strlen(key));
return ex_data;
}
int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size)
void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
{
struct ip_plugin_table_schema* ip_plugin_desc=NULL;
struct scan_result results[size];
int n_result=0, i=0;
int ret=0, i=0;
size_t is_valid_offset=0, valid_len=0;
size_t key_offset=0, key_len=0;
ip_plugin_desc=&(table_schema->ip_plugin);
n_result=ip_matcher_match(table_rt->ip_plugin.ip_matcher, (struct ip_data*)ip, results, size);
for(i=0; i<n_result; i++)
struct plugin_table_schema* plugin_schema=&table_schema->plugin;
struct plugin_runtime* plugin_rt=&table_rt->plugin;
ret=Maat_helper_read_column(row, plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
plugin_rt->acc_line_num++;
if(plugin_schema->have_exdata)
{
ip_plugin_desc->ex_desc.dup_func(table_schema->table_id, &(ex_data_array[i]), &(results[i].tag),
ip_plugin_desc->ex_desc.argl, ip_plugin_desc->ex_desc.argp);
ret=get_column_pos(row, plugin_schema->key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"plugin EX data process error: cannot find column %d of %s",
plugin_schema->key_column, row);
return;
}
if(atoi(row+is_valid_offset)==1)
{
EX_data_rt_row2EX_data(plugin_rt->ex_data_rt, row, row+key_offset, key_len, NULL, logger);
}
else
{
EX_data_rt_delete_by_row(plugin_rt->ex_data_rt, row, row+key_offset, key_len, logger);
}
}
return n_result;
if(plugin_schema->cb_plug_cnt>0)
{
for(i=0; i<plugin_schema->cb_plug_cnt;i++)
{
plugin_schema->cb_plug[i].update(table_schema->table_id, row, plugin_schema->cb_plug[i].u_para);
}
}
if(!plugin_schema->have_exdata && !plugin_schema->cb_plug_cnt)
{
EX_data_rt_cache_row(plugin_rt->ex_data_rt, row);
}
return;
}
void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag)
@@ -518,34 +421,29 @@ int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt)
return q_cnt;
}
TAILQ_HEAD(ip_range_rule_q, wrap_plugin_EX_data);
void walk_ip_plugin_hash(const uchar * key, uint size, void * data, void * user)
{
struct wrap_plugin_EX_data* wrap_plugin_ex=(struct wrap_plugin_EX_data*)data;
struct ip_range_rule_q* queue=(struct ip_range_rule_q*)user;
TAILQ_INSERT_TAIL(queue, wrap_plugin_ex, entries);
return;
}
int Maat_table_runtime_rebuild_ip_matcher(struct Maat_table_runtime* table_rt)
int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt)
{
struct ip_matcher* new_ip_matcher=NULL;
struct ip_range_rule_q queue;//This is for index, no need to free.
struct EX_data_container_q queue;//This is for index, no need to free.
size_t rule_cnt=0;
size_t i=0, mem_use=0;
struct ip_rule *rules=NULL;
struct wrap_plugin_EX_data *p=NULL;
struct EX_data_container *p=NULL;
TAILQ_INIT(&queue);
MESA_htable_iterate(table_rt->ip_plugin.rowid2ex_hash, walk_ip_plugin_hash, &queue);
rule_cnt=(size_t)MESA_htable_get_elem_num(table_rt->ip_plugin.rowid2ex_hash);
rule_cnt=EX_data_rt_list_all(table_rt->ip_plugin.ex_data_rt, &queue);
rules=ALLOC(struct ip_rule, rule_cnt);
TAILQ_FOREACH(p, &queue, entries)
{
rules[i]=p->range_rule;
rules[i]=*((struct ip_rule *)(p->user_data));
assert(rules[i].user_tag==p||rules[i].user_tag==NULL);
rules[i].user_tag=p;
i++;
}
assert(i==rule_cnt);
if(rule_cnt==0)
{
return 0;
}
new_ip_matcher=ip_matcher_new(rules, rule_cnt, &mem_use);
table_rt->ip_plugin.old_ip_matcher=table_rt->ip_plugin.ip_matcher;
table_rt->ip_plugin.ip_matcher=new_ip_matcher;
@@ -558,5 +456,82 @@ struct ip_matcher* Maat_table_runtime_dettach_old_ip_matcher(struct Maat_table_r
table_rt->ip_plugin.old_ip_matcher=NULL;
return old_one;
}
void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
{
struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin);
struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin);
size_t is_valid_offset=0, valid_len=0;
size_t key_offset=0, key_len=0;
struct ip_rule* ip_rule=NULL;
int ret=0;
if(ip_plugin_schema->have_exdata)
{
ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"ip_plugin EX data process error: cannot find is_valid column %d of %s",
ip_plugin_schema->row_id_column, row);
return;
}
ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"ip_plugin EX data process error: cannot find row id column %d of %s",
ip_plugin_schema->row_id_column, row);
return;
}
ip_rule=ip_plugin_row2ip_rule(ip_plugin_schema, row);
if(ip_rule==NULL)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"ip_plugin read ip error: %s", row);
return;
}
if(atoi(row+is_valid_offset)==1)//add
{
EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+key_offset, key_len, ip_rule, logger);
}
else
{
EX_data_rt_delete_by_row(ip_plugin_rt->ex_data_rt, row, row+key_offset, key_len, logger);
}
}
else
{
EX_data_rt_cache_row(ip_plugin_rt->ex_data_rt, row);
}
return;
}
int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger)
{
int i=0;
const char* row=NULL;
EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema);
for(i=0; i<EX_data_rt_get_row_num(table_rt->plugin.ex_data_rt); i++)
{
row=EX_data_rt_get_cached_row(table_rt->plugin.ex_data_rt, i);
Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, row, logger);
}
EX_data_rt_clear_row_cache(table_rt->plugin.ex_data_rt);
Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt);
return 0;
}
int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size)
{
struct scan_result results[size];
int n_result=0, i=0;
n_result=ip_matcher_match(table_rt->ip_plugin.ip_matcher, (struct ip_data*)ip, results, size);
for(i=0; i<n_result; i++)
{
ex_data_array[i]=EX_data_rt_get_EX_data_by_container(table_rt->ip_plugin.ex_data_rt, (struct EX_data_container *)results[i].tag);
}
return n_result;
}

45
src/inc_internal/Maat_ex_data.h Normal file
View File

@@ -0,0 +1,45 @@
#include "dynamic_array.h"
#include "Maat_rule.h"
#include <MESA/MESA_htable.h>
#include <sys/queue.h>
struct EX_data_rt
{
dynamic_array_t *cache_rows;
long long cache_row_num;
long long cache_size;
MESA_htable_handle key2ex_hash;
const struct EX_data_schema* ex_schema;
int table_id;
void (* user_data_free)(void *user_data);
};
struct EX_data_container
{
MAAT_RULE_EX_DATA ex_data;
const struct EX_data_rt* rt;
void* user_data;
TAILQ_ENTRY(EX_data_container) entries;
};
TAILQ_HEAD(EX_data_container_q, EX_data_container);
struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data));
void EX_data_rt_free(struct EX_data_rt* p);
void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema);
void EX_data_rt_cache_row(struct EX_data_rt* p, const char* row);
const char* EX_data_rt_get_cached_row(struct EX_data_rt* p, int i);
void EX_data_rt_clear_row_cache(struct EX_data_rt* p);
int EX_data_rt_get_row_num(struct EX_data_rt* p);
struct EX_data_container* EX_data_rt_row2EX_data(struct EX_data_rt* ex_rt,
const char* row, const char* key, size_t key_len,
void* user_data, void* logger);
int EX_data_rt_delete_by_row(struct EX_data_rt* ex_rt, const char* row, const char* key, size_t key_len, void *logger);
MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const char* key, size_t key_len);
MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container);
size_t EX_data_rt_list_all(struct EX_data_rt* ex_rt, EX_data_container_q* listed);

3
src/inc_internal/Maat_rule_internal.h
View File

@@ -457,9 +457,6 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p);
void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len);
MAAT_RULE_EX_DATA rule_ex_data_new(const struct Maat_rule_head * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc);
void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc);
MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index);
int plugin_EX_data_new(const struct Maat_table_schema* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger);
int plugin_EX_data_free(const char* line, int key_column, MESA_htable_handle key2ex_hash, void *logger);
void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout);

20
src/inc_internal/Maat_table.h
View File

@@ -79,7 +79,7 @@ struct plugin_table_callback_schema
Maat_finish_callback_t *finish;
void* u_para;
};
struct plugin_table_ex_data_schema
struct EX_data_schema
{
Maat_plugin_EX_new_func_t* new_func;
Maat_plugin_EX_free_func_t* free_func;
@@ -99,7 +99,7 @@ struct plugin_table_schema
int have_exdata;
long long estimate_size;
struct plugin_table_callback_schema cb_plug[MAX_PLUGIN_PER_TABLE];
struct plugin_table_ex_data_schema ex_desc;
struct EX_data_schema ex_schema;
};
struct ip_plugin_table_schema
{
@@ -110,7 +110,8 @@ struct ip_plugin_table_schema
int valid_flag_column;
int rule_tag_column;
long long estimate_size;
struct plugin_table_ex_data_schema ex_desc;
int have_exdata;
struct EX_data_schema ex_schema;
};
struct Maat_table_schema
{
@@ -161,12 +162,21 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c
Maat_rule_EX_dup_func_t* dup_func,
long argl, void *argp);
struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx);
int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id,
int Maat_table_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp);
long argl, void *argp,
void* logger);
int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger);
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type);
void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr);

21
src/inc_internal/Maat_table_runtime.h
View File

@@ -1,4 +1,5 @@
#include "Maat_table.h"
#include "Maat_ex_data.h"
#include "IPMatcher.h"
#include "gram_index_engine.h"
#include "alignment_int64.h"
@@ -14,16 +15,13 @@ struct similar_runtime
struct plugin_runtime
{
dynamic_array_t *cache_lines;
long long cache_line_num;
struct EX_data_rt* ex_data_rt;
long long acc_line_num;
long long cache_size;
MESA_htable_handle key2ex_hash;
};
struct ip_plugin_runtime
{
long long row_num;
MESA_htable_handle rowid2ex_hash;
struct EX_data_rt* ex_data_rt;
struct ip_matcher* ip_matcher;
struct ip_matcher* old_ip_matcher;
};
@@ -69,13 +67,18 @@ struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_mana
long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt);
const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line);
int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, void* logger);
int Maat_table_runtime_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, void* logger);
MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, const char* key);
void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag);
void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id);
int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size);
int Maat_table_runtime_rebuild_ip_matcher(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);
int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size);
int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt);
struct ip_matcher* Maat_table_runtime_dettach_old_ip_matcher(struct Maat_table_runtime* table_rt);

9
test/maat_json.json
View File

@@ -1754,6 +1754,15 @@
"1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34",
"161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42"
]
},
{
"table_name": "TEST_IP_PLUGIN_WITH_EXDATA",
"table_content": [
"101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1",
"102\t4\t192.168.30.90\t192.168.30.128\tBigger-range-should-in-the-back\t1",
"103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1",
"104\t6\t2001:db8:1234::1\t2001:db8:1234::5210\tSomething-like-json\t1"
]
}
]
}

3
test/table_info.conf
View File

@@ -47,4 +47,5 @@
24 COMPOSITION_IP_SOURCE virtual IP_PLUS_CONFIG --
25 COMPOSITION_IP_DESTINATION virtual IP_PLUS_CONFIG --
26 COMPOSITION_IP_SESSION virtual IP_PLUS_CONFIG --
27 COMPOSITION_IP composition {"source":"COMPOSITION_IP_SOURCE","destination":"COMPOSITION_IP_DESTINATION","session":"COMPOSITION_IP_SESSION"}
27 COMPOSITION_IP composition {"source":"COMPOSITION_IP_SOURCE","destination":"COMPOSITION_IP_DESTINATION","session":"COMPOSITION_IP_SESSION"}
28 TEST_IP_PLUGIN_WITH_EXDATA ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":6} --

83
test/test_maatframe.cpp
View File

@@ -101,7 +101,7 @@ void scan_with_old_or_new_cfg(Maat_feather_t feather, int is_old)
if(!is_old)
{
EXPECT_EQ(ret, 1);
EXPECT_TRUE(result.config_id==2);
EXPECT_EQ(result.config_id, 2);
}
else
{
@@ -219,7 +219,7 @@ void test_plugin_table(Maat_feather_t feather,const char* table_name,
ASSERT_GT(ret, 0);
}
#define Plugin_callback
TEST(PluginTable, Callback)
{
test_plugin_table(g_feather, "QD_ENTRY_INFO",
@@ -229,6 +229,85 @@ TEST(PluginTable, Callback)
g_feather,
g_logger);
}
#define IP_PLUGIN_EX_DATA
struct ip_plugin_ud
{
int rule_id;
char* buffer;
int ref_cnt;
};
void ip_plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{
int *counter=(int *)argp, ret=0;
size_t column_offset=0, column_len=0;
struct ip_plugin_ud* ud=(struct ip_plugin_ud*)calloc(sizeof(struct ip_plugin_ud), 1);
ret=Maat_helper_read_column(table_line, 1, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->rule_id=atoi(table_line+column_offset);
ret=Maat_helper_read_column(table_line, 5, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->buffer=(char*)calloc(sizeof(char), column_len+1);
strncpy(ud->buffer, table_line+column_offset, column_len);
ud->ref_cnt=1;
*ad=ud;
(*counter)++;
return;
}
void ip_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{
struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*ad);
u->ref_cnt--;
if(u->ref_cnt>0) return;
free(u->buffer);
free(u);
*ad=NULL;
}
void ip_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
{
struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*from);
u->ref_cnt++;
*to=u;
}
TEST(IP_Plugin_Table, EX_DATA)
{
int ip_plugin_ex_data_counter=0, i=0;
const char* table_name="TEST_IP_PLUGIN_WITH_EXDATA";
int table_id=0, ret=0;
table_id=Maat_table_register(g_feather, table_name);
ASSERT_GT(table_id, 0);
ret=Maat_ip_plugin_EX_register(g_feather, table_id,
ip_plugin_EX_new_cb,
ip_plugin_EX_free_cb,
ip_plugin_EX_dup_cb,
0, &ip_plugin_ex_data_counter);
ASSERT_TRUE(ret>=0);
EXPECT_EQ(ip_plugin_ex_data_counter, 4);
struct ip_address ipv4, ipv6;
struct ip_plugin_ud* result[4];
ipv4.ip_type=4;
inet_pton(AF_INET, "192.168.30.100", &(ipv4.ipv4));
ret=Maat_ip_plugin_get_EX_data(g_feather, table_id, &ipv4, (void**)result, 4);
ASSERT_EQ(ret, 2);
EXPECT_EQ(result[0]->rule_id, 101);
EXPECT_EQ(result[1]->rule_id, 102);
for(i=0; i<ret; i++)
{
ip_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
}
ipv6.ip_type=6;
inet_pton(AF_INET6,"2001:db8:1234::5210",&(ipv6.ipv6));
ret=Maat_ip_plugin_get_EX_data(g_feather, table_id, &ipv6, (void**)result, 4);
ASSERT_EQ(ret, 2);
EXPECT_EQ(result[0]->rule_id, 104);
EXPECT_EQ(result[1]->rule_id, 103);
for(i=0; i<ret; i++)
{
ip_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
}
}
TEST(StringScan, Full)