gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FQDN Plugin加载、扫描、更新测试通过，遗留FQDN_engine后缀匹配不命中的Bug待修复。

Browse Source
This commit is contained in:
zhengchao
2020-09-19 21:11:38 +08:00
parent 58daab14ad
commit 0713a4a576
16 changed files with 770 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
inc/Maat_rule.h
View File

@@ -319,6 +319,14 @@ struct ip_address
int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data);
int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
long argl, void *argp);
int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data);
enum MAAT_RULE_OPT
{

2
src/CMakeLists.txt
View File

@@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp entry/FQDN_engine.cpp)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
include_directories(/opt/MESA/include/MESA/)

12
src/entry/FQDN_engine.cpp
View File

@@ -7,7 +7,7 @@
* All rights reserved
*
* Written by: LIU YANBING (liuyanbing@iie.ac.cn)
* Last modification: 2020-09-01
* Last modification: 2020-09-19
*
* This code is the exclusive and proprietary property of IIE-CAS and NELIST.
* Usage for direct or indirect commercial advantage is not allowed without
@@ -23,8 +23,10 @@
#include <string.h>
/*************************************************************************************/
#include <nmmintrin.h>
#define popcnt_u64 _mm_popcnt_u64
//#include <nmmintrin.h>
//#define popcnt_u64 _mm_popcnt_u64
//Use gcc builtin function to replace SSE4.2 instruction for portability
#define popcnt_u64 __builtin_popcountl
#define FOR(i, n) for(int i=0, _n=(int)(n); i<_n; i++)
@@ -241,7 +243,7 @@ int CHashTrieFQDN::initialize(const struct FQDN_rule * rules, size_t n_rule)
m_matched[idx]=&(m_domains[k]);
}
printf("mem_bytes=%u(MB)\n", mem_bytes/(1U<<20));
// printf("mem_bytes=%ll(MB)\n", mem_bytes/(1U<<20));
return 1;
}
@@ -260,7 +262,7 @@ int CHashTrieFQDN::search(const char * FQDN, size_t FQDN_len, struct FQDN_match
{
if(m_num==0 || FQDN_len==0 || FQDN==NULL || n_result==0) return -1;
int match_num=0;
size_t match_num=0;
const unsigned char * pb=(unsigned char *)FQDN;
unsigned long long HASH[16]; /*<2A><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>16*/
unsigned int P[16];

103
src/entry/Maat_api.cpp
View File

@@ -1125,6 +1125,12 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id
struct Maat_table_schema *table_desc=NULL;
struct Maat_table_runtime *table_rt=NULL;
MAAT_RULE_EX_DATA exdata=NULL;
struct timespec start,end;
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
if(_feather->scanner==NULL)
{
return NULL;
@@ -1132,6 +1138,17 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id
table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key);
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_rt, 0, &start, &end, 0);
}
else
{
maat_stat_table(table_rt, 0, NULL, NULL, 0);
}
return exdata;
}
int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id,
@@ -1163,6 +1180,35 @@ int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id,
return 0;
}
int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
long argl, void *argp)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int ret=-1;
struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id);
pthread_mutex_lock(&(_feather->background_update_mutex));
ret=Maat_table_fqdn_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger);
if(ret<0)
{
pthread_mutex_unlock(&(_feather->background_update_mutex));
return -1;
}
struct Maat_table_runtime* table_rt=NULL;
if(_feather->scanner!=NULL)
{
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
Maat_table_runtime_fqdn_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger);
}
pthread_mutex_unlock(&(_feather->background_update_mutex));
return 0;
}
int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
{
@@ -1170,11 +1216,18 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc
struct Maat_table_schema *table_schema=NULL;
struct Maat_table_runtime *table_rt=NULL;
int n_get=0;
struct timespec start,end;
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
if(_feather->scanner==NULL)
{
return 0;
}
table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_IP_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
if(table_rt->table_type!=TABLE_TYPE_IP_PLUGIN)
{
@@ -1190,9 +1243,57 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc
ipv6_ntoh(ip_data.ipv6);
}
n_get=Maat_table_runtime_ip_plugin_get_N_ex_data(table_rt, table_schema, &ip_data, ex_data_array, n_ex_data);
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_rt, 0, &start, &end, 0);
}
else
{
maat_stat_table(table_rt, 0, NULL, NULL, 0);
}
return n_get;
}
int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_table_schema *table_schema=NULL;
struct Maat_table_runtime *table_rt=NULL;
int n_get=0;
struct timespec start,end;
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
if(_feather->scanner==NULL)
{
return 0;
}
table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_FQDN_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN)
{
return -1;
}
n_get=Maat_table_runtime_fqdn_plugin_get_N_ex_data(table_rt, table_schema, fqdn, ex_data_array, n_ex_data);
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_rt, 0, &start, &end, 0);
}
else
{
maat_stat_table(table_rt, 0, NULL, NULL, 0);
}
return n_get;
}
int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
,enum MAAT_CHARSET charset,const char* data,int data_len

48
src/entry/Maat_command.cpp
View File

@@ -201,6 +201,7 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_
break;
case TABLE_TYPE_PLUGIN:
case TABLE_TYPE_IP_PLUGIN:
case TABLE_TYPE_FQDN_PLUGIN:
if(valid_column_seq<0)
{
return -1;
@@ -1624,7 +1625,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
int update_type=CM_UPDATE_TYPE_INC;
long long new_version=0;
enum MAAT_TABLE_TYPE table_type;
struct Maat_table_schema* table_desc=NULL;
struct Maat_table_schema* table_schema=NULL;
void* logger=feather->logger;
if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write
@@ -1713,26 +1714,12 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
}
table_type=Maat_table_get_type_by_id(feather->table_mgr, table_id);
if(rule_list[i].op==MAAT_OP_DEL)
{
if(table_type==TABLE_TYPE_PLUGIN||table_type==TABLE_TYPE_IP_PLUGIN)
{
table_desc=Maat_table_get_scan_by_id(feather->table_mgr, table_id, table_type, NULL);
if(table_type==TABLE_TYPE_PLUGIN)
{
valid_column=table_desc->plugin.valid_flag_column;
}
else
{
valid_column=table_desc->ip_plugin.valid_flag_column;
}
}
else
{
valid_column=-1;
{
table_schema=Maat_table_get_scan_by_id(feather->table_mgr, table_id, table_type, NULL);
valid_column=Maat_table_xx_plugin_table_get_valid_flag_column(table_schema);
ret=invalidate_line(rule_list[i].table_line, table_type, valid_column);
if(ret<0)
{
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Invalidate line failed, invaid format %s ."
,rule_list[i].table_line);
continue;
@@ -1918,22 +1905,17 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_
ret=-1;
goto error_out;
}
int valid_flag_column=0;
int valid_flag_column=0;
valid_flag_column=Maat_table_xx_plugin_table_get_valid_flag_column(p_table);
if(valid_flag_column<0)
{
case TABLE_TYPE_PLUGIN:
valid_flag_column=p_table->plugin.valid_flag_column;
plugin_desc=&(p_table->plugin);
break;
case TABLE_TYPE_IP_PLUGIN:
valid_flag_column=p_table->ip_plugin.valid_flag_column;
break;
default:
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command
,"Command set line id %d failed: table %s is not a plugin or ip_plugin table."
, line_rule[i]->rule_id
, line_rule[i]->table_name);
ret=-1;
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command
,"Command set line id %d failed: table %s is not a plugin or ip_plugin table."
, line_rule[i]->rule_id
, line_rule[i]->table_name);
ret=-1;
goto error_out;
}

6
src/entry/Maat_ex_data.cpp
View File

@@ -57,7 +57,7 @@ void cache_row_free(void*p)
}
UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data))
struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data))
{
struct EX_data_rt* p=ALLOC(struct EX_data_rt, 1);
p->hash_key2ex=NULL;
@@ -207,3 +207,7 @@ void* EX_data_container_get_user_data(struct EX_data_container* ex_container)
{
return ex_container->user_data;
}
size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt)
{
return HASH_COUNT(ex_rt->hash_key2ex);
}

129
src/entry/Maat_rule.cpp
View File

@@ -34,7 +34,7 @@
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
int MAAT_FRAME_VERSION_3_0_20200917=1;
int MAAT_FRAME_VERSION_3_0_20200919=1;
int is_valid_table_name(const char* str)
{
@@ -1392,10 +1392,7 @@ int add_digest_rule(struct Maat_table_schema* table, struct db_digest_rule* db_r
scanner->gie_update_q_size++;
return 0;
}
int add_fqdn_rule(struct Maat_table_schema* table, struct db_digest_rule* db_rule, struct Maat_scanner *scanner,void* logger)
{
}
int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id, int rule_type, struct Maat_scanner *maat_scanner, void* logger)
{
@@ -1435,10 +1432,6 @@ int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id
table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, table->table_id);
Maat_table_runtime_digest_del(table_rt, region->expr_id_lb);
maat_scanner->gie_update_q_size++;
break;
case TABLE_TYPE_FQDN:
case TABLE_TYPE_FQDN_PLUGIN:
break;
default:
assert(0);
@@ -2172,66 +2165,6 @@ error_out:
digest_rule=NULL;
return;
}
void update_fqdn_rule(struct Maat_table_schema* table, const char* table_line, struct Maat_scanner *scanner, void* logger)
{
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
struct db_fqdn_rule* fqdn_rule=ALLOC(struct db_fqdn_rule, 1);
int ret=0;
char fqdn_buff[MAX_TABLE_LINE_SIZE]={'\0'};
ret=sscanf(table_line,"%d\t%d\t%llu\t%s\t%hd\t%d", &(fqdn_rule->region_id),
&(fqdn_rule->group_id),
&(fqdn_rule->is_suffix_match),
fqdn_buff,
&(fqdn_rule->is_valid));
fqdn_rule->fqdn=fqdn_buff;
if(ret!=5)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error, invalid format of fqdn table %s:%s"
,table->table_name[table->updating_name],table_line);
table->udpate_err_cnt++;
goto error_out;
}
if(fqdn_rule->is_valid==FALSE)
{
//digest rule is not build with rulescan, this rule type is useless in count_rs_region funciton.
ret=del_region_rule(table, fqdn_rule->region_id, fqdn_rule->group_id, 0, scanner, logger);
if(ret<0)
{
table->udpate_err_cnt++;
}
else
{
table_rt->origin_rule_num--;
}
}
else
{
ret=add_fqdn_rule(table, digest_rule,scanner,logger);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"duplicate config of intval table %s config_id=%d"
,table->table_name[table->updating_name],digest_rule->region_id);
table->udpate_err_cnt++;
}
else
{
table_rt->origin_rule_num++;
}
}
error_out:
fqdn_rule->fqdn=NULL;
free(fqdn_rule);
fqdn_rule=NULL;
return;
}
void update_plugin_table(struct Maat_table_schema* table_schema, const char* row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
{
int ret=1, matched_tag=1;
@@ -2278,21 +2211,34 @@ void update_plugin_table(struct Maat_table_schema* table_schema, const char* row
}
Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger);
}
void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
void update_xx_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
{
int ret=1, matched_tag=1;
struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin);
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_schema->table_id);
char* copy=NULL;
size_t accept_tag_offset=0, accept_tag_len=0;
if(ip_plugin_schema->rule_tag_column>0&&n_tags>0)
int rule_tag_column=-1;
if(table_schema->table_type==TABLE_TYPE_IP_PLUGIN)
{
ret=Maat_helper_read_column(table_row, ip_plugin_schema->rule_tag_column, &accept_tag_offset, &accept_tag_len);
rule_tag_column=table_schema->ip_plugin.rule_tag_column;
}
else if(table_schema->table_type==TABLE_TYPE_FQDN_PLUGIN)
{
rule_tag_column=table_schema->fqdn_plugin.rule_tag_column;
}
else
{
assert(0);
return;
}
if(rule_tag_column>0&&n_tags>0)
{
ret=Maat_helper_read_column(table_row, rule_tag_column, &accept_tag_offset, &accept_tag_len);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error, could not locate tag in column %d of plugin table_schema %s:%s",
ip_plugin_schema->rule_tag_column,
"update error, could not locate tag in column %d of table %s:%s",
rule_tag_column,
table_schema->table_name[table_schema->updating_name],
table_row);
table_schema->udpate_err_cnt++;
@@ -2322,8 +2268,15 @@ void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char*
return;
}
}
Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger);
scanner->ip_plugin_update_q_size++;
if(table_schema->table_type==TABLE_TYPE_IP_PLUGIN)
{
Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger);
}
else //TABLE_TYPE_FQDN_PLUGIN
{
Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger);
}
scanner->xx_plugin_update_q_size++;
return;
}
@@ -2331,9 +2284,9 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
{
MESA_htable_handle tmp_map=NULL;
struct Maat_table_runtime* table_rt=NULL;
struct ip_matcher* old_ip_matcher=NULL;
int i=0, ret=0;
struct ip_matcher* old_ip_matcher=NULL;
struct FQDN_engine* old_fqdn_engine=NULL;
if(scanner->to_update_compile_cnt+scanner->to_update_group_cnt>0)
{
ret=Maat_hierarchy_rebuild(scanner->hier);
@@ -2368,7 +2321,6 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
{
case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY:
ret=Maat_table_runtime_digest_batch_udpate(table_rt);
if(ret<0)
{
@@ -2387,6 +2339,18 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
}
}
break;
case TABLE_TYPE_FQDN_PLUGIN:
ret=Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt);
if(ret)
{
old_fqdn_engine=Maat_table_runtime_apply_new_fqdn_engine(table_rt);
if(old_fqdn_engine)
{
Maat_garbage_bagging(scanner->ref_garbage_bin, old_fqdn_engine, (void (*)(void*))FQDN_engine_free);
}
}
break;
default:
break;
}
@@ -2403,7 +2367,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
scanner->gie_update_q_size=0;
scanner->to_update_group_cnt=0;
scanner->to_update_compile_cnt=0;
scanner->ip_plugin_update_q_size=0;
scanner->xx_plugin_update_q_size=0;
return;
}
@@ -2468,7 +2432,7 @@ void maat_finish_cb(void* u_para)
feather->scanner->cfg_num=scanner_rule_num(feather->scanner);
feather->scanner->version=feather->maat_version;
expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->ip_plugin_update_q_size;
feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->xx_plugin_update_q_size;
if(time(NULL)-feather->scanner->last_update_time>=feather->effect_interval_ms/1000)
{
do_scanner_update(feather->scanner,
@@ -2548,7 +2512,8 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
update_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
break;
case TABLE_TYPE_IP_PLUGIN:
update_ip_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
case TABLE_TYPE_FQDN_PLUGIN:
update_xx_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
break;
default:
break;

170
src/entry/Maat_table.cpp
View File

@@ -119,6 +119,7 @@ int _read_integer_arrary(char* string, int *array, int size)
}
#define COLUMN_PLUGIN_SCHEMA_JSON 4
#define COLUMN_IP_PLUGIN_SCHEMA_JSON 4
#define COLUMN_FQDN_PLUGIN_SHCEMA_JSON 4
#define COLUMN_COMPOSITION_SCHEMA_JSON 4
int read_plugin_table_schema(const char* line, struct Maat_table_schema* p)
@@ -173,12 +174,6 @@ int read_plugin_table_schema(const char* line, struct Maat_table_schema* p)
assert(tmp->type==cJSON_Number);
plugin_desc->rule_tag_column=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "estimate_size");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
plugin_desc->estimate_size=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "foreign");
if(tmp!=NULL)
{
@@ -271,19 +266,87 @@ int read_ip_plugin_table_schema(const char* line, struct Maat_table_schema* p)
//read_cnt++; Tag is optional, so NOT ++ intentionally.
}
ip_plugin_schema->estimate_size=4096;
tmp=cJSON_GetObjectItem(json, "estimate_size");
cJSON_Delete(json);
free(copy_line);
if(read_cnt<5)
{
return -1;
}
else
{
return 0;
}
error_out:
free(copy_line);
return -1;
}
int read_fqdn_plugin_table_schema(const char* line, struct Maat_table_schema* p)
{
int ret=0, read_cnt=0;
size_t offset=0, len=0;
cJSON* json=NULL, *tmp=NULL;
char* copy_line=NULL, *fqnd_plugin_schema_json=NULL;
struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(p->fqdn_plugin);
copy_line=_maat_strdup(line);
ret=get_column_pos(copy_line, COLUMN_FQDN_PLUGIN_SHCEMA_JSON, &offset, &len);
if(ret<0)
{
goto error_out;
}
if(offset+len<strlen(copy_line))
{
copy_line[offset+len+1]='\0';
}
fqnd_plugin_schema_json=copy_line+offset;
json=cJSON_Parse(fqnd_plugin_schema_json);
if(!json)
{
goto error_out;
}
tmp=cJSON_GetObjectItem(json, "row_id");
if(tmp!=NULL && tmp->type==cJSON_Number)
{
fqdn_plugin_schema->row_id_column=tmp->valueint;
read_cnt++;
}
tmp=cJSON_GetObjectItem(json, "is_suffix_match");
if(tmp!=NULL && tmp->type==cJSON_Number)
{
fqdn_plugin_schema->is_suffix_flag_column=tmp->valueint;
read_cnt++;
}
tmp=cJSON_GetObjectItem(json, "fqdn");
if(tmp!=NULL && tmp->type==cJSON_Number)
{
fqdn_plugin_schema->fqdn_column=tmp->valueint;
read_cnt++;
}
tmp=cJSON_GetObjectItem(json, "valid");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
ip_plugin_schema->estimate_size=tmp->valueint;
//read_cnt++; estimate_size is optional, so NOT ++ intentionally.
fqdn_plugin_schema->valid_flag_column=tmp->valueint;
read_cnt++;
}
fqdn_plugin_schema->rule_tag_column=-1;
tmp=cJSON_GetObjectItem(json, "tag");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
fqdn_plugin_schema->rule_tag_column=tmp->valueint;
//read_cnt++; Tag is optional, so NOT ++ intentionally.
}
cJSON_Delete(json);
free(copy_line);
if(read_cnt<5)
if(read_cnt<4)
{
return -1;
}
@@ -491,7 +554,6 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
map_register(string2int_map,"compile", TABLE_TYPE_COMPILE);
map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN);
map_register(string2int_map,"ip_plugin", TABLE_TYPE_IP_PLUGIN);
map_register(string2int_map,"fqdn", TABLE_TYPE_FQDN);
map_register(string2int_map,"fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN);
map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL);
map_register(string2int_map,"interval", TABLE_TYPE_INTERVAL);
@@ -572,9 +634,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
ret=read_plugin_table_schema(line, p);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n", table_info_path,i);
fprintf(stderr,"Maat read table info %s line %d error:illegal plugin table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal plugin info.", table_info_path,i);
"Maat read table info %s line %d error:illegal plugin table schema.", table_info_path,i);
goto invalid_table;
}
break;
@@ -582,9 +644,19 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
ret=read_ip_plugin_table_schema(line, p);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal ip_plugin info.\n", table_info_path,i);
fprintf(stderr,"Maat read table info %s line %d error:illegal ip_plugin table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal ip_plugin info.", table_info_path,i);
"Maat read table info %s line %d error:illegal ip_plugin table schema.", table_info_path,i);
goto invalid_table;
}
break;
case TABLE_TYPE_FQDN_PLUGIN:
ret=read_fqdn_plugin_table_schema(line, p);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal fqdn_plugin table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal fqdn_plugin table schema.", table_info_path,i);
goto invalid_table;
}
break;
@@ -592,9 +664,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
ret=read_composition_table_schema(line, p, string2int_map);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal composition info.\n", table_info_path,i);
fprintf(stderr,"Maat read table info %s line %d error:illegal composition table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal composition info.", table_info_path,i);
"Maat read table info %s line %d error:illegal composition table schema.", table_info_path,i);
goto invalid_table;
}
break;
@@ -602,9 +674,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
ret=read_virtual_table_schema(line, p, string2int_map);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal virtual info.\n", table_info_path,i);
fprintf(stderr,"Maat read table info %s line %d error:illegal virtual table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal virtual info.", table_info_path,i);
"Maat read table info %s line %d error:illegal virtual table schema.", table_info_path,i);
goto invalid_table;
}
break;
@@ -618,9 +690,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
{
p->compile.user_region_encoding=USER_REGION_ENCODE_NONE;
}
break
case TABLE_TYPE_FQDN:
break;
default:
break;
}
@@ -976,6 +1046,39 @@ int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_sche
table_schema->ip_plugin.have_exdata=1;
return 0;
}
int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger)
{
if(new_func==NULL || free_func==NULL || dup_func==NULL )
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
return -1;
}
if(table_schema->table_type!=TABLE_TYPE_FQDN_PLUGIN)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a fqdn_plugin table.", __FUNCTION__);
return -1;
}
if(table_schema->fqdn_plugin.have_exdata)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__);
return -1;
}
Maat_table_EX_data_schema_set(&table_schema->fqdn_plugin.ex_schema,
new_func, free_func, dup_func, key2index_func, argl, argp);
table_schema->fqdn_plugin.have_exdata=1;
return 0;
}
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type)
{
table_mgr->active_plugin_table_num=0;
@@ -1096,4 +1199,23 @@ int Maat_table_get_child_id(struct Maat_table_schema* p_table, enum MAAT_TABLE_C
}
return ret;
}
int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table)
{
int valid_flag_column=-1;
switch(p_table->table_type)
{
case TABLE_TYPE_PLUGIN:
valid_flag_column=p_table->plugin.valid_flag_column;
break;
case TABLE_TYPE_IP_PLUGIN:
valid_flag_column=p_table->ip_plugin.valid_flag_column;
break;
case TABLE_TYPE_FQDN_PLUGIN:
valid_flag_column=p_table->fqdn_plugin.valid_flag_column;
break;
default:
valid_flag_column=-1;
break;
}
return valid_flag_column;
}

249
src/entry/Maat_table_runtime.cpp
View File

@@ -113,6 +113,47 @@ static void destroy_digest_rule(GIE_digest_t*rule)
rule=NULL;
return;
}
struct FQDN_rule* fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_len, int is_suffix_match)
{
struct FQDN_rule* fqdn_rule=ALLOC(struct FQDN_rule, 1);
fqdn_rule->FQDN=ALLOC(char, fqdn_len+1);
memcpy(fqdn_rule->FQDN, fqdn, fqdn_len);
fqdn_rule->len=fqdn_len;
fqdn_rule->is_suffix_match=is_suffix_match;
fqdn_rule->id=id;
return fqdn_rule;
}
void fqdn_rule_free(struct FQDN_rule* fqdn_rule)
{
assert(fqdn_rule->user_tag==NULL);
free(fqdn_rule->FQDN);
fqdn_rule->FQDN=NULL;
free(fqdn_rule);
return;
}
void _notype_fqdn_rule_free(void* p)
{
fqdn_rule_free((struct FQDN_rule*)p);
return;
}
struct xx_plugin_ex_free_wrapper
{
struct EX_data_rt* ex_data_rt;
char* row;
size_t key_offset;
size_t key_len;
void* logger;
};
void xx_plugin_ex_data_wrapper_free(void* ex_data)
{
struct xx_plugin_ex_free_wrapper* wrapper=(struct xx_plugin_ex_free_wrapper*)ex_data;
EX_data_rt_delete_by_row(wrapper->ex_data_rt, wrapper->row, wrapper->row + wrapper->key_offset, wrapper->key_len, wrapper->logger);
free(wrapper->row);
wrapper->key_offset=0;
wrapper->key_len=0;
free(wrapper);
return;
}
static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num)
{
@@ -127,7 +168,6 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
break;
case TABLE_TYPE_PLUGIN:
table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->plugin.estimate_size,
table_schema->plugin.ex_schema.key2index_func,
NULL);
if(table_schema->plugin.have_exdata)
@@ -137,7 +177,6 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
break;
case TABLE_TYPE_IP_PLUGIN:
table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->ip_plugin.estimate_size,
table_schema->ip_plugin.ex_schema.key2index_func,
free);
if(table_schema->ip_plugin.have_exdata)
@@ -146,6 +185,16 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
}
table_rt->ip_plugin.bin=Maat_garbage_bin_new(0);
break;
case TABLE_TYPE_FQDN_PLUGIN:
table_rt->fqdn_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->fqdn_plugin.ex_schema.key2index_func,
_notype_fqdn_rule_free);
if(table_schema->fqdn_plugin.have_exdata)
{
EX_data_rt_set_schema(table_rt->fqdn_plugin.ex_data_rt, &table_schema->fqdn_plugin.ex_schema);
}
table_rt->fqdn_plugin.bin=Maat_garbage_bin_new(0);
break;
default:
break;
}
@@ -196,6 +245,12 @@ static void table_runtime_free(struct Maat_table_runtime* p)
EX_data_rt_free(p->ip_plugin.ex_data_rt);
assert(p->ip_plugin.new_ip_matcher==NULL);
break;
case TABLE_TYPE_FQDN_PLUGIN:
FQDN_engine_free(p->fqdn_plugin.fqdn_engine);
Maat_garbage_bin_free(p->fqdn_plugin.bin);
EX_data_rt_free(p->fqdn_plugin.ex_data_rt);
assert(p->fqdn_plugin.new_fqdn_engine==NULL);
break;
case TABLE_TYPE_PLUGIN:
EX_data_rt_free(p->plugin.ex_data_rt);
break;
@@ -358,10 +413,156 @@ void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr
MESA_lqueue_join_tail(table_rt->similar.update_q,&digest_rule, sizeof(void*));
return;
}
void Maat_table_runtime_fqdn_add(struct Maat_table_runtime* table_rt, int expr_id, const char* fqdn, int is_suffix_match, void* tag)
{
void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
{
struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(table_schema->fqdn_plugin);
struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin);
size_t is_valid_offset=0, valid_len=0;
size_t is_suffix_flag_offset=0, is_suffix_flag_len=0;
size_t row_id_offset=0, row_id_len=0;
size_t fqdn_offset=0, fqdn_len=0;
struct FQDN_rule* fqdn_rule=NULL;
int ret=0;
struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL;
if(fqdn_plugin_schema->have_exdata)
{
ret=Maat_helper_read_column(row, fqdn_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"fqdn_plugin EX data process error: cannot find is_valid column %d of %s",
fqdn_plugin_schema->valid_flag_column, row);
return;
}
ret=Maat_helper_read_column(row, fqdn_plugin_schema->row_id_column, &row_id_offset, &row_id_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"fqdn_plugin EX data process error: cannot find row id column %d of %s",
fqdn_plugin_schema->row_id_column, row);
return;
}
ret=Maat_helper_read_column(row, fqdn_plugin_schema->is_suffix_flag_column, &is_suffix_flag_offset, &is_suffix_flag_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"fqdn_plugin EX data process error: cannot find is_suffix_match column %d of %s",
fqdn_plugin_schema->is_suffix_flag_column, row);
return;
}
ret=Maat_helper_read_column(row, fqdn_plugin_schema->fqdn_column, &fqdn_offset, &fqdn_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"fqdn_plugin EX data process error: cannot find fqdn column %d of %s",
fqdn_plugin_schema->fqdn_column, row);
return;
}
fqdn_rule=fqdn_rule_new((unsigned int)atoi(row+row_id_offset), row+fqdn_offset, fqdn_len, atoi(row+is_suffix_flag_offset));
if(atoi(row+is_valid_offset)==1)//add
{
EX_data_rt_row2EX_data(fqdn_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, fqdn_rule, logger);
}
else
{
wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1);
wrapper_for_free->row=_maat_strdup(row);
wrapper_for_free->ex_data_rt=fqdn_plugin_rt->ex_data_rt;
wrapper_for_free->key_len=row_id_len;
wrapper_for_free->key_offset=row_id_offset;
wrapper_for_free->logger=logger;
Maat_garbage_bagging(fqdn_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free);
}
}
else
{
EX_data_rt_cache_row_put(fqdn_plugin_rt->ex_data_rt, row);
}
fqdn_plugin_rt->changed_flag=1;
return;
}
int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt)
{
struct FQDN_engine* new_fqdn_engine=NULL;
struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin;
assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN);
struct EX_data_container **exc_array=NULL;
Maat_garbage_collect_routine(fqdn_rt->bin);
struct FQDN_rule* rules=NULL;
size_t rule_cnt=0, i=0;
if(!fqdn_rt->changed_flag)
{
assert(0==Maat_garbage_bin_get_size(fqdn_rt->bin));
return 0;
}
rule_cnt=EX_data_rt_list_all_ex_container(fqdn_rt->ex_data_rt, &exc_array);
rules=ALLOC(struct FQDN_rule, rule_cnt);
for(i=0; i<rule_cnt; i++)
{
rules[i]=*((struct FQDN_rule *)(EX_data_container_get_user_data(exc_array[i])));
assert(rules[i].user_tag==exc_array[i]||rules[i].user_tag==NULL);
rules[i].user_tag=exc_array[i];
}
if(rule_cnt>0)
{
new_fqdn_engine=FQDN_engine_new(rules, rule_cnt);
}
fqdn_rt->new_fqdn_engine=new_fqdn_engine;
free(rules);
free(exc_array);
return 1;
}
struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt)
{
struct FQDN_engine* old_one=table_rt->fqdn_plugin.fqdn_engine;
table_rt->fqdn_plugin.fqdn_engine=table_rt->fqdn_plugin.new_fqdn_engine;
assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN);
table_rt->fqdn_plugin.new_fqdn_engine=NULL;
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->fqdn_plugin.ex_data_rt);
return old_one;
}
int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger)
{
size_t i=0;
const char* row=NULL;
struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin);
EX_data_rt_set_schema(fqdn_plugin_rt->ex_data_rt, &table_schema->fqdn_plugin.ex_schema);
for(i=0; i<EX_data_rt_get_cached_row_num(fqdn_plugin_rt->ex_data_rt); i++)
{
row=EX_data_rt_cached_row_get(fqdn_plugin_rt->ex_data_rt, i);
Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, row, logger);
}
EX_data_rt_clear_row_cache(fqdn_plugin_rt->ex_data_rt);
Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt);
Maat_table_runtime_apply_new_fqdn_engine(table_rt);//returned NULL.
return 0;
}
int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size)
{
struct FQDN_match results[size];
int n_result=0, i=0;
if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN)
{
return -1;
}
if(!table_rt->fqdn_plugin.fqdn_engine)
{
return 0;
}
n_result=FQDN_engine_search(table_rt->fqdn_plugin.fqdn_engine, query_fqdn, strlen(query_fqdn), results, size);
for(i=0; i<n_result; i++)
{
ex_data_array[i]=EX_data_rt_get_EX_data_by_container(table_rt->fqdn_plugin.ex_data_rt, (struct EX_data_container *)results[i].user_tag);
}
return n_result;
}
int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt)
{
long i=0,data_size=0;
@@ -463,41 +664,28 @@ int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime*
ip_plugin->changed_flag=0;
return 1;
}
struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt)
{
struct ip_matcher* old_one=table_rt->ip_plugin.ip_matcher;
table_rt->ip_plugin.ip_matcher=table_rt->ip_plugin.new_ip_matcher;
assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN);
table_rt->ip_plugin.new_ip_matcher=NULL;
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->ip_plugin.ex_data_rt);
return old_one;
}
struct ip_plugin_ex_free_wrapper
{
struct EX_data_rt* ex_data_rt;
char* row;
size_t key_offset;
size_t key_len;
void* logger;
};
void ip_plugin_ex_data_wrapper_free(void* ex_data)
{
struct ip_plugin_ex_free_wrapper* wrapper=(struct ip_plugin_ex_free_wrapper*)ex_data;
EX_data_rt_delete_by_row(wrapper->ex_data_rt, wrapper->row, wrapper->row + wrapper->key_offset, wrapper->key_len, wrapper->logger);
free(wrapper->row);
wrapper->key_offset=0;
wrapper->key_len=0;
free(wrapper);
return;
}
void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
{
struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin);
struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin);
size_t is_valid_offset=0, valid_len=0;
size_t key_offset=0, key_len=0;
size_t row_id_offset=0, row_id_len=0;
struct ip_rule* ip_rule=NULL;
int ret=0;
struct ip_plugin_ex_free_wrapper* wrapper_for_free=NULL;
struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL;
if(ip_plugin_schema->have_exdata)
{
ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
@@ -508,7 +696,7 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s
ip_plugin_schema->row_id_column, row);
return;
}
ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &key_offset, &key_len);
ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &row_id_offset, &row_id_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
@@ -527,17 +715,18 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s
if(atoi(row+is_valid_offset)==1)//add
{
EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+key_offset, key_len, ip_rule, logger);
EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, ip_rule, logger);
}
else
{
wrapper_for_free=ALLOC(struct ip_plugin_ex_free_wrapper, 1);
wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1);
wrapper_for_free->row=_maat_strdup(row);
wrapper_for_free->ex_data_rt=ip_plugin_rt->ex_data_rt;
wrapper_for_free->key_len=key_len;
wrapper_for_free->key_offset=key_offset;
wrapper_for_free->key_len=row_id_len;
wrapper_for_free->key_offset=row_id_offset;
wrapper_for_free->logger=logger;
Maat_garbage_bagging(ip_plugin_rt->bin, wrapper_for_free, ip_plugin_ex_data_wrapper_free);
Maat_garbage_bagging(ip_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free);
free(ip_rule);
}
}
else

3
src/inc_internal/Maat_ex_data.h
View File

@@ -3,7 +3,7 @@
struct EX_data_rt;
struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data));
struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data));
void EX_data_rt_free(struct EX_data_rt* p);
void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema);
void EX_data_rt_cache_row_put(struct EX_data_rt* p, const char* row);
@@ -23,4 +23,5 @@ MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const
MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container);
size_t EX_data_rt_list_all_ex_container(struct EX_data_rt* ex_rt, struct EX_data_container*** ex_container_array);
void* EX_data_container_get_user_data(struct EX_data_container* ex_container);
size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt);

2
src/inc_internal/Maat_rule_internal.h
View File

@@ -214,7 +214,7 @@ struct Maat_scanner
mcore_long_t ref_cnt;
rule_scanner_t region;
size_t gie_update_q_size;
size_t ip_plugin_update_q_size;
size_t xx_plugin_update_q_size;
size_t to_update_group_cnt;
size_t to_update_compile_cnt;

23
src/inc_internal/Maat_table.h
View File

@@ -37,7 +37,6 @@ enum MAAT_TABLE_TYPE
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN,
TABLE_TYPE_INTERVAL_PLUS,
TABLE_TYPE_FQDN,
TABLE_TYPE_FQDN_PLUGIN
};
@@ -102,7 +101,6 @@ struct plugin_table_schema
int foreign_columns[MAX_FOREIGN_CLMN_NUM];
int cb_plug_cnt;
int have_exdata;
long long estimate_size;
struct plugin_table_callback_schema cb_plug[MAX_PLUGIN_PER_TABLE];
struct EX_data_schema ex_schema;
};
@@ -114,7 +112,17 @@ struct ip_plugin_table_schema
int end_ip_column;
int valid_flag_column;
int rule_tag_column;
long long estimate_size;
int have_exdata;
struct EX_data_schema ex_schema;
};
struct fqdn_plugin_table_schema
{
int row_id_column;
int is_suffix_flag_column;
int fqdn_column;
int valid_flag_column;
int rule_tag_column;
int have_exdata;
struct EX_data_schema ex_schema;
};
@@ -131,6 +139,7 @@ struct Maat_table_schema
struct expr_table_schema expr;
struct plugin_table_schema plugin;
struct ip_plugin_table_schema ip_plugin;
struct fqdn_plugin_table_schema fqdn_plugin;
struct virtual_table_schema virtual_table;
struct composition_table_schema composition;
void* others;//group, ip, interval and digest don't have any special schema.
@@ -183,6 +192,13 @@ int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_sche
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger);
int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger);
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type);
void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr);
@@ -198,4 +214,5 @@ enum MAAT_TABLE_CHILD_TYPE
CHILD_TABLE_TYPE_SESSION
};
int Maat_table_get_child_id(struct Maat_table_schema* p_table, enum MAAT_TABLE_CHILD_TYPE type);
int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table);

14
src/inc_internal/Maat_table_runtime.h
View File

@@ -15,10 +15,10 @@ struct similar_runtime
GIE_handle_t* gie_handle;
MESA_lqueue_head update_q;
};
struct fqdn_runtime
struct fqdn_plugin_runtime
{
struct FQDN_engine* fqdn_engine;
struct FQDN_engine* old_fqdn_engine;
struct FQDN_engine* new_fqdn_engine;
struct EX_data_rt* ex_data_rt; //for fqdn_plugin ONLY
struct Maat_garbage_bin* bin;
int changed_flag;
@@ -59,7 +59,7 @@ struct Maat_table_runtime
union
{
struct similar_runtime similar; //for digest and similarity
struct fqdn_runtime fqdn;//for fqdn and fqdn_plugin
struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin
struct plugin_runtime plugin;
struct ip_plugin_runtime ip_plugin;
struct expr_runtime expr;
@@ -95,4 +95,12 @@ int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_
int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* table_rt);
struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt);
struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);
int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size);

42
test/maat_json.json
View File

@@ -1856,45 +1856,55 @@
{
"table_name": "QD_ENTRY_INFO",
"table_content": [
"1\t192.168.0.1\t101\t1",
"2\t192.168.0.2\t101\t1",
"3\t192.168.1.1\t102\t1"
"1\t192.168.0.1\t101\t1",
"2\t192.168.0.2\t101\t1",
"3\t192.168.1.1\t102\t1"
]
},
{
"table_name": "TEST_PLUGIN_TABLE",
"table_content": [
"1\t3388\t99\t1",
"2\t3355\t66\t1",
"3\tcccc\t11\t1"
"1\t3388\t99\t1",
"2\t3355\t66\t1",
"3\tcccc\t11\t1"
]
},
{
"table_name": "TEST_EFFECTIVE_RANGE_TABLE",
"table_content": [
"1\tSUCCESS\t99\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"移动\"]}]]}\t1111",
"2\tSUCCESS\t66\t1\t0\t222",
"3\tFAILED\t11\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\",\"上海/浦东/陆家嘴\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"联通\"]}],[{\"tag\":\"location\",\"value\":[\"北京\"]},{\"tag\":\"isp\",\"value\":[\"联通\"]}]]}\t333",
"4\tSUCCESS\t66\t1\t{}\t444",
"5\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京\"]}]]}\t444",
"6\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"weather\",\"value\":[\"hot\"]}]]}\t444"
"1\tSUCCESS\t99\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"移动\"]}]]}\t1111",
"2\tSUCCESS\t66\t1\t0\t222",
"3\tFAILED\t11\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\",\"上海/浦东/陆家嘴\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"联通\"]}],[{\"tag\":\"location\",\"value\":[\"北京\"]},{\"tag\":\"isp\",\"value\":[\"联通\"]}]]}\t333",
"4\tSUCCESS\t66\t1\t{}\t444",
"5\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京\"]}]]}\t444",
"6\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"weather\",\"value\":[\"hot\"]}]]}\t444"
]
},
{
"table_name": "IR_INTERCEPT_IP",
"table_content": [
"1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34",
"161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42"
"1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34",
"161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42"
]
},
{
"table_name": "TEST_IP_PLUGIN_WITH_EXDATA",
"table_content": [
"101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1",
"101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1",
"102\t4\t192.168.30.90\t192.168.30.128\tBigger-range-should-in-the-back\t1",
"103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1",
"103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1",
"104\t6\t2001:db8:1234::1\t2001:db8:1234::5210\tSomething-like-json\t1"
]
},
{
"table_name": "TEST_FQDN_PLUGIN_WITH_EXDATA",
"table_content": [
"201\t0\twww.example1.com\tcatid=1\t1",
"202\t1\t.example1.com\tcatid=1\t1",
"203\t0\tnews.example1.com\tcatid=2\t1",
"204\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1",
"205\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1"
]
}
]
}

1
test/table_info.conf
View File

@@ -56,3 +56,4 @@
33 GeoLocation expr UTF8 UTF8 yes 0
34 SOURCE_IP_GEO virtual GeoLocation --
35 INTERGER_PLUS intval_plus --
36 TEST_FQDN_PLUGIN_WITH_EXDATA fqdn_plugin {"row_id":1,"is_suffix_match":2,"fqdn":3,"valid":5} --

155
test/test_maatframe.cpp
View File

@@ -310,6 +310,86 @@ TEST(IP_Plugin_Table, EX_DATA)
}
#define FQDN_PLUGIN_EX_DATA
struct fqdn_plugin_ud
{
int rule_id;
int catid;
int ref_cnt;
};
void fqdn_plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{
int *counter=(int *)argp, ret=0;
size_t column_offset=0, column_len=0;
struct fqdn_plugin_ud* ud=(struct fqdn_plugin_ud*)calloc(sizeof(struct fqdn_plugin_ud), 1);
ret=Maat_helper_read_column(table_line, 1, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->rule_id=atoi(table_line+column_offset);
ret=Maat_helper_read_column(table_line, 4, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
sscanf(table_line+column_offset, "catid=%d",&ud->catid);
ud->ref_cnt=1;
*ad=ud;
(*counter)++;
return;
}
void fqdn_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{
struct fqdn_plugin_ud* u=(struct fqdn_plugin_ud*)(*ad);
u->ref_cnt--;
if(u->ref_cnt>0) return;
free(u);
*ad=NULL;
}
void fqdn_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
{
struct fqdn_plugin_ud* u=(struct fqdn_plugin_ud*)(*from);
u->ref_cnt++;
*to=u;
}
TEST(FQDN_Plugin_Table, EX_DATA)
{
int fqdn_plugin_ex_data_counter=0, i=0;
const char* table_name="TEST_FQDN_PLUGIN_WITH_EXDATA";
int table_id=0, ret=0;
table_id=Maat_table_register(g_feather, table_name);
ASSERT_GT(table_id, 0);
ret=Maat_fqdn_plugin_EX_register(g_feather, table_id,
fqdn_plugin_EX_new_cb,
fqdn_plugin_EX_free_cb,
fqdn_plugin_EX_dup_cb,
0, &fqdn_plugin_ex_data_counter);
ASSERT_TRUE(ret>=0);
EXPECT_EQ(fqdn_plugin_ex_data_counter, 5);
struct fqdn_plugin_ud* result[4];
ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "www.example1.com", (void**)result, 4);
ASSERT_EQ(ret, 2);
EXPECT_EQ(result[0]->rule_id, 201);
EXPECT_EQ(result[1]->rule_id, 202);
for(i=0; i<ret; i++)
{
fqdn_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
}
ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "www.example3.com", (void**)result, 4);
EXPECT_EQ(ret, 0);
ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4);
ASSERT_EQ(ret, 2);
EXPECT_EQ(result[0]->rule_id, 204);
EXPECT_EQ(result[1]->rule_id, 205);
for(i=0; i<ret; i++)
{
fqdn_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
}
}
TEST(StringScan, Full)
{
int ret=0;
@@ -3110,9 +3190,9 @@ TEST_F(MaatCmdTest, PluginEXData)
return;
}
TEST_F(MaatCmdTest, IPPluginEXData)
TEST_F(MaatCmdTest, UpdateIPPlugin)
{
#define ip_plugin_EX_data
#define IP_Plugin_EX_data
Maat_feather_t feather=MaatCmdTest::_shared_feather;
int ret=0, i=0;
@@ -3178,7 +3258,7 @@ TEST_F(MaatCmdTest, IPPluginEXData)
ip_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
}
ret=Maat_cmd_set_lines(feather, p_line , TEST_CMD_LINE_NUM, MAAT_OP_DEL);
ret=Maat_cmd_set_lines(feather, p_line, TEST_CMD_LINE_NUM, MAAT_OP_DEL);
EXPECT_GT(ret, 0);
usleep(WAIT_FOR_EFFECTIVE_US);
@@ -3188,6 +3268,75 @@ TEST_F(MaatCmdTest, IPPluginEXData)
return;
}
TEST_F(MaatCmdTest, UpdateFQDNPlugin)
{
#define FQDN_Plugin_EX_data
Maat_feather_t feather=MaatCmdTest::_shared_feather;
int ret=0, i=0;
int table_id=0, fqdn_plugin_ex_data_counter=0;
const char* table_name="TEST_FQDN_PLUGIN_WITH_EXDATA";
const int TEST_CMD_LINE_NUM=5;
const struct Maat_cmd_line *p_line[TEST_CMD_LINE_NUM];
struct Maat_cmd_line line_rule[TEST_CMD_LINE_NUM];
const char* table_line[TEST_CMD_LINE_NUM]={
"201\t0\twww.example1.com\tcatid=1\t1",
"202\t1\t.example1.com\tcatid=1\t1",
"203\t0\tnews.example1.com\tcatid=2\t1",
"204\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1",
"205\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1"
};
table_id=Maat_table_register(feather, table_name);
ASSERT_GT(table_id, 0);
memset(&line_rule,0,sizeof(line_rule));
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
line_rule[i].label_id=0;
line_rule[i].rule_id=(int)Maat_cmd_incrby(feather,"TEST_PLUG_SEQ", 1);
line_rule[i].table_name=table_name;
line_rule[i].table_line=table_line[i];
line_rule[i].expire_after=0;
p_line[i]=line_rule+i;
}
ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
EXPECT_GT(ret, 0);
usleep(WAIT_FOR_EFFECTIVE_US);
ret=Maat_fqdn_plugin_EX_register(feather, table_id,
fqdn_plugin_EX_new_cb,
fqdn_plugin_EX_free_cb,
fqdn_plugin_EX_dup_cb,
0, &fqdn_plugin_ex_data_counter);
ASSERT_TRUE(ret>=0);
EXPECT_EQ(fqdn_plugin_ex_data_counter, 5);
struct fqdn_plugin_ud* result[4];
ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4);
ASSERT_EQ(ret, 2);
for(i=0; i<ret; i++)
{
fqdn_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
}
ret=Maat_cmd_set_lines(feather, p_line+3, TEST_CMD_LINE_NUM-3, MAAT_OP_DEL);
EXPECT_GT(ret, 0);
usleep(WAIT_FOR_EFFECTIVE_US*5);
ret=Maat_fqdn_plugin_get_EX_data(feather, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4);
ASSERT_EQ(ret, 0);
return;
}
#define TEST_HIT_PATH
TEST_F(MaatCmdTest, HitPath)
{