diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index f825658..fcdeee4 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -319,6 +319,14 @@ struct ip_address int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data); +int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + long argl, void *argp); + +int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data); + enum MAAT_RULE_OPT { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7be2077..db37b65 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) -set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp) +set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp entry/FQDN_engine.cpp) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) include_directories(/opt/MESA/include/MESA/) diff --git a/src/entry/FQDN_engine.cpp b/src/entry/FQDN_engine.cpp index 7546850..1174397 100644 --- a/src/entry/FQDN_engine.cpp +++ b/src/entry/FQDN_engine.cpp @@ -7,7 +7,7 @@ * All rights reserved * * Written by: LIU YANBING (liuyanbing@iie.ac.cn) - * Last modification: 2020-09-01 + * Last modification: 2020-09-19 * * This code is the exclusive and proprietary property of IIE-CAS and NELIST. * Usage for direct or indirect commercial advantage is not allowed without @@ -23,8 +23,10 @@ #include /*************************************************************************************/ -#include -#define popcnt_u64 _mm_popcnt_u64 +//#include +//#define popcnt_u64 _mm_popcnt_u64 +//Use gcc builtin function to replace SSE4.2 instruction for portability +#define popcnt_u64 __builtin_popcountl #define FOR(i, n) for(int i=0, _n=(int)(n); i<_n; i++) @@ -241,7 +243,7 @@ int CHashTrieFQDN::initialize(const struct FQDN_rule * rules, size_t n_rule) m_matched[idx]=&(m_domains[k]); } - printf("mem_bytes=%u(MB)\n", mem_bytes/(1U<<20)); +// printf("mem_bytes=%ll(MB)\n", mem_bytes/(1U<<20)); return 1; } @@ -260,7 +262,7 @@ int CHashTrieFQDN::search(const char * FQDN, size_t FQDN_len, struct FQDN_match { if(m_num==0 || FQDN_len==0 || FQDN==NULL || n_result==0) return -1; - int match_num=0; + size_t match_num=0; const unsigned char * pb=(unsigned char *)FQDN; unsigned long long HASH[16]; /*假设域名级数不超过16*/ unsigned int P[16]; diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index c8c5f8b..0debece 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -1125,6 +1125,12 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id struct Maat_table_schema *table_desc=NULL; struct Maat_table_runtime *table_rt=NULL; MAAT_RULE_EX_DATA exdata=NULL; + struct timespec start,end; + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&start); + } + if(_feather->scanner==NULL) { return NULL; @@ -1132,6 +1138,17 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key); + + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&end); + maat_stat_table(table_rt, 0, &start, &end, 0); + } + else + { + maat_stat_table(table_rt, 0, NULL, NULL, 0); + } + return exdata; } int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id, @@ -1163,6 +1180,35 @@ int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id, return 0; } +int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + long argl, void *argp) +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + int ret=-1; + struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id); + pthread_mutex_lock(&(_feather->background_update_mutex)); + ret=Maat_table_fqdn_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger); + + if(ret<0) + { + pthread_mutex_unlock(&(_feather->background_update_mutex)); + return -1; + } + + struct Maat_table_runtime* table_rt=NULL; + if(_feather->scanner!=NULL) + { + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + Maat_table_runtime_fqdn_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger); + } + pthread_mutex_unlock(&(_feather->background_update_mutex)); + + return 0; + +} int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data) { @@ -1170,11 +1216,18 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc struct Maat_table_schema *table_schema=NULL; struct Maat_table_runtime *table_rt=NULL; int n_get=0; + + struct timespec start,end; + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&start); + } if(_feather->scanner==NULL) { return 0; } - table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + + table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_IP_PLUGIN, NULL); table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); if(table_rt->table_type!=TABLE_TYPE_IP_PLUGIN) { @@ -1190,9 +1243,57 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc ipv6_ntoh(ip_data.ipv6); } n_get=Maat_table_runtime_ip_plugin_get_N_ex_data(table_rt, table_schema, &ip_data, ex_data_array, n_ex_data); + + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&end); + maat_stat_table(table_rt, 0, &start, &end, 0); + } + else + { + maat_stat_table(table_rt, 0, NULL, NULL, 0); + } + return n_get; } +int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data) +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct Maat_table_schema *table_schema=NULL; + struct Maat_table_runtime *table_rt=NULL; + int n_get=0; + + struct timespec start,end; + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&start); + } + if(_feather->scanner==NULL) + { + return 0; + } + table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_FQDN_PLUGIN, NULL); + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN) + { + return -1; + } + n_get=Maat_table_runtime_fqdn_plugin_get_N_ex_data(table_rt, table_schema, fqdn, ex_data_array, n_ex_data); + + + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&end); + maat_stat_table(table_rt, 0, &start, &end, 0); + } + else + { + maat_stat_table(table_rt, 0, NULL, NULL, 0); + } + + return n_get; +} int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id ,enum MAAT_CHARSET charset,const char* data,int data_len diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index fa48b80..21655e6 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -201,6 +201,7 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_ break; case TABLE_TYPE_PLUGIN: case TABLE_TYPE_IP_PLUGIN: + case TABLE_TYPE_FQDN_PLUGIN: if(valid_column_seq<0) { return -1; @@ -1624,7 +1625,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m int update_type=CM_UPDATE_TYPE_INC; long long new_version=0; enum MAAT_TABLE_TYPE table_type; - struct Maat_table_schema* table_desc=NULL; + struct Maat_table_schema* table_schema=NULL; void* logger=feather->logger; if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write @@ -1713,26 +1714,12 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m table_type=Maat_table_get_type_by_id(feather->table_mgr, table_id); if(rule_list[i].op==MAAT_OP_DEL) { - if(table_type==TABLE_TYPE_PLUGIN||table_type==TABLE_TYPE_IP_PLUGIN) - { - table_desc=Maat_table_get_scan_by_id(feather->table_mgr, table_id, table_type, NULL); - if(table_type==TABLE_TYPE_PLUGIN) - { - valid_column=table_desc->plugin.valid_flag_column; - } - else - { - valid_column=table_desc->ip_plugin.valid_flag_column; - } - } - else - { - valid_column=-1; - } + table_schema=Maat_table_get_scan_by_id(feather->table_mgr, table_id, table_type, NULL); + valid_column=Maat_table_xx_plugin_table_get_valid_flag_column(table_schema); ret=invalidate_line(rule_list[i].table_line, table_type, valid_column); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Invaid format %s ." + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Invalidate line failed, invaid format %s ." ,rule_list[i].table_line); continue; } @@ -1918,22 +1905,17 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_ goto error_out; } int valid_flag_column=0; - switch(p_table->table_type) + + valid_flag_column=Maat_table_xx_plugin_table_get_valid_flag_column(p_table); + if(valid_flag_column<0) { - case TABLE_TYPE_PLUGIN: - valid_flag_column=p_table->plugin.valid_flag_column; - plugin_desc=&(p_table->plugin); - break; - case TABLE_TYPE_IP_PLUGIN: - valid_flag_column=p_table->ip_plugin.valid_flag_column; - break; - default: - MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command - ,"Command set line id %d failed: table %s is not a plugin or ip_plugin table." - , line_rule[i]->rule_id - , line_rule[i]->table_name); - ret=-1; - goto error_out; + MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command + ,"Command set line id %d failed: table %s is not a plugin or ip_plugin table." + , line_rule[i]->rule_id + , line_rule[i]->table_name); + ret=-1; + goto error_out; + } if(op==MAAT_OP_ADD) diff --git a/src/entry/Maat_ex_data.cpp b/src/entry/Maat_ex_data.cpp index 7cf2a77..a98645b 100644 --- a/src/entry/Maat_ex_data.cpp +++ b/src/entry/Maat_ex_data.cpp @@ -57,7 +57,7 @@ void cache_row_free(void*p) } UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free}; -struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)) +struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)) { struct EX_data_rt* p=ALLOC(struct EX_data_rt, 1); p->hash_key2ex=NULL; @@ -207,3 +207,7 @@ void* EX_data_container_get_user_data(struct EX_data_container* ex_container) { return ex_container->user_data; } +size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt) +{ + return HASH_COUNT(ex_rt->hash_key2ex); +} diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 3630135..e099b03 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -34,7 +34,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_3_0_20200917=1; +int MAAT_FRAME_VERSION_3_0_20200919=1; int is_valid_table_name(const char* str) { @@ -1392,10 +1392,7 @@ int add_digest_rule(struct Maat_table_schema* table, struct db_digest_rule* db_r scanner->gie_update_q_size++; return 0; } -int add_fqdn_rule(struct Maat_table_schema* table, struct db_digest_rule* db_rule, struct Maat_scanner *scanner,void* logger) -{ -} int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id, int rule_type, struct Maat_scanner *maat_scanner, void* logger) { @@ -1435,10 +1432,6 @@ int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, table->table_id); Maat_table_runtime_digest_del(table_rt, region->expr_id_lb); maat_scanner->gie_update_q_size++; - break; - case TABLE_TYPE_FQDN: - case TABLE_TYPE_FQDN_PLUGIN: - break; default: assert(0); @@ -2172,66 +2165,6 @@ error_out: digest_rule=NULL; return; } -void update_fqdn_rule(struct Maat_table_schema* table, const char* table_line, struct Maat_scanner *scanner, void* logger) -{ - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); - struct db_fqdn_rule* fqdn_rule=ALLOC(struct db_fqdn_rule, 1); - int ret=0; - char fqdn_buff[MAX_TABLE_LINE_SIZE]={'\0'}; - - ret=sscanf(table_line,"%d\t%d\t%llu\t%s\t%hd\t%d", &(fqdn_rule->region_id), - &(fqdn_rule->group_id), - &(fqdn_rule->is_suffix_match), - fqdn_buff, - &(fqdn_rule->is_valid)); - - fqdn_rule->fqdn=fqdn_buff; - if(ret!=5) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error, invalid format of fqdn table %s:%s" - ,table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - goto error_out; - } - - if(fqdn_rule->is_valid==FALSE) - { - //digest rule is not build with rulescan, this rule type is useless in count_rs_region funciton. - ret=del_region_rule(table, fqdn_rule->region_id, fqdn_rule->group_id, 0, scanner, logger); - if(ret<0) - { - table->udpate_err_cnt++; - } - else - { - table_rt->origin_rule_num--; - } - - } - else - { - ret=add_fqdn_rule(table, digest_rule,scanner,logger); - if(ret<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "duplicate config of intval table %s config_id=%d" - ,table->table_name[table->updating_name],digest_rule->region_id); - table->udpate_err_cnt++; - } - else - { - table_rt->origin_rule_num++; - } - - } -error_out: - fqdn_rule->fqdn=NULL; - free(fqdn_rule); - fqdn_rule=NULL; - return; -} - void update_plugin_table(struct Maat_table_schema* table_schema, const char* row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) { int ret=1, matched_tag=1; @@ -2278,21 +2211,34 @@ void update_plugin_table(struct Maat_table_schema* table_schema, const char* row } Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger); } -void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) +void update_xx_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) { int ret=1, matched_tag=1; - struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin); struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_schema->table_id); char* copy=NULL; size_t accept_tag_offset=0, accept_tag_len=0; - if(ip_plugin_schema->rule_tag_column>0&&n_tags>0) + int rule_tag_column=-1; + if(table_schema->table_type==TABLE_TYPE_IP_PLUGIN) { - ret=Maat_helper_read_column(table_row, ip_plugin_schema->rule_tag_column, &accept_tag_offset, &accept_tag_len); + rule_tag_column=table_schema->ip_plugin.rule_tag_column; + } + else if(table_schema->table_type==TABLE_TYPE_FQDN_PLUGIN) + { + rule_tag_column=table_schema->fqdn_plugin.rule_tag_column; + } + else + { + assert(0); + return; + } + if(rule_tag_column>0&&n_tags>0) + { + ret=Maat_helper_read_column(table_row, rule_tag_column, &accept_tag_offset, &accept_tag_len); if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error, could not locate tag in column %d of plugin table_schema %s:%s", - ip_plugin_schema->rule_tag_column, + "update error, could not locate tag in column %d of table %s:%s", + rule_tag_column, table_schema->table_name[table_schema->updating_name], table_row); table_schema->udpate_err_cnt++; @@ -2322,8 +2268,15 @@ void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char* return; } } - Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger); - scanner->ip_plugin_update_q_size++; + if(table_schema->table_type==TABLE_TYPE_IP_PLUGIN) + { + Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger); + } + else //TABLE_TYPE_FQDN_PLUGIN + { + Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger); + } + scanner->xx_plugin_update_q_size++; return; } @@ -2331,9 +2284,9 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* { MESA_htable_handle tmp_map=NULL; struct Maat_table_runtime* table_rt=NULL; - struct ip_matcher* old_ip_matcher=NULL; int i=0, ret=0; - + struct ip_matcher* old_ip_matcher=NULL; + struct FQDN_engine* old_fqdn_engine=NULL; if(scanner->to_update_compile_cnt+scanner->to_update_group_cnt>0) { ret=Maat_hierarchy_rebuild(scanner->hier); @@ -2367,8 +2320,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* switch(table_rt->table_type) { case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - + case TABLE_TYPE_SIMILARITY: ret=Maat_table_runtime_digest_batch_udpate(table_rt); if(ret<0) { @@ -2376,7 +2328,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* "GIE_update error."); } break; - case TABLE_TYPE_IP_PLUGIN: + case TABLE_TYPE_IP_PLUGIN: ret=Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt); if(ret) { @@ -2387,6 +2339,18 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* } } break; + case TABLE_TYPE_FQDN_PLUGIN: + + ret=Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt); + if(ret) + { + old_fqdn_engine=Maat_table_runtime_apply_new_fqdn_engine(table_rt); + if(old_fqdn_engine) + { + Maat_garbage_bagging(scanner->ref_garbage_bin, old_fqdn_engine, (void (*)(void*))FQDN_engine_free); + } + } + break; default: break; } @@ -2403,7 +2367,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* scanner->gie_update_q_size=0; scanner->to_update_group_cnt=0; scanner->to_update_compile_cnt=0; - scanner->ip_plugin_update_q_size=0; + scanner->xx_plugin_update_q_size=0; return; } @@ -2468,7 +2432,7 @@ void maat_finish_cb(void* u_para) feather->scanner->cfg_num=scanner_rule_num(feather->scanner); feather->scanner->version=feather->maat_version; expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); - feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->ip_plugin_update_q_size; + feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->xx_plugin_update_q_size; if(time(NULL)-feather->scanner->last_update_time>=feather->effect_interval_ms/1000) { do_scanner_update(feather->scanner, @@ -2548,8 +2512,9 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) update_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); break; case TABLE_TYPE_IP_PLUGIN: - update_ip_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); - break; + case TABLE_TYPE_FQDN_PLUGIN: + update_xx_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); + break; default: break; diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table.cpp index 3efc780..098bdbf 100644 --- a/src/entry/Maat_table.cpp +++ b/src/entry/Maat_table.cpp @@ -119,6 +119,7 @@ int _read_integer_arrary(char* string, int *array, int size) } #define COLUMN_PLUGIN_SCHEMA_JSON 4 #define COLUMN_IP_PLUGIN_SCHEMA_JSON 4 +#define COLUMN_FQDN_PLUGIN_SHCEMA_JSON 4 #define COLUMN_COMPOSITION_SCHEMA_JSON 4 int read_plugin_table_schema(const char* line, struct Maat_table_schema* p) @@ -173,12 +174,6 @@ int read_plugin_table_schema(const char* line, struct Maat_table_schema* p) assert(tmp->type==cJSON_Number); plugin_desc->rule_tag_column=tmp->valueint; } - tmp=cJSON_GetObjectItem(json, "estimate_size"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - plugin_desc->estimate_size=tmp->valueint; - } tmp=cJSON_GetObjectItem(json, "foreign"); if(tmp!=NULL) { @@ -271,19 +266,87 @@ int read_ip_plugin_table_schema(const char* line, struct Maat_table_schema* p) //read_cnt++; Tag is optional, so NOT ++ intentionally. } - ip_plugin_schema->estimate_size=4096; - tmp=cJSON_GetObjectItem(json, "estimate_size"); + cJSON_Delete(json); + + free(copy_line); + if(read_cnt<5) + { + return -1; + } + else + { + return 0; + } +error_out: + free(copy_line); + return -1; + +} +int read_fqdn_plugin_table_schema(const char* line, struct Maat_table_schema* p) +{ + int ret=0, read_cnt=0; + size_t offset=0, len=0; + cJSON* json=NULL, *tmp=NULL; + char* copy_line=NULL, *fqnd_plugin_schema_json=NULL; + struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(p->fqdn_plugin); + + copy_line=_maat_strdup(line); + ret=get_column_pos(copy_line, COLUMN_FQDN_PLUGIN_SHCEMA_JSON, &offset, &len); + if(ret<0) + { + goto error_out; + } + if(offset+lentype==cJSON_Number) + { + fqdn_plugin_schema->row_id_column=tmp->valueint; + read_cnt++; + } + + tmp=cJSON_GetObjectItem(json, "is_suffix_match"); + if(tmp!=NULL && tmp->type==cJSON_Number) + { + fqdn_plugin_schema->is_suffix_flag_column=tmp->valueint; + read_cnt++; + } + tmp=cJSON_GetObjectItem(json, "fqdn"); + if(tmp!=NULL && tmp->type==cJSON_Number) + { + fqdn_plugin_schema->fqdn_column=tmp->valueint; + read_cnt++; + } + + tmp=cJSON_GetObjectItem(json, "valid"); if(tmp!=NULL) { assert(tmp->type==cJSON_Number); - ip_plugin_schema->estimate_size=tmp->valueint; - //read_cnt++; estimate_size is optional, so NOT ++ intentionally. + fqdn_plugin_schema->valid_flag_column=tmp->valueint; + read_cnt++; + } + fqdn_plugin_schema->rule_tag_column=-1; + tmp=cJSON_GetObjectItem(json, "tag"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + fqdn_plugin_schema->rule_tag_column=tmp->valueint; + //read_cnt++; Tag is optional, so NOT ++ intentionally. } cJSON_Delete(json); free(copy_line); - if(read_cnt<5) + if(read_cnt<4) { return -1; } @@ -491,7 +554,6 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path map_register(string2int_map,"compile", TABLE_TYPE_COMPILE); map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN); map_register(string2int_map,"ip_plugin", TABLE_TYPE_IP_PLUGIN); - map_register(string2int_map,"fqdn", TABLE_TYPE_FQDN); map_register(string2int_map,"fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN); map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL); map_register(string2int_map,"interval", TABLE_TYPE_INTERVAL); @@ -572,9 +634,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path ret=read_plugin_table_schema(line, p); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n", table_info_path,i); + fprintf(stderr,"Maat read table info %s line %d error:illegal plugin table schema.\n", table_info_path,i); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal plugin info.", table_info_path,i); + "Maat read table info %s line %d error:illegal plugin table schema.", table_info_path,i); goto invalid_table; } break; @@ -582,19 +644,29 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path ret=read_ip_plugin_table_schema(line, p); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal ip_plugin info.\n", table_info_path,i); + fprintf(stderr,"Maat read table info %s line %d error:illegal ip_plugin table schema.\n", table_info_path,i); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal ip_plugin info.", table_info_path,i); + "Maat read table info %s line %d error:illegal ip_plugin table schema.", table_info_path,i); goto invalid_table; } - break; + break; + case TABLE_TYPE_FQDN_PLUGIN: + ret=read_fqdn_plugin_table_schema(line, p); + if(ret<0) + { + fprintf(stderr,"Maat read table info %s line %d error:illegal fqdn_plugin table schema.\n", table_info_path,i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:illegal fqdn_plugin table schema.", table_info_path,i); + goto invalid_table; + } + break; case TABLE_TYPE_COMPOSITION: ret=read_composition_table_schema(line, p, string2int_map); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal composition info.\n", table_info_path,i); + fprintf(stderr,"Maat read table info %s line %d error:illegal composition table schema.\n", table_info_path,i); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal composition info.", table_info_path,i); + "Maat read table info %s line %d error:illegal composition table schema.", table_info_path,i); goto invalid_table; } break; @@ -602,9 +674,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path ret=read_virtual_table_schema(line, p, string2int_map); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal virtual info.\n", table_info_path,i); + fprintf(stderr,"Maat read table info %s line %d error:illegal virtual table schema.\n", table_info_path,i); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal virtual info.", table_info_path,i); + "Maat read table info %s line %d error:illegal virtual table schema.", table_info_path,i); goto invalid_table; } break; @@ -618,9 +690,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path { p->compile.user_region_encoding=USER_REGION_ENCODE_NONE; } - break - case TABLE_TYPE_FQDN: - + break; default: break; } @@ -975,7 +1045,40 @@ int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_sche new_func, free_func, dup_func, key2index_func, argl, argp); table_schema->ip_plugin.have_exdata=1; return 0; -} +} +int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp, + void* logger) +{ + if(new_func==NULL || free_func==NULL || dup_func==NULL ) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); + return -1; + } + if(table_schema->table_type!=TABLE_TYPE_FQDN_PLUGIN) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a fqdn_plugin table.", __FUNCTION__); + return -1; + } + if(table_schema->fqdn_plugin.have_exdata) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__); + return -1; + } + Maat_table_EX_data_schema_set(&table_schema->fqdn_plugin.ex_schema, + new_func, free_func, dup_func, key2index_func, argl, argp); + table_schema->fqdn_plugin.have_exdata=1; + return 0; +} + + void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type) { table_mgr->active_plugin_table_num=0; @@ -1096,4 +1199,23 @@ int Maat_table_get_child_id(struct Maat_table_schema* p_table, enum MAAT_TABLE_C } return ret; } - +int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table) +{ + int valid_flag_column=-1; + switch(p_table->table_type) + { + case TABLE_TYPE_PLUGIN: + valid_flag_column=p_table->plugin.valid_flag_column; + break; + case TABLE_TYPE_IP_PLUGIN: + valid_flag_column=p_table->ip_plugin.valid_flag_column; + break; + case TABLE_TYPE_FQDN_PLUGIN: + valid_flag_column=p_table->fqdn_plugin.valid_flag_column; + break; + default: + valid_flag_column=-1; + break; + } + return valid_flag_column; +} diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp index 48a5388..5130eab 100644 --- a/src/entry/Maat_table_runtime.cpp +++ b/src/entry/Maat_table_runtime.cpp @@ -113,6 +113,47 @@ static void destroy_digest_rule(GIE_digest_t*rule) rule=NULL; return; } +struct FQDN_rule* fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_len, int is_suffix_match) +{ + struct FQDN_rule* fqdn_rule=ALLOC(struct FQDN_rule, 1); + fqdn_rule->FQDN=ALLOC(char, fqdn_len+1); + memcpy(fqdn_rule->FQDN, fqdn, fqdn_len); + fqdn_rule->len=fqdn_len; + fqdn_rule->is_suffix_match=is_suffix_match; + fqdn_rule->id=id; + return fqdn_rule; +} +void fqdn_rule_free(struct FQDN_rule* fqdn_rule) +{ + assert(fqdn_rule->user_tag==NULL); + free(fqdn_rule->FQDN); + fqdn_rule->FQDN=NULL; + free(fqdn_rule); + return; +} +void _notype_fqdn_rule_free(void* p) +{ + fqdn_rule_free((struct FQDN_rule*)p); + return; +} +struct xx_plugin_ex_free_wrapper +{ + struct EX_data_rt* ex_data_rt; + char* row; + size_t key_offset; + size_t key_len; + void* logger; +}; +void xx_plugin_ex_data_wrapper_free(void* ex_data) +{ + struct xx_plugin_ex_free_wrapper* wrapper=(struct xx_plugin_ex_free_wrapper*)ex_data; + EX_data_rt_delete_by_row(wrapper->ex_data_rt, wrapper->row, wrapper->row + wrapper->key_offset, wrapper->key_len, wrapper->logger); + free(wrapper->row); + wrapper->key_offset=0; + wrapper->key_len=0; + free(wrapper); + return; +} static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num) { @@ -127,7 +168,6 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche break; case TABLE_TYPE_PLUGIN: table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, - table_schema->plugin.estimate_size, table_schema->plugin.ex_schema.key2index_func, NULL); if(table_schema->plugin.have_exdata) @@ -136,8 +176,7 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche } break; case TABLE_TYPE_IP_PLUGIN: - table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, - table_schema->ip_plugin.estimate_size, + table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, table_schema->ip_plugin.ex_schema.key2index_func, free); if(table_schema->ip_plugin.have_exdata) @@ -146,6 +185,16 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche } table_rt->ip_plugin.bin=Maat_garbage_bin_new(0); break; + case TABLE_TYPE_FQDN_PLUGIN: + table_rt->fqdn_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, + table_schema->fqdn_plugin.ex_schema.key2index_func, + _notype_fqdn_rule_free); + if(table_schema->fqdn_plugin.have_exdata) + { + EX_data_rt_set_schema(table_rt->fqdn_plugin.ex_data_rt, &table_schema->fqdn_plugin.ex_schema); + } + table_rt->fqdn_plugin.bin=Maat_garbage_bin_new(0); + break; default: break; } @@ -196,6 +245,12 @@ static void table_runtime_free(struct Maat_table_runtime* p) EX_data_rt_free(p->ip_plugin.ex_data_rt); assert(p->ip_plugin.new_ip_matcher==NULL); break; + case TABLE_TYPE_FQDN_PLUGIN: + FQDN_engine_free(p->fqdn_plugin.fqdn_engine); + Maat_garbage_bin_free(p->fqdn_plugin.bin); + EX_data_rt_free(p->fqdn_plugin.ex_data_rt); + assert(p->fqdn_plugin.new_fqdn_engine==NULL); + break; case TABLE_TYPE_PLUGIN: EX_data_rt_free(p->plugin.ex_data_rt); break; @@ -358,10 +413,156 @@ void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr MESA_lqueue_join_tail(table_rt->similar.update_q,&digest_rule, sizeof(void*)); return; } -void Maat_table_runtime_fqdn_add(struct Maat_table_runtime* table_rt, int expr_id, const char* fqdn, int is_suffix_match, void* tag) -{ +void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) +{ + struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(table_schema->fqdn_plugin); + struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin); + size_t is_valid_offset=0, valid_len=0; + size_t is_suffix_flag_offset=0, is_suffix_flag_len=0; + size_t row_id_offset=0, row_id_len=0; + size_t fqdn_offset=0, fqdn_len=0; + struct FQDN_rule* fqdn_rule=NULL; + int ret=0; + struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL; + if(fqdn_plugin_schema->have_exdata) + { + ret=Maat_helper_read_column(row, fqdn_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "fqdn_plugin EX data process error: cannot find is_valid column %d of %s", + fqdn_plugin_schema->valid_flag_column, row); + return; + } + ret=Maat_helper_read_column(row, fqdn_plugin_schema->row_id_column, &row_id_offset, &row_id_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "fqdn_plugin EX data process error: cannot find row id column %d of %s", + fqdn_plugin_schema->row_id_column, row); + return; + } + ret=Maat_helper_read_column(row, fqdn_plugin_schema->is_suffix_flag_column, &is_suffix_flag_offset, &is_suffix_flag_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "fqdn_plugin EX data process error: cannot find is_suffix_match column %d of %s", + fqdn_plugin_schema->is_suffix_flag_column, row); + return; + } + ret=Maat_helper_read_column(row, fqdn_plugin_schema->fqdn_column, &fqdn_offset, &fqdn_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "fqdn_plugin EX data process error: cannot find fqdn column %d of %s", + fqdn_plugin_schema->fqdn_column, row); + return; + } + fqdn_rule=fqdn_rule_new((unsigned int)atoi(row+row_id_offset), row+fqdn_offset, fqdn_len, atoi(row+is_suffix_flag_offset)); + + if(atoi(row+is_valid_offset)==1)//add + { + + EX_data_rt_row2EX_data(fqdn_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, fqdn_rule, logger); + } + else + { + wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1); + wrapper_for_free->row=_maat_strdup(row); + wrapper_for_free->ex_data_rt=fqdn_plugin_rt->ex_data_rt; + wrapper_for_free->key_len=row_id_len; + wrapper_for_free->key_offset=row_id_offset; + wrapper_for_free->logger=logger; + Maat_garbage_bagging(fqdn_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free); + } + } + else + { + EX_data_rt_cache_row_put(fqdn_plugin_rt->ex_data_rt, row); + } + fqdn_plugin_rt->changed_flag=1; + return; } + +int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt) +{ + struct FQDN_engine* new_fqdn_engine=NULL; + struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin; + assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN); + struct EX_data_container **exc_array=NULL; + Maat_garbage_collect_routine(fqdn_rt->bin); + struct FQDN_rule* rules=NULL; + size_t rule_cnt=0, i=0; + if(!fqdn_rt->changed_flag) + { + assert(0==Maat_garbage_bin_get_size(fqdn_rt->bin)); + return 0; + } + + rule_cnt=EX_data_rt_list_all_ex_container(fqdn_rt->ex_data_rt, &exc_array); + rules=ALLOC(struct FQDN_rule, rule_cnt); + for(i=0; i0) + { + new_fqdn_engine=FQDN_engine_new(rules, rule_cnt); + } + fqdn_rt->new_fqdn_engine=new_fqdn_engine; + free(rules); + free(exc_array); + return 1; +} +struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt) +{ + struct FQDN_engine* old_one=table_rt->fqdn_plugin.fqdn_engine; + table_rt->fqdn_plugin.fqdn_engine=table_rt->fqdn_plugin.new_fqdn_engine; + assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN); + table_rt->fqdn_plugin.new_fqdn_engine=NULL; + table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->fqdn_plugin.ex_data_rt); + return old_one; +} + +int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger) +{ + size_t i=0; + const char* row=NULL; + struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin); + EX_data_rt_set_schema(fqdn_plugin_rt->ex_data_rt, &table_schema->fqdn_plugin.ex_schema); + for(i=0; iex_data_rt); i++) + { + row=EX_data_rt_cached_row_get(fqdn_plugin_rt->ex_data_rt, i); + Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, row, logger); + } + EX_data_rt_clear_row_cache(fqdn_plugin_rt->ex_data_rt); + Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt); + Maat_table_runtime_apply_new_fqdn_engine(table_rt);//returned NULL. + return 0; +} +int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size) +{ + struct FQDN_match results[size]; + int n_result=0, i=0; + if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN) + { + return -1; + } + if(!table_rt->fqdn_plugin.fqdn_engine) + { + return 0; + } + n_result=FQDN_engine_search(table_rt->fqdn_plugin.fqdn_engine, query_fqdn, strlen(query_fqdn), results, size); + for(i=0; ifqdn_plugin.ex_data_rt, (struct EX_data_container *)results[i].user_tag); + } + return n_result; +} + int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt) { long i=0,data_size=0; @@ -463,41 +664,28 @@ int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* ip_plugin->changed_flag=0; return 1; } + + + struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt) { struct ip_matcher* old_one=table_rt->ip_plugin.ip_matcher; table_rt->ip_plugin.ip_matcher=table_rt->ip_plugin.new_ip_matcher; assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN); table_rt->ip_plugin.new_ip_matcher=NULL; + table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->ip_plugin.ex_data_rt); return old_one; } -struct ip_plugin_ex_free_wrapper -{ - struct EX_data_rt* ex_data_rt; - char* row; - size_t key_offset; - size_t key_len; - void* logger; -}; -void ip_plugin_ex_data_wrapper_free(void* ex_data) -{ - struct ip_plugin_ex_free_wrapper* wrapper=(struct ip_plugin_ex_free_wrapper*)ex_data; - EX_data_rt_delete_by_row(wrapper->ex_data_rt, wrapper->row, wrapper->row + wrapper->key_offset, wrapper->key_len, wrapper->logger); - free(wrapper->row); - wrapper->key_offset=0; - wrapper->key_len=0; - free(wrapper); - return; -} + void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) { struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin); struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin); size_t is_valid_offset=0, valid_len=0; - size_t key_offset=0, key_len=0; + size_t row_id_offset=0, row_id_len=0; struct ip_rule* ip_rule=NULL; int ret=0; - struct ip_plugin_ex_free_wrapper* wrapper_for_free=NULL; + struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL; if(ip_plugin_schema->have_exdata) { ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); @@ -508,7 +696,7 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s ip_plugin_schema->row_id_column, row); return; } - ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &key_offset, &key_len); + ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &row_id_offset, &row_id_len); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, @@ -527,17 +715,18 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s if(atoi(row+is_valid_offset)==1)//add { - EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+key_offset, key_len, ip_rule, logger); + EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, ip_rule, logger); } else { - wrapper_for_free=ALLOC(struct ip_plugin_ex_free_wrapper, 1); + wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1); wrapper_for_free->row=_maat_strdup(row); wrapper_for_free->ex_data_rt=ip_plugin_rt->ex_data_rt; - wrapper_for_free->key_len=key_len; - wrapper_for_free->key_offset=key_offset; + wrapper_for_free->key_len=row_id_len; + wrapper_for_free->key_offset=row_id_offset; wrapper_for_free->logger=logger; - Maat_garbage_bagging(ip_plugin_rt->bin, wrapper_for_free, ip_plugin_ex_data_wrapper_free); + Maat_garbage_bagging(ip_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free); + free(ip_rule); } } else diff --git a/src/inc_internal/Maat_ex_data.h b/src/inc_internal/Maat_ex_data.h index baa3d15..17230ab 100644 --- a/src/inc_internal/Maat_ex_data.h +++ b/src/inc_internal/Maat_ex_data.h @@ -3,7 +3,7 @@ struct EX_data_rt; -struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)); +struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)); void EX_data_rt_free(struct EX_data_rt* p); void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema); void EX_data_rt_cache_row_put(struct EX_data_rt* p, const char* row); @@ -23,4 +23,5 @@ MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container); size_t EX_data_rt_list_all_ex_container(struct EX_data_rt* ex_rt, struct EX_data_container*** ex_container_array); void* EX_data_container_get_user_data(struct EX_data_container* ex_container); +size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt); diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 634f3d5..dba92e2 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -214,7 +214,7 @@ struct Maat_scanner mcore_long_t ref_cnt; rule_scanner_t region; size_t gie_update_q_size; - size_t ip_plugin_update_q_size; + size_t xx_plugin_update_q_size; size_t to_update_group_cnt; size_t to_update_compile_cnt; diff --git a/src/inc_internal/Maat_table.h b/src/inc_internal/Maat_table.h index 85d0eb3..e6662a0 100644 --- a/src/inc_internal/Maat_table.h +++ b/src/inc_internal/Maat_table.h @@ -37,7 +37,6 @@ enum MAAT_TABLE_TYPE TABLE_TYPE_PLUGIN, TABLE_TYPE_IP_PLUGIN, TABLE_TYPE_INTERVAL_PLUS, - TABLE_TYPE_FQDN, TABLE_TYPE_FQDN_PLUGIN }; @@ -102,7 +101,6 @@ struct plugin_table_schema int foreign_columns[MAX_FOREIGN_CLMN_NUM]; int cb_plug_cnt; int have_exdata; - long long estimate_size; struct plugin_table_callback_schema cb_plug[MAX_PLUGIN_PER_TABLE]; struct EX_data_schema ex_schema; }; @@ -114,7 +112,17 @@ struct ip_plugin_table_schema int end_ip_column; int valid_flag_column; int rule_tag_column; - long long estimate_size; + int have_exdata; + struct EX_data_schema ex_schema; +}; +struct fqdn_plugin_table_schema +{ + + int row_id_column; + int is_suffix_flag_column; + int fqdn_column; + int valid_flag_column; + int rule_tag_column; int have_exdata; struct EX_data_schema ex_schema; }; @@ -131,6 +139,7 @@ struct Maat_table_schema struct expr_table_schema expr; struct plugin_table_schema plugin; struct ip_plugin_table_schema ip_plugin; + struct fqdn_plugin_table_schema fqdn_plugin; struct virtual_table_schema virtual_table; struct composition_table_schema composition; void* others;//group, ip, interval and digest don't have any special schema. @@ -183,6 +192,13 @@ int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_sche Maat_plugin_EX_key2index_func_t* key2index_func, long argl, void *argp, void* logger); +int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp, + void* logger); void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type); void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr); @@ -198,4 +214,5 @@ enum MAAT_TABLE_CHILD_TYPE CHILD_TABLE_TYPE_SESSION }; int Maat_table_get_child_id(struct Maat_table_schema* p_table, enum MAAT_TABLE_CHILD_TYPE type); +int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table); diff --git a/src/inc_internal/Maat_table_runtime.h b/src/inc_internal/Maat_table_runtime.h index 60d1f49..0ac2846 100644 --- a/src/inc_internal/Maat_table_runtime.h +++ b/src/inc_internal/Maat_table_runtime.h @@ -15,10 +15,10 @@ struct similar_runtime GIE_handle_t* gie_handle; MESA_lqueue_head update_q; }; -struct fqdn_runtime +struct fqdn_plugin_runtime { struct FQDN_engine* fqdn_engine; - struct FQDN_engine* old_fqdn_engine; + struct FQDN_engine* new_fqdn_engine; struct EX_data_rt* ex_data_rt; //for fqdn_plugin ONLY struct Maat_garbage_bin* bin; int changed_flag; @@ -59,7 +59,7 @@ struct Maat_table_runtime union { struct similar_runtime similar; //for digest and similarity - struct fqdn_runtime fqdn;//for fqdn and fqdn_plugin + struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin struct plugin_runtime plugin; struct ip_plugin_runtime ip_plugin; struct expr_runtime expr; @@ -95,4 +95,12 @@ int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_ int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* table_rt); struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt); +int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt); + +void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); +int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt); +struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt); +int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger); +int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size); + diff --git a/test/maat_json.json b/test/maat_json.json index f7d8d59..11e1520 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -1856,45 +1856,55 @@ { "table_name": "QD_ENTRY_INFO", "table_content": [ - "1\t192.168.0.1\t101\t1", - "2\t192.168.0.2\t101\t1", - "3\t192.168.1.1\t102\t1" + "1\t192.168.0.1\t101\t1", + "2\t192.168.0.2\t101\t1", + "3\t192.168.1.1\t102\t1" ] }, { "table_name": "TEST_PLUGIN_TABLE", "table_content": [ - "1\t3388\t99\t1", - "2\t3355\t66\t1", - "3\tcccc\t11\t1" + "1\t3388\t99\t1", + "2\t3355\t66\t1", + "3\tcccc\t11\t1" ] }, { "table_name": "TEST_EFFECTIVE_RANGE_TABLE", "table_content": [ - "1\tSUCCESS\t99\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含/鏈濋槼/鍗庝弗鍖楅噷\"]},{\"tag\":\"isp\",\"value\":[\"鐢典俊\",\"绉诲姩\"]}]]}\t1111", - "2\tSUCCESS\t66\t1\t0\t222", - "3\tFAILED\t11\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含/鏈濋槼/鍗庝弗鍖楅噷\",\"涓婃捣/娴︿笢/闄嗗鍢碶"]},{\"tag\":\"isp\",\"value\":[\"鐢典俊\",\"鑱旈歕"]}],[{\"tag\":\"location\",\"value\":[\"鍖椾含\"]},{\"tag\":\"isp\",\"value\":[\"鑱旈歕"]}]]}\t333", - "4\tSUCCESS\t66\t1\t{}\t444", - "5\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含\"]}]]}\t444", - "6\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"weather\",\"value\":[\"hot\"]}]]}\t444" + "1\tSUCCESS\t99\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含/鏈濋槼/鍗庝弗鍖楅噷\"]},{\"tag\":\"isp\",\"value\":[\"鐢典俊\",\"绉诲姩\"]}]]}\t1111", + "2\tSUCCESS\t66\t1\t0\t222", + "3\tFAILED\t11\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含/鏈濋槼/鍗庝弗鍖楅噷\",\"涓婃捣/娴︿笢/闄嗗鍢碶"]},{\"tag\":\"isp\",\"value\":[\"鐢典俊\",\"鑱旈歕"]}],[{\"tag\":\"location\",\"value\":[\"鍖椾含\"]},{\"tag\":\"isp\",\"value\":[\"鑱旈歕"]}]]}\t333", + "4\tSUCCESS\t66\t1\t{}\t444", + "5\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含\"]}]]}\t444", + "6\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"weather\",\"value\":[\"hot\"]}]]}\t444" ] }, { "table_name": "IR_INTERCEPT_IP", "table_content": [ - "1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34", - "161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42" + "1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34", + "161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42" ] }, { "table_name": "TEST_IP_PLUGIN_WITH_EXDATA", "table_content": [ - "101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1", + "101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1", "102\t4\t192.168.30.90\t192.168.30.128\tBigger-range-should-in-the-back\t1", - "103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1", + "103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1", "104\t6\t2001:db8:1234::1\t2001:db8:1234::5210\tSomething-like-json\t1" ] + }, + { + "table_name": "TEST_FQDN_PLUGIN_WITH_EXDATA", + "table_content": [ + "201\t0\twww.example1.com\tcatid=1\t1", + "202\t1\t.example1.com\tcatid=1\t1", + "203\t0\tnews.example1.com\tcatid=2\t1", + "204\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1", + "205\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1" + ] } ] } diff --git a/test/table_info.conf b/test/table_info.conf index 64387a3..c92aa17 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -55,4 +55,5 @@ 32 DESTINATION_IP_ASN virtual AS_NUMBER -- 33 GeoLocation expr UTF8 UTF8 yes 0 34 SOURCE_IP_GEO virtual GeoLocation -- -35 INTERGER_PLUS intval_plus -- \ No newline at end of file +35 INTERGER_PLUS intval_plus -- +36 TEST_FQDN_PLUGIN_WITH_EXDATA fqdn_plugin {"row_id":1,"is_suffix_match":2,"fqdn":3,"valid":5} -- \ No newline at end of file diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 6d9e01a..48603f0 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -310,6 +310,86 @@ TEST(IP_Plugin_Table, EX_DATA) } +#define FQDN_PLUGIN_EX_DATA +struct fqdn_plugin_ud +{ + int rule_id; + int catid; + int ref_cnt; +}; +void fqdn_plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) +{ + int *counter=(int *)argp, ret=0; + size_t column_offset=0, column_len=0; + struct fqdn_plugin_ud* ud=(struct fqdn_plugin_ud*)calloc(sizeof(struct fqdn_plugin_ud), 1); + ret=Maat_helper_read_column(table_line, 1, &column_offset, &column_len); + EXPECT_EQ(ret, 0); + ud->rule_id=atoi(table_line+column_offset); + ret=Maat_helper_read_column(table_line, 4, &column_offset, &column_len); + EXPECT_EQ(ret, 0); + sscanf(table_line+column_offset, "catid=%d",&ud->catid); + ud->ref_cnt=1; + *ad=ud; + (*counter)++; + return; +} +void fqdn_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) +{ + struct fqdn_plugin_ud* u=(struct fqdn_plugin_ud*)(*ad); + u->ref_cnt--; + if(u->ref_cnt>0) return; + free(u); + *ad=NULL; +} +void fqdn_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) +{ + struct fqdn_plugin_ud* u=(struct fqdn_plugin_ud*)(*from); + u->ref_cnt++; + *to=u; +} +TEST(FQDN_Plugin_Table, EX_DATA) +{ + + int fqdn_plugin_ex_data_counter=0, i=0; + const char* table_name="TEST_FQDN_PLUGIN_WITH_EXDATA"; + int table_id=0, ret=0; + table_id=Maat_table_register(g_feather, table_name); + ASSERT_GT(table_id, 0); + ret=Maat_fqdn_plugin_EX_register(g_feather, table_id, + fqdn_plugin_EX_new_cb, + fqdn_plugin_EX_free_cb, + fqdn_plugin_EX_dup_cb, + 0, &fqdn_plugin_ex_data_counter); + ASSERT_TRUE(ret>=0); + EXPECT_EQ(fqdn_plugin_ex_data_counter, 5); + + struct fqdn_plugin_ud* result[4]; + + ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "www.example1.com", (void**)result, 4); + ASSERT_EQ(ret, 2); + EXPECT_EQ(result[0]->rule_id, 201); + EXPECT_EQ(result[1]->rule_id, 202); + + for(i=0; irule_id, 204); + EXPECT_EQ(result[1]->rule_id, 205); + for(i=0; i=0); + EXPECT_EQ(fqdn_plugin_ex_data_counter, 5); + + + struct fqdn_plugin_ud* result[4]; + + ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4); + ASSERT_EQ(ret, 2); + for(i=0; i