diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 124c9ad..cea3938 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -299,9 +299,29 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, Maat_plugin_EX_dup_func_t* dup_func, Maat_plugin_EX_key2index_func_t* key2index_func, long argl, void *argp); -//Data is duplicated by dup_func of Maat_plugin_EX_register, caller is responsible to free the data. +//Data is duplicated by dup_func of Maat_plugin_EX_register, caller is responsible to FREE the data. MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key); + +int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + long argl, void *argp); + +struct ip_address +{ + int ip_type; //4: IPv4, 6: IPv6 + union + { + unsigned int ipv4; //network order + unsigned int ipv6[4]; + }; +}; + +int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data); + + enum MAAT_RULE_OPT { MAAT_RULE_SERV_DEFINE //VALUE is a char* buffer,SIZE= buffer size. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f19b34e..9ebce39 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) -set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp) +set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) include_directories(/opt/MESA/include/MESA/) @@ -21,6 +21,7 @@ target_include_directories(maat_frame_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} #target_include_directories(maat_frame_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/hiredis) target_link_libraries(maat_frame_static hiredis-vip-static) target_link_libraries(maat_frame_static igraph-static) +target_link_libraries(maat_frame_static ipmatcher-static) # Shared Library Output add_library(maat_frame_shared SHARED ${MAAT_SRC}) @@ -34,6 +35,7 @@ target_include_directories(maat_frame_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} target_include_directories(maat_frame_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/hiredis) target_link_libraries(maat_frame_shared hiredis-vip-static ${MAAT_DEPEND_DYN_LIB}) target_link_libraries(maat_frame_shared igraph-static ${MAAT_DEPEND_DYN_LIB}) +target_link_libraries(maat_frame_shared ipmatcher-static) install(FILES ${PROJECT_SOURCE_DIR}/inc/Maat_rule.h DESTINATION /opt/MESA/include/MESA/ COMPONENT HEADER) install(FILES ${PROJECT_SOURCE_DIR}/inc/Maat_command.h DESTINATION /opt/MESA/include/MESA/ COMPONENT HEADER) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index c8122a6..b25867a 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -1331,38 +1331,24 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - int idx=-1; + int ret=0; + struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id); - if(new_func==NULL || free_func==NULL || dup_func==NULL ) - { - assert(0); - MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); - return -1; - } pthread_mutex_lock(&(_feather->background_update_mutex)); + ret=Maat_table_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, key2index_func,argl, argp, _feather->logger); - idx=Maat_table_plugin_new_ex_index(_feather->table_mgr, table_id, - new_func, - free_func, - dup_func, - key2index_func, - argl, argp); - if(idx<0) + if(ret<0) { pthread_mutex_unlock(&(_feather->background_update_mutex)); return -1; } - struct Maat_table_schema *table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); struct Maat_table_runtime* table_rt=NULL; - - if(_feather->scanner!=NULL) { table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); - Maat_table_runtime_plugin_new_ex_idx(table_rt, table_desc, _feather->logger); + Maat_table_runtime_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger); } pthread_mutex_unlock(&(_feather->background_update_mutex)); - return 0; } MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key) @@ -1380,6 +1366,65 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key); return exdata; } +int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + long argl, void *argp) + +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + int ret=-1; + struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id); + pthread_mutex_lock(&(_feather->background_update_mutex)); + ret=Maat_table_ip_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger); + + if(ret<0) + { + pthread_mutex_unlock(&(_feather->background_update_mutex)); + return -1; + } + + struct Maat_table_runtime* table_rt=NULL; + if(_feather->scanner!=NULL) + { + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + Maat_table_runtime_ip_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger); + } + pthread_mutex_unlock(&(_feather->background_update_mutex)); + + return 0; +} + +int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data) +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct Maat_table_schema *table_schema=NULL; + struct Maat_table_runtime *table_rt=NULL; + int n_get=0; + if(_feather->scanner==NULL) + { + return 0; + } + table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + if(table_rt->table_type!=TABLE_TYPE_IP_PLUGIN) + { + return -1; + } + struct ip_data ip_data=*(const struct ip_data*)ip; + if(ip_data.type==IPv4) + { + ip_data.ipv4=ntohl(ip_data.ipv4); + } + else + { + ipv6_ntoh(ip_data.ipv6); + } + n_get=Maat_table_runtime_ip_plugin_get_N_ex_data(table_rt, table_schema, &ip_data, ex_data_array, n_ex_data); + return n_get; + +} int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id ,enum MAAT_CHARSET charset,const char* data,int data_len diff --git a/src/entry/Maat_ex_data.cpp b/src/entry/Maat_ex_data.cpp new file mode 100644 index 0000000..25f6bee --- /dev/null +++ b/src/entry/Maat_ex_data.cpp @@ -0,0 +1,193 @@ +#include "Maat_ex_data.h" +#include "Maat_table.h" +#include "Maat_utils.h" + +#include +#include + +void EX_data_container_free(void *data) +{ + struct EX_data_container* wrap_data=(struct EX_data_container*)data; + const struct EX_data_schema* ex_schema=wrap_data->rt->ex_schema; + ex_schema->free_func(wrap_data->rt->table_id, &(wrap_data->ex_data), ex_schema->argl, ex_schema->argp); + if(wrap_data->user_data && wrap_data->rt->user_data_free) + { + wrap_data->rt->user_data_free(wrap_data->user_data); + } + wrap_data->user_data=NULL; + wrap_data->rt=NULL; + free(wrap_data); + return; +} +static MESA_htable_handle EX_data_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index) +{ + MESA_htable_handle key2ex_hash=NULL; + unsigned int slot_size=1; + while(estimate_size!=0) + { + estimate_size=estimate_size>>1; + slot_size*=2; + } + if(slot_size==1) + { + slot_size=4096; + } + + MESA_htable_create_args_t hargs; + memset(&hargs,0,sizeof(hargs)); + hargs.thread_safe=8; + hargs.hash_slot_size = slot_size; + hargs.max_elem_num = 0; + hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; + hargs.expire_time = 0; + hargs.key_comp = NULL; + hargs.key2index = NULL; //Not supported yet. + hargs.recursive = 1; + hargs.data_free = EX_data_container_free; + hargs.data_expire_with_condition = NULL; + key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(key2ex_hash, 0); + return key2ex_hash; +} + +struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)) +{ + struct EX_data_rt* p=ALLOC(struct EX_data_rt, 1); + p->key2ex_hash=EX_data_hash_new(estimate_size, key2index); + p->cache_rows=dynamic_array_create(4, 1024); + p->table_id=table_id; + p->user_data_free=user_data_free; + return p; +}; +void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema) +{ + p->ex_schema=schema; +} +void EX_data_rt_free(struct EX_data_rt* p) +{ + if(p->cache_rows) + { + dynamic_array_destroy(p->cache_rows, free); + p->cache_rows=NULL; + } + MESA_htable_destroy(p->key2ex_hash, NULL); + return; +} +void EX_data_rt_cache_row(struct EX_data_rt* p, const char* row) +{ + size_t len=strlen(row)+1; + char* row_copy=ALLOC(char, len); + memcpy(row_copy, row, len); + p->cache_size+=len; + dynamic_array_write(p->cache_rows, p->cache_row_num, row_copy); + p->cache_row_num++; + return; +} +const char* EX_data_rt_get_cached_row(struct EX_data_rt* p, int i) +{ + const char* row=NULL; + row=(const char*)dynamic_array_read(p->cache_rows, i); + return row; +} +void EX_data_rt_clear_row_cache(struct EX_data_rt* p) +{ + dynamic_array_destroy(p->cache_rows, free); + p->cache_rows=NULL; + p->cache_row_num=0; + p->cache_size=0; +} +int EX_data_rt_get_row_num(struct EX_data_rt* p) +{ + return p->cache_row_num; +} +struct EX_data_container* EX_data_rt_row2EX_data(struct EX_data_rt* ex_rt, + const char* row, const char* key, size_t key_len, + void* user_data, void* logger) +{ + + MAAT_RULE_EX_DATA ex_data=NULL; + int ret=0; + const struct EX_data_schema* ex_schema=ex_rt->ex_schema; + struct EX_data_container* ex_container=ALLOC(struct EX_data_container, 1); + ex_schema->new_func(ex_rt->table_id, key, row, &ex_data, + ex_schema->argl, ex_schema->argp); + ex_container->ex_data=ex_data; + ex_container->rt=ex_rt; + ex_container->user_data=user_data; + ret=MESA_htable_add(ex_rt->key2ex_hash, (unsigned char*)key, key_len, ex_container); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "EX data add error: duplicated key %.*s of %s", + key_len, key, row); + EX_data_container_free(ex_container); + return NULL; + } + return ex_container; +} +int EX_data_rt_delete_by_row(struct EX_data_rt* ex_rt, const char* row, const char* key, size_t key_len, + void *logger) +{ + int ret=0; + ret=MESA_htable_del(ex_rt->key2ex_hash, (const unsigned char*)key, key_len, NULL); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "EX data del error: no such key %.*s of %s", + key_len, key, row); + return -1; + } + return 0; +} + +MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const char* key, size_t key_len) +{ + struct EX_data_container* container=NULL; + MAAT_RULE_EX_DATA ex_data=NULL; + + if(!ex_rt->ex_schema) + { + assert(0); + return NULL; + } + container=(struct EX_data_container*)MESA_htable_search(ex_rt->key2ex_hash, + (const unsigned char*)key, strlen(key)); + if(container!=NULL) + { + ex_rt->ex_schema->dup_func(ex_rt->table_id, &(ex_data), &(container->ex_data), + ex_rt->ex_schema->argl, ex_rt->ex_schema->argp); + } + return ex_data; +} +MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container) +{ + MAAT_RULE_EX_DATA ex_data=NULL; + ex_rt->ex_schema->dup_func(ex_rt->table_id, &(ex_data), &(container->ex_data), + ex_rt->ex_schema->argl, ex_rt->ex_schema->argp); + return ex_data; +} +struct key2EX_hash_walker +{ + EX_data_container_q* listed; + size_t count; +}; + +void walk_key2EX_hash(const uchar * key, uint size, void * data, void * user) +{ + struct key2EX_hash_walker *walker=(struct key2EX_hash_walker*)user; + struct EX_data_container* ex_container=(struct EX_data_container*)data; + TAILQ_INSERT_TAIL(walker->listed, ex_container, entries); + walker->count++; + return; +} +size_t EX_data_rt_list_all(struct EX_data_rt* ex_rt, EX_data_container_q* listed) +{ + size_t ex_data_cnt; + struct key2EX_hash_walker walker={listed, 0}; + TAILQ_INIT(listed); + MESA_htable_iterate(ex_rt->key2ex_hash, walk_key2EX_hash, &walker); + ex_data_cnt=(size_t)MESA_htable_get_elem_num(ex_rt->key2ex_hash); + assert(walker.count==ex_data_cnt); + return ex_data_cnt; +} + diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index b6aaca5..902fab7 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -2410,142 +2410,7 @@ error_out: free(maat_str_rule); maat_str_rule=NULL; } -enum MAAT_IP_FORMAT -{ - FORMAT_RANGE, - FORMAT_MASK, - FORMAT_CIDR, - FORMAT_UNKNOWN -}; -enum MAAT_IP_FORMAT ip_format_str2int(const char* format) -{ - if(0==strcasecmp(format, "range")) - { - return FORMAT_RANGE; - } - else if(0==strcasecmp(format, "mask")) - { - return FORMAT_MASK; - } - else if(0==strcasecmp(format, "CIDR")) - { - return FORMAT_CIDR; - } - else - { - assert(0); - } - return FORMAT_UNKNOWN; -} -int ip_format2range(int ip_type, enum MAAT_IP_FORMAT format, const char* ip1, const char* ip2, unsigned int range_begin[], unsigned int range_end[]) -{ - unsigned int ipv4_addr=0, ipv4_mask=0, ipv4_range_end=0; - unsigned int ipv6_addr[4]={0}, ipv6_mask[4]={0}, ipv6_range_end[4]={0}; - int cidr=0, bit32=0; - int ret=0, i=0; - if(ip_type!=4 && ip_type!=6) - { - assert(0); - return -1; - } - if(ip_type==4) - { - ret=inet_pton(AF_INET, ip1, &ipv4_addr); - if(ret<=0) - { - return -1; - } - ipv4_addr=ntohl(ipv4_addr); - switch (format) - { - case FORMAT_RANGE: - range_begin[0]=ipv4_addr; - ret=inet_pton(AF_INET, ip2, &ipv4_range_end); - if(ret<=0) - { - return -1; - } - ipv4_range_end=ntohl(ipv4_range_end); - range_end[0]=ipv4_range_end; - break; - case FORMAT_MASK: - ret=inet_pton(AF_INET, ip2, &ipv4_mask); - if(ret<=0) - { - return -1; - } - ipv4_mask=ntohl(ipv4_mask); - range_begin[0]=ipv4_addr&ipv4_mask; - range_end[0]=ipv4_addr|~ipv4_mask; - break; - case FORMAT_CIDR: - cidr=atoi(ip2); - if(cidr>32||cidr<0) - { - return -1; - } - ipv4_mask = (0xFFFFFFFFUL << (32 - cidr)) & 0xFFFFFFFFUL; - range_begin[0]=ipv4_addr&ipv4_mask; - range_end[0]=ipv4_addr|~ipv4_mask; - break; - default: - assert(0); - } - } - else //ipv6 - { - ret=inet_pton(AF_INET6, ip1, ipv6_addr); - if(ret<=0) - { - return -1; - } - ipv6_ntoh(ipv6_addr); - switch(format) - { - case FORMAT_RANGE: - ret=inet_pton(AF_INET6, ip2, ipv6_range_end); - if(ret<=0) - { - return -1; - } - ipv6_ntoh(ipv6_range_end); - memcpy(range_begin, ipv6_addr, sizeof(ipv6_addr)); - memcpy(range_end, ipv6_range_end, sizeof(ipv6_range_end)); - break; - case FORMAT_MASK: - ret=inet_pton(AF_INET6, ip2, ipv6_mask); - if(ret<=0) - { - return -1; - } - ipv6_ntoh(ipv6_mask); - for(i=0; i<4; i++) - { - range_begin[i]=ipv6_addr[i]&ipv6_mask[i]; - range_end[i] = ipv6_addr[i]|~ipv6_mask[i]; - } - break; - case FORMAT_CIDR: - cidr=atoi(ip2); - if(cidr>128||cidr<0) - { - return -1; - } - for(i=0; i<4; i++) - { - bit32=128-cidr-32*(3-i); - if(bit32<0) bit32=0; - ipv6_mask[i]=(0xFFFFFFFFUL << bit32) & 0xFFFFFFFFUL; - range_begin[i]=ipv6_addr[i]&ipv6_mask[i]; - range_end[i] = ipv6_addr[i]|~ipv6_mask[i]; - } - break; - default: - assert(0); - } - } - return 0; -} + void update_ip_rule(struct Maat_table_schema* table, const char* table_line, struct Maat_scanner *scanner, void* logger) { struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); @@ -3032,12 +2897,12 @@ void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeo MESA_lqueue_join_tail(garbage_q,&bag,sizeof(void*)); return; } -void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q) +void garbage_bagging(enum maat_garbage_type type, void *p, MESA_lqueue_head garbage_q) { garbage_bagging_with_timeout(type, p, -1, garbage_q); return; } -void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger) +void garbage_bury(MESA_lqueue_head garbage_q, int timeout, void *logger) { UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; _maat_garbage_t* bag=NULL; @@ -3117,6 +2982,10 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger) free(bag->filename); bag->filename=NULL; break; + case GARBAGE_IP_MATCHER: + ip_matcher_free(bag->a_ip_matcher); + bag->a_ip_matcher=NULL; + break; default: assert(0); } @@ -3131,44 +3000,41 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger) q_cnt,bury_cnt); } } -void update_plugin_table(struct Maat_table_schema* table,const char* table_line,Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) +void update_plugin_table(struct Maat_table_schema* table_schema, const char* row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) { - int i=0, ret=1, matched_tag=1; - unsigned int len=strlen(table_line)+1; - struct plugin_table_schema* plugin_desc=&(table->plugin); - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); - char *p=NULL; + int ret=1, matched_tag=1; + struct plugin_table_schema* plugin_desc=&(table_schema->plugin); + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_schema->table_id); char* copy=NULL; - size_t is_valid_offset=0, valid_len=0; size_t accept_tag_offset=0, accept_tag_len=0; if(plugin_desc->rule_tag_column>0&&n_tags>0) { - ret=Maat_helper_read_column(table_line, plugin_desc->rule_tag_column, &accept_tag_offset, &accept_tag_len); + ret=Maat_helper_read_column(row, plugin_desc->rule_tag_column, &accept_tag_offset, &accept_tag_len); if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error, could not locate tag in column %d of plugin table %s:%s", + "update error, could not locate tag in column %d of plugin table_schema %s:%s", plugin_desc->rule_tag_column, - table->table_name[table->updating_name], - table_line); - table->udpate_err_cnt++; + table_schema->table_name[table_schema->updating_name], + row); + table_schema->udpate_err_cnt++; return; } if(accept_tag_len>2) { copy=ALLOC(char, accept_tag_len+1); - memcpy(copy, table_line+accept_tag_offset, accept_tag_len); + memcpy(copy, row+accept_tag_offset, accept_tag_len); matched_tag=compare_accept_tag(copy, tags, n_tags); if(matched_tag<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,invalid tag format of plugin table %s:%s" - ,table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; + "update error,invalid tag format of plugin table_schema %s:%s" + ,table_schema->table_name[table_schema->updating_name],row); + table_schema->udpate_err_cnt++; } if(matched_tag==0) { - table->unmatch_tag_cnt++; + table_schema->unmatch_tag_cnt++; } free(copy); copy=NULL; @@ -3178,42 +3044,56 @@ void update_plugin_table(struct Maat_table_schema* table,const char* table_line, return; } } - - table_rt->plugin.acc_line_num++; - if(plugin_desc->have_exdata || plugin_desc->cb_plug_cnt>0) - { - - if(plugin_desc->have_exdata) - { - ret=Maat_helper_read_column(table_line, plugin_desc->valid_flag_column, &is_valid_offset, &valid_len); - //thread safe is protected by background_update_mutex - if(atoi(table_line+is_valid_offset)==1) - { - plugin_EX_data_new(table, table_line, table_rt->plugin.key2ex_hash, logger); - } - else - { - plugin_EX_data_free(table, table_line, table_rt->plugin.key2ex_hash, logger); - } - } - if(plugin_desc->cb_plug_cnt>0) - { - for(i=0;icb_plug_cnt;i++) - { - plugin_desc->cb_plug[i].update(table->table_id,table_line,plugin_desc->cb_plug[i].u_para); - } - } - - } - else - { - p=ALLOC(char, len); - memcpy(p,table_line,len); - table_rt->plugin.cache_size+=len; - dynamic_array_write(table_rt->plugin.cache_lines,table_rt->plugin.cache_line_num,p); - table_rt->plugin.cache_line_num++; - } + Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger); } +void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) +{ + int ret=1, matched_tag=1; + struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin); + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_schema->table_id); + char* copy=NULL; + size_t accept_tag_offset=0, accept_tag_len=0; + if(ip_plugin_schema->rule_tag_column>0&&n_tags>0) + { + ret=Maat_helper_read_column(table_row, ip_plugin_schema->rule_tag_column, &accept_tag_offset, &accept_tag_len); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error, could not locate tag in column %d of plugin table_schema %s:%s", + ip_plugin_schema->rule_tag_column, + table_schema->table_name[table_schema->updating_name], + table_row); + table_schema->udpate_err_cnt++; + return; + } + if(accept_tag_len>2) + { + copy=ALLOC(char, accept_tag_len+1); + memcpy(copy, table_row+accept_tag_offset, accept_tag_len); + matched_tag=compare_accept_tag(copy, tags, n_tags); + if(matched_tag<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "update error, invalid tag format of ip_plugin table_schema %s:%s", + table_schema->table_name[table_schema->updating_name], table_row); + table_schema->udpate_err_cnt++; + } + if(matched_tag==0) + { + table_schema->unmatch_tag_cnt++; + } + free(copy); + copy=NULL; + } + if(!matched_tag) + { + return; + } + } + Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger); + return; +} + void vector_print(igraph_vector_t *v) { long int i; for (i=0; igroup_graph), &is_dag); @@ -3355,7 +3236,10 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, "GIE_update error."); } break; - case TABLE_TYPE_PLUGIN: + case TABLE_TYPE_IP_PLUGIN: + Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt); + old_ip_matcher=Maat_table_runtime_dettach_old_ip_matcher(table_rt); + garbage_bagging(GARBAGE_IP_MATCHER, old_ip_matcher, garbage_q); break; default: break; @@ -3513,6 +3397,10 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) break; case TABLE_TYPE_PLUGIN: update_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); + break; + case TABLE_TYPE_IP_PLUGIN: + update_ip_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); + break; default: break; diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index f26d1d9..c283618 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -219,7 +219,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) switch(p_table->table_type) { case TABLE_TYPE_PLUGIN: - plugin_cache_num+=table_rt->plugin.cache_line_num; + plugin_cache_num+=table_rt->plugin.ex_data_rt->cache_row_num; plugin_acc_num+=table_rt->plugin.acc_line_num; break; case TABLE_TYPE_GROUP: diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table.cpp index 65ea347..e73ed79 100644 --- a/src/entry/Maat_table.cpp +++ b/src/entry/Maat_table.cpp @@ -74,7 +74,7 @@ int read_expr_table_info(const char* line, struct Maat_table_schema* table, MESA } return 0; } -int read_virtual_table_info(const char* line, struct Maat_table_schema* table, MESA_htable_handle string2int_map) +int read_virtual_table_schema(const char* line, struct Maat_table_schema* table, MESA_htable_handle string2int_map) { int ret=0; char table_type[16]; @@ -117,8 +117,11 @@ int _read_integer_arrary(char* string, int *array, int size) } return i; } -#define COLUMN_PLUGIN_DESCR_JSON 4 -int read_plugin_table_description(const char* line, struct Maat_table_schema* p) +#define COLUMN_PLUGIN_SCHEMA_JSON 4 +#define COLUMN_IP_PLUGIN_SCHEMA_JSON 4 +#define COLUMN_COMPOSITION_SCHEMA_JSON 4 + +int read_plugin_table_schema(const char* line, struct Maat_table_schema* p) { int i=0,ret=0; size_t offset=0, len=0; @@ -126,7 +129,7 @@ int read_plugin_table_description(const char* line, struct Maat_table_schema* p) char* copy_line=NULL, *plug_info=NULL; struct plugin_table_schema* plugin_desc=&(p->plugin); copy_line=_maat_strdup(line); - ret=get_column_pos(copy_line, COLUMN_PLUGIN_DESCR_JSON, &offset, &len); + ret=get_column_pos(copy_line, COLUMN_PLUGIN_SCHEMA_JSON, &offset, &len); if(ret<0) { goto error_out; @@ -202,7 +205,97 @@ error_out: free(copy_line); return -1; } -#define COLUMN_COMPOSITION_SCHEMA_JSON 4 +int read_ip_plugin_table_schema(const char* line, struct Maat_table_schema* p) +{ + int ret=0, read_cnt=0; + size_t offset=0, len=0; + cJSON* json=NULL, *tmp=NULL; + char* copy_line=NULL, *ip_plugin_info=NULL; + struct ip_plugin_table_schema* ip_plugin_schema=&(p->ip_plugin); + copy_line=_maat_strdup(line); + ret=get_column_pos(copy_line, COLUMN_IP_PLUGIN_SCHEMA_JSON, &offset, &len); + if(ret<0) + { + goto error_out; + } + if(offset+lentype==cJSON_Number) + { + ip_plugin_schema->row_id_column=tmp->valueint; + read_cnt++; + } + + tmp=cJSON_GetObjectItem(json, "ip_type"); + if(tmp!=NULL && tmp->type==cJSON_Number) + { + ip_plugin_schema->ip_type_column=tmp->valueint; + read_cnt++; + } + tmp=cJSON_GetObjectItem(json, "start_ip"); + if(tmp!=NULL && tmp->type==cJSON_Number) + { + ip_plugin_schema->start_ip_column=tmp->valueint; + read_cnt++; + } + tmp=cJSON_GetObjectItem(json, "end_ip"); + if(tmp!=NULL && tmp->type==cJSON_Number) + { + ip_plugin_schema->end_ip_column=tmp->valueint; + read_cnt++; + } + + tmp=cJSON_GetObjectItem(json, "valid"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + ip_plugin_schema->valid_flag_column=tmp->valueint; + read_cnt++; + } + ip_plugin_schema->rule_tag_column=-1; + tmp=cJSON_GetObjectItem(json, "tag"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + ip_plugin_schema->rule_tag_column=tmp->valueint; + //read_cnt++; Tag is optional, so NOT ++ intentionally. + } + + ip_plugin_schema->estimate_size=4096; + tmp=cJSON_GetObjectItem(json, "estimate_size"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + ip_plugin_schema->estimate_size=tmp->valueint; + //read_cnt++; estimate_size is optional, so NOT ++ intentionally. + } + + cJSON_Delete(json); + + free(copy_line); + if(read_cnt<5) + { + return -1; + } + else + { + return 0; + } +error_out: + free(copy_line); + return -1; + +} int read_composition_table_schema(const char* line, struct Maat_table_schema* p, MESA_htable_handle string2int_map) { @@ -397,6 +490,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path map_register(string2int_map,"ip_plus", TABLE_TYPE_IP_PLUS); map_register(string2int_map,"compile", TABLE_TYPE_COMPILE); map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN); + map_register(string2int_map,"ip_plugin", TABLE_TYPE_IP_PLUGIN); map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL); map_register(string2int_map,"digest", TABLE_TYPE_DIGEST); map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS); @@ -468,7 +562,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path } break; case TABLE_TYPE_PLUGIN: - ret=read_plugin_table_description(line, p); + ret=read_plugin_table_schema(line, p); if(ret<0) { fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n", table_info_path,i); @@ -477,6 +571,16 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path goto invalid_table; } break; + case TABLE_TYPE_IP_PLUGIN: + ret=read_ip_plugin_table_schema(line, p); + if(ret<0) + { + fprintf(stderr,"Maat read table info %s line %d error:illegal ip_plugin info.\n", table_info_path,i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:illegal ip_plugin info.", table_info_path,i); + goto invalid_table; + } + break; case TABLE_TYPE_COMPOSITION: ret=read_composition_table_schema(line, p, string2int_map); if(ret<0) @@ -488,7 +592,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path } break; case TABLE_TYPE_VIRTUAL: - ret=read_virtual_table_info(line, p, string2int_map); + ret=read_virtual_table_schema(line, p, string2int_map); if(ret<0) { fprintf(stderr,"Maat read table info %s line %d error:illegal virtual info.\n", table_info_path,i); @@ -765,32 +869,89 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c return idx; } -int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id, +void Maat_table_EX_data_schema_set(struct EX_data_schema* ex_schema, Maat_plugin_EX_new_func_t* new_func, Maat_plugin_EX_free_func_t* free_func, Maat_plugin_EX_dup_func_t* dup_func, Maat_plugin_EX_key2index_func_t* key2index_func, long argl, void *argp) - { - struct Maat_table_schema *table_desc=NULL;; - table_desc=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); - struct plugin_table_schema* plugin_desc=&(table_desc->plugin); - - if(plugin_desc->have_exdata - || plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0) + ex_schema->new_func=new_func; + ex_schema->free_func=free_func; + ex_schema->dup_func=dup_func; + ex_schema->key2index_func=key2index_func;//Set but not used. + ex_schema->argl=argl; + ex_schema->argp=argp; + return; +} +int Maat_table_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp, + void* logger) +{ + if(new_func==NULL || free_func==NULL || dup_func==NULL ) { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); return -1; } - plugin_desc->ex_desc.new_func=new_func; - plugin_desc->ex_desc.free_func=free_func; - plugin_desc->ex_desc.dup_func=dup_func; - plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used. - plugin_desc->ex_desc.argl=argl; - plugin_desc->ex_desc.argp=argp; - plugin_desc->have_exdata=1; + if(table_schema->table_type!=TABLE_TYPE_PLUGIN) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, Regist target is not a plugin table.", __FUNCTION__); + return -1; + } + if(table_schema->plugin.have_exdata) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__); + return -1; + } + if(table_schema->plugin.key_column==0 || table_schema->plugin.valid_flag_column==0) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, not enough schema information.", __FUNCTION__); + return -1; + } + Maat_table_EX_data_schema_set(&table_schema->plugin.ex_schema, + new_func, free_func, dup_func, key2index_func, argl, argp); + table_schema->plugin.have_exdata=1; return 0; } +int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp, + void* logger) +{ + if(new_func==NULL || free_func==NULL || dup_func==NULL ) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); + return -1; + } + if(table_schema->table_type!=TABLE_TYPE_IP_PLUGIN) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a ip_plugin table.", __FUNCTION__); + return -1; + } + if(table_schema->ip_plugin.have_exdata) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__); + return -1; + } + Maat_table_EX_data_schema_set(&table_schema->ip_plugin.ex_schema, + new_func, free_func, dup_func, key2index_func, argl, argp); + table_schema->ip_plugin.have_exdata=1; + return 0; +} void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type) { table_mgr->active_plugin_table_num=0; diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp index d145e31..accdc83 100644 --- a/src/entry/Maat_table_runtime.cpp +++ b/src/entry/Maat_table_runtime.cpp @@ -5,114 +5,79 @@ #include #include #include +#include -struct wrap_plugin_EX_data -{ - MAAT_RULE_EX_DATA exdata; - const struct Maat_table_schema* ref_plugin_table; -}; -void wrap_plugin_EX_data_free(void *data) -{ - struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data; - const struct plugin_table_ex_data_schema* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc); - ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp); - wrap_data->ref_plugin_table=NULL; - free(wrap_data); - return; -} -MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index) -{ - MESA_htable_handle key2ex_hash=NULL; - unsigned int slot_size=1; - while(estimate_size!=0) - { - estimate_size=estimate_size>>1; - slot_size*=2; - } - if(slot_size==1) - { - slot_size=4096; - } - MESA_htable_create_args_t hargs; - memset(&hargs,0,sizeof(hargs)); - hargs.thread_safe=8; - hargs.hash_slot_size = slot_size; - hargs.max_elem_num = 0; - hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; - hargs.expire_time = 0; - hargs.key_comp = NULL; - hargs.key2index = NULL; //Not supported yet. - hargs.recursive = 1; - hargs.data_free = wrap_plugin_EX_data_free; - hargs.data_expire_with_condition = NULL; - key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); - MESA_htable_print_crtl(key2ex_hash, 0); - return key2ex_hash; -} - -int plugin_EX_data_free(const struct Maat_table_schema* plugin_table, const char* line, +int plugin_EX_data_free(const char* row, int key_column, MESA_htable_handle key2ex_hash, void *logger) { size_t key_offset=0, key_len=0; - const struct plugin_table_schema* plugin_desc= &(plugin_table->plugin); int ret=0; - ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + ret=get_column_pos(row, key_column, &key_offset, &key_len); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data del error: cannot find column %d of %s", - plugin_desc->key_column, line); + "plugin/ip_plugin EX data del error: cannot find column %d of %s", + key_column, row); return -1; } - ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL); + ret=MESA_htable_del(key2ex_hash, (const unsigned char*)row+key_offset, key_len, NULL); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data del error: no such key %.*s of %s", - key_len, line+key_offset, line); + "plugin/ip_plugin EX data del error: no such key %.*s of %s", + key_len, row+key_offset, row); return -1; } return 0; } - -int plugin_EX_data_new(const struct Maat_table_schema* plugin_table, const char* line, - MESA_htable_handle key2ex_hash, void *logger) +struct ip_rule* ip_plugin_row2ip_rule(const struct ip_plugin_table_schema* schema, const char* row) { - char* key=NULL; - size_t key_offset=0, key_len=0; - MAAT_RULE_EX_DATA exdata=NULL; - struct wrap_plugin_EX_data* wrap_data=NULL; - const struct plugin_table_schema* plugin_desc= &(plugin_table->plugin); - int ret=0; - ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data add error: cannot find column %d of %s", - plugin_desc->key_column, line); - return -1; - } - key=ALLOC(char, key_len+1); - memcpy(key, line+key_offset, key_len); - plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata, - plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); - wrap_data=ALLOC(struct wrap_plugin_EX_data, 1); - wrap_data->exdata=exdata; - wrap_data->ref_plugin_table=plugin_table; - ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data); - free(key); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data add error: duplicated key %.*s of %s", - key_len, line+key_offset, line); - wrap_plugin_EX_data_free(wrap_data); - return -1; - } - return 0; -} + struct ip_rule* range_rule=ALLOC(struct ip_rule, 1); + int ret[4]={0}; + size_t column_offset=0, column_len=0; + char start_ip[128]={0}, end_ip[128]={0}; + ret[0]=get_column_pos(row, schema->row_id_column, &column_offset, &column_len); + range_rule->rule_id=atoi(row+column_offset); + + ret[1]=get_column_pos(row, schema->ip_type_column, &column_offset, &column_len); + int ip_type=atoi(row+column_offset); + + ret[2]=get_column_pos(row, schema->start_ip_column, &column_offset, &column_len); + strncpy(start_ip, row+column_offset, MIN(column_len, sizeof(start_ip))); + + ret[3]=get_column_pos(row, schema->end_ip_column, &column_offset, &column_len); + strncpy(end_ip, row+column_offset, MIN(column_len, sizeof(end_ip))); + if(ret[0]<0||ret[1]<0||ret[2]<0||ret[3]<0) + { + free(range_rule); + return NULL; + } + + if(ip_type==4) + { + ret[0]=ip_format2range(ip_type, FORMAT_RANGE, start_ip, end_ip, &(range_rule->ipv4_rule.start_ip), &(range_rule->ipv4_rule.end_ip)); + range_rule->type=IPv4; + } + else if(ip_type==6) + { + ret[0]=ip_format2range(ip_type, FORMAT_RANGE, start_ip, end_ip, range_rule->ipv6_rule.start_ip, range_rule->ipv6_rule.end_ip); + range_rule->type=IPv6; + } + else + { + free(range_rule); + return NULL; + } + if(ret[0]<0) + { + free(range_rule); + return NULL; + } + range_rule->user_tag=NULL; + return range_rule; +} struct Maat_table_runtime_manager { struct Maat_table_runtime** table_rt; @@ -149,23 +114,35 @@ static void destroy_digest_rule(GIE_digest_t*rule) return; } -static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_desc, int max_thread_num) +static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num) { struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1); - table_rt->table_type=table_desc->table_type; - switch(table_desc->table_type) + table_rt->table_type=table_schema->table_type; + switch(table_schema->table_type) { case TABLE_TYPE_DIGEST: case TABLE_TYPE_SIMILARITY: table_rt->similar.update_q=MESA_lqueue_create(0,0); break; case TABLE_TYPE_PLUGIN: - table_rt->plugin.cache_lines=dynamic_array_create(1, 1024); - if(table_desc->plugin.have_exdata) + table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, + table_schema->plugin.estimate_size, + table_schema->plugin.ex_schema.key2index_func, + NULL); + if(table_schema->plugin.have_exdata) { - table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, - table_desc->plugin.ex_desc.key2index_func); + EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema); + } + break; + case TABLE_TYPE_IP_PLUGIN: + table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, + table_schema->plugin.estimate_size, + table_schema->plugin.ex_schema.key2index_func, + free); + if(table_schema->ip_plugin.have_exdata) + { + EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema); } break; default: @@ -211,13 +188,15 @@ static void table_runtime_free(struct Maat_table_runtime* p) MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL); } break; + + case TABLE_TYPE_IP_PLUGIN: + ip_matcher_free(p->ip_plugin.ip_matcher); + EX_data_rt_free(p->ip_plugin.ex_data_rt); + assert(p->ip_plugin.old_ip_matcher==NULL); + break; case TABLE_TYPE_PLUGIN: - dynamic_array_destroy(p->plugin.cache_lines, free); - p->plugin.cache_lines=NULL; - if(p->plugin.key2ex_hash!=NULL) - { - MESA_htable_destroy(p->plugin.key2ex_hash, NULL); - } + EX_data_rt_free(p->plugin.ex_data_rt); + break; default: break; } @@ -271,66 +250,84 @@ struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_mana long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt) { struct plugin_runtime* plugin_rt=&(table_rt->plugin); - return plugin_rt->cache_line_num; + return plugin_rt->ex_data_rt->cache_row_num; } const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line) { const char* line=NULL; struct plugin_runtime* plugin_rt=&(table_rt->plugin); - line=(const char*)dynamic_array_read(plugin_rt->cache_lines, Nth_line); + line=(const char*)dynamic_array_read(plugin_rt->ex_data_rt->cache_rows, Nth_line); return line; } -MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_schema* plugin_table, - struct dynamic_array_t* lines, size_t line_cnt, void* logger) +int Maat_table_runtime_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger) { - MESA_htable_handle key2ex_hash=NULL; - size_t i=0; - const char* line=NULL; - const struct plugin_table_schema* plugin_desc= &(plugin_table->plugin); + int i=0; + const char* row=NULL; + EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema); - key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func); - - for(i=0; i< line_cnt; i++) + for(i=0; iplugin.ex_data_rt); i++) { - line=(const char*)dynamic_array_read(lines, i); - plugin_EX_data_new(plugin_table, line, key2ex_hash, logger); + row=EX_data_rt_get_cached_row(table_rt->plugin.ex_data_rt, i); + Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger); } - return key2ex_hash; -} -int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, void* logger) -{ - assert(table_rt->plugin.key2ex_hash==NULL); - if(table_rt->plugin.key2ex_hash) - { - return -1; - } - table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines, - table_rt->plugin.cache_line_num, logger); return 0; } -MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, const char* key) +MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* key) { - struct plugin_table_schema* plugin_desc=NULL; - struct wrap_plugin_EX_data* wrap_data=NULL; - MAAT_RULE_EX_DATA exdata=NULL; - - plugin_desc=&(table_desc->plugin); - if(!plugin_desc->have_exdata) + MAAT_RULE_EX_DATA ex_data=NULL; + if(!table_schema->plugin.have_exdata) { assert(0); return NULL; } - wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash, - (const unsigned char*)key, strlen(key)); - if(wrap_data!=NULL) - { - plugin_desc->ex_desc.dup_func(table_desc->table_id, &(exdata), &(wrap_data->exdata), - plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); - } - return exdata; + ex_data=EX_data_rt_get_EX_data_by_key(table_rt->plugin.ex_data_rt, key, strlen(key)); + return ex_data; } +void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) +{ + int ret=0, i=0; + size_t is_valid_offset=0, valid_len=0; + size_t key_offset=0, key_len=0; + + struct plugin_table_schema* plugin_schema=&table_schema->plugin; + struct plugin_runtime* plugin_rt=&table_rt->plugin; + ret=Maat_helper_read_column(row, plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); + plugin_rt->acc_line_num++; + if(plugin_schema->have_exdata) + { + ret=get_column_pos(row, plugin_schema->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "plugin EX data process error: cannot find column %d of %s", + plugin_schema->key_column, row); + return; + } + if(atoi(row+is_valid_offset)==1) + { + EX_data_rt_row2EX_data(plugin_rt->ex_data_rt, row, row+key_offset, key_len, NULL, logger); + } + else + { + EX_data_rt_delete_by_row(plugin_rt->ex_data_rt, row, row+key_offset, key_len, logger); + } + } + if(plugin_schema->cb_plug_cnt>0) + { + for(i=0; icb_plug_cnt;i++) + { + plugin_schema->cb_plug[i].update(table_schema->table_id, row, plugin_schema->cb_plug[i].u_para); + } + } + if(!plugin_schema->have_exdata && !plugin_schema->cb_plug_cnt) + { + EX_data_rt_cache_row(plugin_rt->ex_data_rt, row); + } + + return; +} void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag) { @@ -424,4 +421,117 @@ int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt) return q_cnt; } +int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt) +{ + struct ip_matcher* new_ip_matcher=NULL; + struct EX_data_container_q queue;//This is for index, no need to free. + size_t rule_cnt=0; + size_t i=0, mem_use=0; + struct ip_rule *rules=NULL; + struct EX_data_container *p=NULL; + TAILQ_INIT(&queue); + rule_cnt=EX_data_rt_list_all(table_rt->ip_plugin.ex_data_rt, &queue); + rules=ALLOC(struct ip_rule, rule_cnt); + TAILQ_FOREACH(p, &queue, entries) + { + rules[i]=*((struct ip_rule *)(p->user_data)); + assert(rules[i].user_tag==p||rules[i].user_tag==NULL); + rules[i].user_tag=p; + i++; + } + assert(i==rule_cnt); + if(rule_cnt==0) + { + return 0; + } + new_ip_matcher=ip_matcher_new(rules, rule_cnt, &mem_use); + table_rt->ip_plugin.old_ip_matcher=table_rt->ip_plugin.ip_matcher; + table_rt->ip_plugin.ip_matcher=new_ip_matcher; + return 0; +} +struct ip_matcher* Maat_table_runtime_dettach_old_ip_matcher(struct Maat_table_runtime* table_rt) +{ + struct ip_matcher* old_one=table_rt->ip_plugin.old_ip_matcher; + assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN); + table_rt->ip_plugin.old_ip_matcher=NULL; + return old_one; +} +void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) +{ + struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin); + struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin); + size_t is_valid_offset=0, valid_len=0; + size_t key_offset=0, key_len=0; + struct ip_rule* ip_rule=NULL; + int ret=0; + + if(ip_plugin_schema->have_exdata) + { + ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "ip_plugin EX data process error: cannot find is_valid column %d of %s", + ip_plugin_schema->row_id_column, row); + return; + } + ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "ip_plugin EX data process error: cannot find row id column %d of %s", + ip_plugin_schema->row_id_column, row); + return; + } + ip_rule=ip_plugin_row2ip_rule(ip_plugin_schema, row); + if(ip_rule==NULL) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "ip_plugin read ip error: %s", row); + return; + + } + if(atoi(row+is_valid_offset)==1)//add + { + EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+key_offset, key_len, ip_rule, logger); + } + else + { + EX_data_rt_delete_by_row(ip_plugin_rt->ex_data_rt, row, row+key_offset, key_len, logger); + } + } + else + { + EX_data_rt_cache_row(ip_plugin_rt->ex_data_rt, row); + } + return; +} +int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger) +{ + int i=0; + const char* row=NULL; + EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema); + for(i=0; iplugin.ex_data_rt); i++) + { + row=EX_data_rt_get_cached_row(table_rt->plugin.ex_data_rt, i); + Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, row, logger); + } + EX_data_rt_clear_row_cache(table_rt->plugin.ex_data_rt); + Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt); + + return 0; +} + +int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size) +{ + struct scan_result results[size]; + int n_result=0, i=0; + n_result=ip_matcher_match(table_rt->ip_plugin.ip_matcher, (struct ip_data*)ip, results, size); + for(i=0; iip_plugin.ex_data_rt, (struct EX_data_container *)results[i].tag); + } + return n_result; +} + diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 8462ba7..745952a 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -390,4 +390,133 @@ int decrypt_open(const char* file_name, const char* key, const char* algorithm, file_buff=NULL; return ret; } +enum MAAT_IP_FORMAT ip_format_str2int(const char* format) +{ + if(0==strcasecmp(format, "range")) + { + return FORMAT_RANGE; + } + else if(0==strcasecmp(format, "mask")) + { + return FORMAT_MASK; + } + else if(0==strcasecmp(format, "CIDR")) + { + return FORMAT_CIDR; + } + else + { + assert(0); + } + return FORMAT_UNKNOWN; +} +int ip_format2range(int ip_type, enum MAAT_IP_FORMAT format, const char* ip1, const char* ip2, unsigned int range_begin[], unsigned int range_end[]) +{ + unsigned int ipv4_addr=0, ipv4_mask=0, ipv4_range_end=0; + unsigned int ipv6_addr[4]={0}, ipv6_mask[4]={0}, ipv6_range_end[4]={0}; + int cidr=0, bit32=0; + int ret=0, i=0; + if(ip_type!=4 && ip_type!=6) + { + assert(0); + return -1; + } + if(ip_type==4) + { + ret=inet_pton(AF_INET, ip1, &ipv4_addr); + if(ret<=0) + { + return -1; + } + ipv4_addr=ntohl(ipv4_addr); + switch (format) + { + case FORMAT_RANGE: + range_begin[0]=ipv4_addr; + ret=inet_pton(AF_INET, ip2, &ipv4_range_end); + if(ret<=0) + { + return -1; + } + ipv4_range_end=ntohl(ipv4_range_end); + range_end[0]=ipv4_range_end; + break; + case FORMAT_MASK: + ret=inet_pton(AF_INET, ip2, &ipv4_mask); + if(ret<=0) + { + return -1; + } + ipv4_mask=ntohl(ipv4_mask); + range_begin[0]=ipv4_addr&ipv4_mask; + range_end[0]=ipv4_addr|~ipv4_mask; + break; + case FORMAT_CIDR: + cidr=atoi(ip2); + if(cidr>32||cidr<0) + { + return -1; + } + ipv4_mask = (0xFFFFFFFFUL << (32 - cidr)) & 0xFFFFFFFFUL; + range_begin[0]=ipv4_addr&ipv4_mask; + range_end[0]=ipv4_addr|~ipv4_mask; + break; + default: + assert(0); + } + } + else //ipv6 + { + ret=inet_pton(AF_INET6, ip1, ipv6_addr); + if(ret<=0) + { + return -1; + } + ipv6_ntoh(ipv6_addr); + switch(format) + { + case FORMAT_RANGE: + ret=inet_pton(AF_INET6, ip2, ipv6_range_end); + if(ret<=0) + { + return -1; + } + ipv6_ntoh(ipv6_range_end); + memcpy(range_begin, ipv6_addr, sizeof(ipv6_addr)); + memcpy(range_end, ipv6_range_end, sizeof(ipv6_range_end)); + break; + case FORMAT_MASK: + ret=inet_pton(AF_INET6, ip2, ipv6_mask); + if(ret<=0) + { + return -1; + } + ipv6_ntoh(ipv6_mask); + for(i=0; i<4; i++) + { + range_begin[i]=ipv6_addr[i]&ipv6_mask[i]; + range_end[i] = ipv6_addr[i]|~ipv6_mask[i]; + } + break; + case FORMAT_CIDR: + cidr=atoi(ip2); + if(cidr>128||cidr<0) + { + return -1; + } + for(i=0; i<4; i++) + { + bit32=128-cidr-32*(3-i); + if(bit32<0) bit32=0; + ipv6_mask[i]=(0xFFFFFFFFUL << bit32) & 0xFFFFFFFFUL; + range_begin[i]=ipv6_addr[i]&ipv6_mask[i]; + range_end[i] = ipv6_addr[i]|~ipv6_mask[i]; + } + break; + default: + assert(0); + } + } + return 0; +} diff --git a/src/inc_internal/Maat_ex_data.h b/src/inc_internal/Maat_ex_data.h new file mode 100644 index 0000000..47d7e7f --- /dev/null +++ b/src/inc_internal/Maat_ex_data.h @@ -0,0 +1,45 @@ + +#include "dynamic_array.h" +#include "Maat_rule.h" +#include + +#include + +struct EX_data_rt +{ + dynamic_array_t *cache_rows; + long long cache_row_num; + long long cache_size; + MESA_htable_handle key2ex_hash; + const struct EX_data_schema* ex_schema; + int table_id; + void (* user_data_free)(void *user_data); +}; +struct EX_data_container +{ + MAAT_RULE_EX_DATA ex_data; + const struct EX_data_rt* rt; + void* user_data; + TAILQ_ENTRY(EX_data_container) entries; +}; + +TAILQ_HEAD(EX_data_container_q, EX_data_container); +struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)); +void EX_data_rt_free(struct EX_data_rt* p); +void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema); +void EX_data_rt_cache_row(struct EX_data_rt* p, const char* row); + +const char* EX_data_rt_get_cached_row(struct EX_data_rt* p, int i); + +void EX_data_rt_clear_row_cache(struct EX_data_rt* p); +int EX_data_rt_get_row_num(struct EX_data_rt* p); + +struct EX_data_container* EX_data_rt_row2EX_data(struct EX_data_rt* ex_rt, + const char* row, const char* key, size_t key_len, + void* user_data, void* logger); + +int EX_data_rt_delete_by_row(struct EX_data_rt* ex_rt, const char* row, const char* key, size_t key_len, void *logger); +MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const char* key, size_t key_len); +MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container); +size_t EX_data_rt_list_all(struct EX_data_rt* ex_rt, EX_data_container_q* listed); + diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 57772e0..4a0d04c 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -13,6 +13,7 @@ #include "dynamic_array.h" #include "bool_matcher.h" #include "hiredis.h" +#include "IPMatcher.h" #include "igraph/igraph.h" #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" @@ -204,7 +205,8 @@ enum maat_garbage_type GARBAGE_COMPILE_GOURP_RELATION, GARBAGE_BOOL_MATCHER, GARBAGE_MAP_STR2INT, - GARBAGE_FOREIGN_FILE + GARBAGE_FOREIGN_FILE, + GARBAGE_IP_MATCHER }; struct iconv_handle_t { @@ -402,6 +404,7 @@ struct _maat_garbage_t struct Maat_compile_rule* compile_rule; struct Maat_compile_group_relation * compile_group_relation; struct bool_matcher* bool_matcher; + struct ip_matcher* a_ip_matcher; void * raw; MESA_htable_handle str2int_map; char* filename; @@ -454,9 +457,6 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p); void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len); MAAT_RULE_EX_DATA rule_ex_data_new(const struct Maat_rule_head * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc); void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc); -MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index); -int plugin_EX_data_new(const struct Maat_table_schema* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger); -int plugin_EX_data_free(const struct Maat_table_schema* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger); void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout); diff --git a/src/inc_internal/Maat_table.h b/src/inc_internal/Maat_table.h index 3e14f98..3249b7b 100644 --- a/src/inc_internal/Maat_table.h +++ b/src/inc_internal/Maat_table.h @@ -32,7 +32,8 @@ enum MAAT_TABLE_TYPE TABLE_TYPE_COMPOSITION, TABLE_TYPE_GROUP, TABLE_TYPE_COMPILE, - TABLE_TYPE_PLUGIN + TABLE_TYPE_PLUGIN, + TABLE_TYPE_IP_PLUGIN }; struct compile_ex_data_idx @@ -78,7 +79,7 @@ struct plugin_table_callback_schema Maat_finish_callback_t *finish; void* u_para; }; -struct plugin_table_ex_data_schema +struct EX_data_schema { Maat_plugin_EX_new_func_t* new_func; Maat_plugin_EX_free_func_t* free_func; @@ -98,9 +99,20 @@ struct plugin_table_schema int have_exdata; long long estimate_size; struct plugin_table_callback_schema cb_plug[MAX_PLUGIN_PER_TABLE]; - struct plugin_table_ex_data_schema ex_desc; + struct EX_data_schema ex_schema; +}; +struct ip_plugin_table_schema +{ + int row_id_column; + int ip_type_column; + int start_ip_column; + int end_ip_column; + int valid_flag_column; + int rule_tag_column; + long long estimate_size; + int have_exdata; + struct EX_data_schema ex_schema; }; - struct Maat_table_schema { int table_id; @@ -113,9 +125,10 @@ struct Maat_table_schema struct compile_table_schema compile; struct expr_table_schema expr; struct plugin_table_schema plugin; + struct ip_plugin_table_schema ip_plugin; struct virtual_table_schema virtual_table; struct composition_table_schema composition; - void* others;//group, ip, interval and digest don't have sperate description info. + void* others;//group, ip, interval and digest don't have any special schema. }; //for stat>>>>>>>> unsigned long long udpate_err_cnt; @@ -149,12 +162,21 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c Maat_rule_EX_dup_func_t* dup_func, long argl, void *argp); struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx); -int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id, +int Maat_table_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, Maat_plugin_EX_new_func_t* new_func, Maat_plugin_EX_free_func_t* free_func, Maat_plugin_EX_dup_func_t* dup_func, Maat_plugin_EX_key2index_func_t* key2index_func, - long argl, void *argp); + long argl, void *argp, + void* logger); +int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp, + void* logger); + void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type); void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr); diff --git a/src/inc_internal/Maat_table_runtime.h b/src/inc_internal/Maat_table_runtime.h index acd1a69..936e7c1 100644 --- a/src/inc_internal/Maat_table_runtime.h +++ b/src/inc_internal/Maat_table_runtime.h @@ -1,4 +1,6 @@ #include "Maat_table.h" +#include "Maat_ex_data.h" +#include "IPMatcher.h" #include "gram_index_engine.h" #include "alignment_int64.h" #include "dynamic_array.h" @@ -13,11 +15,15 @@ struct similar_runtime struct plugin_runtime { - dynamic_array_t *cache_lines; - long long cache_line_num; + struct EX_data_rt* ex_data_rt; long long acc_line_num; - long long cache_size; - MESA_htable_handle key2ex_hash; +}; + +struct ip_plugin_runtime +{ + struct EX_data_rt* ex_data_rt; + struct ip_matcher* ip_matcher; + struct ip_matcher* old_ip_matcher; }; struct expr_runtime { @@ -42,6 +48,7 @@ struct Maat_table_runtime { struct similar_runtime similar; //for digest and similarity struct plugin_runtime plugin; + struct ip_plugin_runtime ip_plugin; struct expr_runtime expr; struct ip_runtime ip; struct group_runtime group; @@ -60,10 +67,18 @@ struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_mana long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt); const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line); -int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, void* logger); +int Maat_table_runtime_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, void* logger); MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, const char* key); void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag); void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id); int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt); +void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); +void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); + +int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger); +int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size); +int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt); +struct ip_matcher* Maat_table_runtime_dettach_old_ip_matcher(struct Maat_table_runtime* table_rt); + diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index ac4054b..4d8b63c 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -82,5 +82,14 @@ int decrypt_open(const char* file_name, const char* key, const char* algorithm, int load_file_to_memory(const char* file_name, unsigned char**pp_out, size_t *out_sz); //do_encrypt: 1 for encryption, 0 for decryption. int crypt_memory(const unsigned char* inbuf, size_t inlen, unsigned char** pp_out, size_t *out_sz, const char* key, const char* algorithm, int do_encrypt, char* err_str, size_t err_str_sz); +enum MAAT_IP_FORMAT +{ + FORMAT_RANGE, + FORMAT_MASK, + FORMAT_CIDR, + FORMAT_UNKNOWN +}; +enum MAAT_IP_FORMAT ip_format_str2int(const char* format); +int ip_format2range(int ip_type, enum MAAT_IP_FORMAT format, const char* ip1, const char* ip2, unsigned int range_begin[], unsigned int range_end[]); diff --git a/src/inc_internal/view_only/IPMatcher.h b/src/inc_internal/view_only/IPMatcher.h new file mode 100644 index 0000000..0e387f3 --- /dev/null +++ b/src/inc_internal/view_only/IPMatcher.h @@ -0,0 +1,117 @@ +/* + * + * Copyright (c) 2020 + * String Algorithms Research Group + * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) + * National Engineering Laboratory for Information Security Technologies (NELIST) + * All rights reserved + * + * Written by: LU YUHAI (luyuhai@iie.ac.cn) + * Last modification: 2020-04-20 + * + * This code is the exclusive and proprietary property of IIE-CAS and NELIST. + * Usage for direct or indirect commercial advantage is not allowed without + * written permission from the authors. + * + */ + +#ifndef H_IP_MATCHER_H +#define H_IP_MATCHER_H +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + + enum IP_TYPE + { + IPv4, + IPv6 + }; + + /* 带掩码的单点IPv4规则 */ + struct ipv4_range + { + unsigned int start_ip; /* IP范围下界 */ + unsigned int end_ip; /* IP范围上界 */ + }; + + /* 带掩码的单点IPv6规则 */ + struct ipv6_range + { + unsigned int start_ip[4]; /* IP范围下界 */ + unsigned int end_ip[4]; /* IP范围上界 */ + }; + + /* 通用的ip规则类型 */ + struct ip_rule + { + enum IP_TYPE type; /* 规则类型，ipv4或ipv6 */ + unsigned int rule_id; /* 规则ID */ + void* user_tag; /* 用户自定义数据，命中时随匹配结果返回 */ + union + { + struct ipv4_range ipv4_rule; /*带掩码的单点IPv4规则*/ + struct ipv6_range ipv6_rule; /*带掩码的单点IPv6规则*/ + }; + }; + + /* 通用的待扫描数据类型 */ + struct ip_data + { + enum IP_TYPE type; /* 规则类型，ipv4或ipv6 */ + union /* 根据rule_type决定数据负载是ipv4还是ipv6 */ + { + unsigned int ipv4; /* ipv4数据*/ + unsigned int ipv6[4]; /* ipv6数据*/ + }; + }; + + + /* 布尔表达式的扫描结果类型 */ + struct scan_result + { + unsigned int rule_id; /* 规则的ID */ + void * tag; /* 用户自定义数据，命中时随匹配结果返回 */ + }; + + + struct ip_matcher; + + /* + 功能：根据输入的规则生成扫描器 + 参数： + rules[in]：一组ip规则 + rule_num[in]：输入的规则数量 + mem_use[out]：内存消耗量 + 返回值： + ip扫描器,返回空指针生成扫描器失败 + */ + struct ip_matcher* ip_matcher_new(struct ip_rule * rules, size_t rule_num, size_t * mem_use); + + /* + 功能：调用ip扫描器对输入的ip数据进行扫描 + 参数： + matcher[in]：ip扫描器 + data[in]：输入的待扫描ip数据 + result[in]：返回结果存储数组 + size[in]：结果数组的大小 + 返回值： + 命中结果的数量（<=size）；返回值为-1表示出错。 + + */ + int ip_matcher_match(struct ip_matcher* matcher, struct ip_data * data, struct scan_result* result, size_t size); + + /* + 功能：销毁一个ip扫描器 + 参数： + matcher[in]：待销毁的ip扫描器指针 + */ + void ip_matcher_free(struct ip_matcher* matcher); + +#ifdef __cplusplus +} +#endif + +#endif /* !defined(H_IP_MATCHER_H) */ diff --git a/test/maat_json.json b/test/maat_json.json index 00e00bb..5692650 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -1754,6 +1754,15 @@ "1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34", "161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42" ] + }, + { + "table_name": "TEST_IP_PLUGIN_WITH_EXDATA", + "table_content": [ + "101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1", + "102\t4\t192.168.30.90\t192.168.30.128\tBigger-range-should-in-the-back\t1", + "103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1", + "104\t6\t2001:db8:1234::1\t2001:db8:1234::5210\tSomething-like-json\t1" + ] } ] } diff --git a/test/table_info.conf b/test/table_info.conf index 82e6924..f577ffb 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -47,4 +47,5 @@ 24 COMPOSITION_IP_SOURCE virtual IP_PLUS_CONFIG -- 25 COMPOSITION_IP_DESTINATION virtual IP_PLUS_CONFIG -- 26 COMPOSITION_IP_SESSION virtual IP_PLUS_CONFIG -- -27 COMPOSITION_IP composition {"source":"COMPOSITION_IP_SOURCE","destination":"COMPOSITION_IP_DESTINATION","session":"COMPOSITION_IP_SESSION"} \ No newline at end of file +27 COMPOSITION_IP composition {"source":"COMPOSITION_IP_SOURCE","destination":"COMPOSITION_IP_DESTINATION","session":"COMPOSITION_IP_SESSION"} +28 TEST_IP_PLUGIN_WITH_EXDATA ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":6} -- \ No newline at end of file diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 103f4fb..d17c940 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -101,7 +101,7 @@ void scan_with_old_or_new_cfg(Maat_feather_t feather, int is_old) if(!is_old) { EXPECT_EQ(ret, 1); - EXPECT_TRUE(result.config_id==2); + EXPECT_EQ(result.config_id, 2); } else { @@ -219,7 +219,7 @@ void test_plugin_table(Maat_feather_t feather,const char* table_name, ASSERT_GT(ret, 0); } - +#define Plugin_callback TEST(PluginTable, Callback) { test_plugin_table(g_feather, "QD_ENTRY_INFO", @@ -229,6 +229,85 @@ TEST(PluginTable, Callback) g_feather, g_logger); +} +#define IP_PLUGIN_EX_DATA +struct ip_plugin_ud +{ + int rule_id; + char* buffer; + int ref_cnt; +}; +void ip_plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) +{ + int *counter=(int *)argp, ret=0; + size_t column_offset=0, column_len=0; + struct ip_plugin_ud* ud=(struct ip_plugin_ud*)calloc(sizeof(struct ip_plugin_ud), 1); + ret=Maat_helper_read_column(table_line, 1, &column_offset, &column_len); + EXPECT_EQ(ret, 0); + ud->rule_id=atoi(table_line+column_offset); + ret=Maat_helper_read_column(table_line, 5, &column_offset, &column_len); + EXPECT_EQ(ret, 0); + ud->buffer=(char*)calloc(sizeof(char), column_len+1); + strncpy(ud->buffer, table_line+column_offset, column_len); + ud->ref_cnt=1; + *ad=ud; + (*counter)++; + return; +} +void ip_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) +{ + struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*ad); + u->ref_cnt--; + if(u->ref_cnt>0) return; + free(u->buffer); + free(u); + *ad=NULL; +} +void ip_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) +{ + struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*from); + u->ref_cnt++; + *to=u; +} + +TEST(IP_Plugin_Table, EX_DATA) +{ + + int ip_plugin_ex_data_counter=0, i=0; + const char* table_name="TEST_IP_PLUGIN_WITH_EXDATA"; + int table_id=0, ret=0; + table_id=Maat_table_register(g_feather, table_name); + ASSERT_GT(table_id, 0); + ret=Maat_ip_plugin_EX_register(g_feather, table_id, + ip_plugin_EX_new_cb, + ip_plugin_EX_free_cb, + ip_plugin_EX_dup_cb, + 0, &ip_plugin_ex_data_counter); + ASSERT_TRUE(ret>=0); + EXPECT_EQ(ip_plugin_ex_data_counter, 4); + struct ip_address ipv4, ipv6; + struct ip_plugin_ud* result[4]; + ipv4.ip_type=4; + inet_pton(AF_INET, "192.168.30.100", &(ipv4.ipv4)); + ret=Maat_ip_plugin_get_EX_data(g_feather, table_id, &ipv4, (void**)result, 4); + ASSERT_EQ(ret, 2); + EXPECT_EQ(result[0]->rule_id, 101); + EXPECT_EQ(result[1]->rule_id, 102); + for(i=0; irule_id, 104); + EXPECT_EQ(result[1]->rule_id, 103); + for(i=0; i install + BUILD_IN_SOURCE 1) + +ExternalProject_Get_Property(ipmatcher INSTALL_DIR) +file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + +add_library(ipmatcher-static STATIC IMPORTED GLOBAL) +add_dependencies(ipmatcher-static ipmatcher) +set_property(TARGET ipmatcher-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/ipmatcher.a) +set_property(TARGET ipmatcher-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) diff --git a/vendor/IPMatcher-20200513.zip b/vendor/IPMatcher-20200513.zip new file mode 100644 index 0000000..5b9ce89 Binary files /dev/null and b/vendor/IPMatcher-20200513.zip differ