diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..3c60a47 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required (VERSION 3.5) +project (maatframe) +set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} -Wall) +include_directories(${PROJECT_SOURCE_DIR}/inc/) +include_directories(/opt/MESA/include/MESA/) + +enable_testing() + +add_subdirectory (vendor) +add_subdirectory (src) +add_subdirectory (test) +add_subdirectory (tools) diff --git a/Makefile b/Makefile index 53a95f3..ac23531 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,41 @@ -#opt: OPTFLAGS = -O2 -#export OPTFLAGS +BUILD_DIR = $(CURDIR)/build +LOCAL_DIR = $(CURDIR) +DEBUG_FLAGS = -DCMAKE_BUILD_TYPE=Debug +REL_FLAGS = -DCMAKE_BUILD_TYPE=RelWithDebInfo -.PHONY: all clean opt +ifneq ($(INSTALL_PREFIX),) +DEBUG_FLAGS += -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) +REL_FLAGS += -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) +endif -all: - cd src/entry/ && $(MAKE) - cd test && $(MAKE) - cd tools && $(MAKE) -clean: - cd src/entry/ && $(MAKE) clean - cd test && $(MAKE) clean - cd tools && $(MAKE) clean - -opt: - $(MAKE) all +all: _make_build_dir _compile_rel + +PHONY: all _make_build_dir _compile_debug _compile_rel _install \ + build_release build_debug install + +_make_build_dir: + mkdir -p $(BUILD_DIR) + +_compile_debug: + cd $(BUILD_DIR) && cmake $(LOCAL_DIR) $(DEBUG_FLAGS) && make + +_compile_rel: + cd $(BUILD_DIR) && cmake $(LOCAL_DIR) $(REL_FLAGS) && make + +_install: + cd $(BUILD_DIR) && make install +_package: + cd $(BUILD_DIR) && make package +_clean: + rm -rf $(BUILD_DIR) + +# Release Version, No Debug Symbol and Optimized with -O2 +release: _make_build_dir _compile_rel +# Debug Version, Optimized with -O0 +debug: _make_build_dir _compile_debug +# Install +install: _install +# Package +package: _package +# Clean +clean: _clean diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index ba9a209..61be910 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -6,7 +6,7 @@ * to reside in the heart) of the departed would reach the paradise of afterlife * successfully. * Author: zhengchao@iie.ac.cn,MESA -* Version 2018-07-27 huge service_define +* Version 2018-09-21 rule tags * NOTE: MUST compile with G++ * All right reserved by Institute of Infomation Engineering,Chinese Academic of Science 2014~2018 ********************************************************* @@ -156,7 +156,8 @@ enum MAAT_INIT_OPT MAAT_OPT_CUMULATIVE_UPDATE_OFF, //VALUE is NULL,SIZE is 0. Default: CUMMULATIVE UPDATE ON. MAAT_OPT_LOAD_VERSION_FROM, //VALUE is a long long, SIZE=sizeof(long long). Default: Load the Latest. Only valid in redis mode, and maybe failed for too old. //This option also disables background update. - MAAT_OPT_ENABLE_UPDATE //VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled. DEFAULT: Backgroud update is enabled. Runtime setting is allowed. + MAAT_OPT_ENABLE_UPDATE, //VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled. DEFAULT: Backgroud update is enabled. Runtime setting is allowed. + MAAT_OPT_ACCEPT_TAGS //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]} }; //return -1 if failed, return 0 on success; int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..9099334 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,36 @@ +cmake_minimum_required(VERSION 2.6) +project(maatframe) + +set(MAAT_FRAME_MAJOR_VERSION 2) +set(MAAT_FRAME_MINOR_VERSION 2) +set(MAAT_FRAME_PATCH_VERSION 20180923) +set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION}) + +message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") + +add_definitions(-fPIC) +set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/UniversalBoolMatch.cpp) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) +include_directories(/opt/MESA/include/MESA/) + +# Static Library Output +add_library(maat_frame_static STATIC ${MAAT_SRC}) +set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe) +set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) +target_include_directories(maat_frame_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/) +target_include_directories(maat_frame_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/hiredis) + +# Shared Library Output +add_library(maat_frame_shared SHARED ${MAAT_SRC}) +set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe) +set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) +set_target_properties(maat_frame_shared PROPERTIES VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}) +set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION}) +target_include_directories(maat_frame_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/) +target_include_directories(maat_frame_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/hiredis) +target_link_libraries(maat_frame_shared MESA_handle_logger MESA_htable pcre rulescan pthread m pcre MESA_field_stat2 crypto hiredis_vip) + +install(DIRECTORY ${PROJECT_SOURCE_DIR}/inc/ DESTINATION /opt/MESA/include/MESA/ FILES_MATCHING PATTERN "*.h") +install(TARGETS maat_frame_shared DESTINATION /opt/MESA/lib/) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index dae9a60..eec2285 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -666,6 +666,13 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo "Maat load version from %lld, stops backgroud update." ,_feather->load_version_from); break; + case MAAT_OPT_ACCEPT_TAGS: + _feather->n_tags=parse_accept_tag((const char*) value, &_feather->accept_tags, _feather->logger); + if(_feather->n_tags==0) + { + return -1; + } + break; default: return -1; } @@ -1219,20 +1226,20 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id switch(addr->addrtype) { case ADDR_TYPE_IPV4: - ip_scan_data.ipv4_data.saddr=ntohl(addr->paddr.v4->saddr); - ip_scan_data.ipv4_data.daddr=ntohl(addr->paddr.v4->daddr); - ip_scan_data.ipv4_data.sport=ntohs(addr->paddr.v4->source); - ip_scan_data.ipv4_data.dport=ntohs(addr->paddr.v4->dest); + ip_scan_data.ipv4_data.saddr=ntohl(addr->v4->saddr); + ip_scan_data.ipv4_data.daddr=ntohl(addr->v4->daddr); + ip_scan_data.ipv4_data.sport=ntohs(addr->v4->source); + ip_scan_data.ipv4_data.dport=ntohs(addr->v4->dest); ip_scan_data.ipv4_data.proto=proto; break; case ADDR_TYPE_IPV6: ip_scan_data.rule_type=RULETYPE_IPv6; - memcpy(ip_scan_data.ipv6_data.saddr,addr->paddr.v6->saddr,sizeof(ip_scan_data.ipv6_data.saddr)); + memcpy(ip_scan_data.ipv6_data.saddr,addr->v6->saddr,sizeof(ip_scan_data.ipv6_data.saddr)); ipv6_ntoh(ip_scan_data.ipv6_data.saddr); - memcpy(ip_scan_data.ipv6_data.daddr,addr->paddr.v6->daddr,sizeof(ip_scan_data.ipv6_data.daddr)); + memcpy(ip_scan_data.ipv6_data.daddr,addr->v6->daddr,sizeof(ip_scan_data.ipv6_data.daddr)); ipv6_ntoh(ip_scan_data.ipv6_data.daddr); - ip_scan_data.ipv6_data.sport=ntohs(addr->paddr.v6->source); - ip_scan_data.ipv6_data.dport=ntohs(addr->paddr.v6->dest); + ip_scan_data.ipv6_data.sport=ntohs(addr->v6->source); + ip_scan_data.ipv6_data.dport=ntohs(addr->v6->dest); ip_scan_data.ipv6_data.proto=proto; break; default: diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 28aeba3..2dbcec5 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -695,7 +695,7 @@ int _get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int ru { rule_list[idx].table_line=_maat_strdup(reply->str); } - else if(reply->type==REDIS_REPLY_ERROR)//Handle: "Loading Redis is loading the database in memory" + else if(reply->type==REDIS_REPLY_ERROR)//Deal with Redis response: "Loading Redis is loading the database in memory" { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor ,"Redis cmd=%s Error, Reply type=%d, str=%s",redis_cmd, reply->type, reply->str); diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 4a9b55b..b0a0f82 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -20,6 +20,7 @@ #include "Maat_rule.h" #include "Maat_rule_internal.h" #include "json2iris.h" +#include "cJSON.h" #include "dynamic_array.h" #include "aligment_int64.h" #include "config_monitor.h" @@ -30,7 +31,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_2_20180808=1; +int MAAT_FRAME_VERSION_2_2_20180921=1; const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; @@ -319,7 +320,7 @@ error_out: return ret; } -char* strlwr(char* string) +char* str_tolower(char* string) { int i=0; for(i=0;i<(int)strlen(string);i++) @@ -473,7 +474,166 @@ int cnt_maskbits(struct in6_addr mask) } return bits_cnt; } +//@param value is a JSON, like {"tags":[{"tag":"location","value":"北京/朝阳/华严北里/甲22号},{"tag":"isp","value":"电信"}]} +int parse_accept_tag(const char* value, struct rule_tag** result, void* logger) +{ + cJSON* json=NULL, *array=NULL,*tag=NULL, *tmp=NULL; + struct rule_tag* p=NULL; + int n_tags=0; + json=cJSON_Parse(value); + if(!json) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, + "MAAT_OPT_ACCEPT_TAGS Error before: %-200.200s",cJSON_GetErrorPtr()); + return 0; + } + array=cJSON_GetObjectItem(json, "tags"); + n_tags=cJSON_GetArraySize(array); + p=(struct rule_tag*)calloc(sizeof(struct rule_tag), n_tags); + for(int i=0;ivaluestring); + tmp=cJSON_GetObjectItem(tag, "value"); + p[i].tag_val=_maat_strdup(tmp->valuestring); + } + cJSON_Delete(json); + *result=p; + return n_tags; +} +static int compare_each_tag(cJSON* tag_obj, const struct rule_tag* accept_tags, int n_accept) +{ + const char* tag_name; + const char* tag_val; + int n_val; + cJSON *tab_name_obj=NULL, *tag_vals_array=NULL, *tag_val_obj=NULL; + int i=0, j=0, name_matched=0; + tab_name_obj=cJSON_GetObjectItem(tag_obj,"tag"); + if(!tab_name_obj||tab_name_obj->type!=cJSON_String) + { + goto error_out; + } + tag_name=tab_name_obj->valuestring; + tag_vals_array=cJSON_GetObjectItem(tag_obj,"value"); + if(!tag_vals_array||tag_vals_array->type!=cJSON_Array) + { + goto error_out; + } + n_val=cJSON_GetArraySize(tag_vals_array); + for(i=0;itype!=cJSON_String) + { + goto error_out; + } + tag_val=tag_val_obj->valuestring; + // compare a/b/c with a/b/c/d is a miss. + if(strlen(accept_tags[i].tag_val)0) + { + return 0; + } + else + { + return 1; + } +error_out: + return -1; + +} +//@param tag_set likes [{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}] +static int compare_each_tag_set(cJSON* tag_set, const struct rule_tag* accept_tags, int n_accept) +{ + cJSON *tag_obj=NULL; + int n_tag=0, ret=0, matched=0; + n_tag=cJSON_GetArraySize(tag_set); + for(int i=0; itype!=cJSON_Object) + { + goto error_out; + } + ret=compare_each_tag(tag_obj, accept_tags, n_accept); + if(ret<0) + { + return -1; + } + if(ret==1) + { + matched++; + } + } + if(matched==n_tag) + { + return 1; + } + else + { + return 0; + } +error_out: + return -1; +} +//@param value {"tag_sets":[[{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}],[{"tag":"location","value":["北京"]},{"tag":"isp","value":["联通"]}]]} +//@return 1 on match, 0 on not match, -1 on error. +static int compare_accept_tag(const char* value,const struct rule_tag* accept_tags, int n_tags) +{ + cJSON *json=NULL; + cJSON *tag_set_array=NULL, *tag_set=NULL; + int ret=-1, n_set=0; + json=cJSON_Parse(value); + if(!json) + { + goto error_out; + } + tag_set_array=cJSON_GetObjectItem(json, "tag_sets"); + if(!tag_set_array||tag_set_array->type!=cJSON_Array) + { + goto error_out; + } + n_set=cJSON_GetArraySize(tag_set_array); + for(int i=0; itype!=cJSON_Array) + { + goto error_out; + } + ret=compare_each_tag_set(tag_set, accept_tags, n_tags); + if(ret!=0)//match or error occurs. + { + break; + } + } +error_out: + cJSON_Delete(json); + return ret; +} int lqueue_destroy_cb(void *data, long data_len, void *arg) { assert(0); @@ -544,12 +704,12 @@ int read_expr_table_info(const char* line,int line_num,struct _Maat_table_info_t ,&(p->cross_cache_size) ,quick_str_scan); memset(ret,0,sizeof(ret)); - ret[0]=map_str2int(string2int_map,strlwr(table_type),(int*)&(p->table_type)); - ret[1]=map_str2int(string2int_map,strlwr(src_charset),(int*)&(p->src_charset)); - ret[2]=map_str2int(string2int_map,strlwr(merge),&(p->do_charset_merge)); + ret[0]=map_str2int(string2int_map,str_tolower(table_type),(int*)&(p->table_type)); + ret[1]=map_str2int(string2int_map,str_tolower(src_charset),(int*)&(p->src_charset)); + ret[2]=map_str2int(string2int_map,str_tolower(merge),&(p->do_charset_merge)); if(strlen(quick_str_scan)>0) { - ret[3]=map_str2int(string2int_map,strlwr(quick_str_scan),&(p->quick_expr_switch)); + ret[3]=map_str2int(string2int_map,str_tolower(quick_str_scan),&(p->quick_expr_switch)); } memset(quick_str_scan,0,sizeof(quick_str_scan)); @@ -566,7 +726,7 @@ int read_expr_table_info(const char* line,int line_num,struct _Maat_table_info_t sub_token= strtok_r(token,"/", &saveptr); if (sub_token == NULL) break; - ret[3]=map_str2int(string2int_map,strlwr(sub_token),(int*)&(p->dst_charset[j])); + ret[3]=map_str2int(string2int_map,str_tolower(sub_token),(int*)&(p->dst_charset[j])); if(ret[3]>0) { if(p->dst_charset[j]==p->src_charset) @@ -588,7 +748,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* FILE*fp=NULL; char line[MAX_TABLE_LINE_SIZE]; int i=0,ret=0,table_cnt=0; - char table_type_str[16],not_care[1024], user_region_encoding[32]; + char table_type_str[16],not_care[1024], tmp_str[32]; MESA_htable_handle string2int_map=map_create(); struct _Maat_table_info_t*p=NULL; struct _Maat_table_info_t*conj_table=NULL; @@ -640,7 +800,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* } p=create_table_info(max_thread_num); - ret=sscanf(line,"%hu\t%s\t%s\t%s",&(p->table_id) + ret=sscanf(line,"%hu\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id) ,p->table_name[0] ,table_type_str ,not_care); @@ -650,7 +810,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* "Maat read table info %s line %d error: not enough column.",table_info_path,i); continue; } - ret=map_str2int(string2int_map,strlwr(table_type_str),(int*)&(p->table_type)); + ret=map_str2int(string2int_map,str_tolower(table_type_str),(int*)&(p->table_type)); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, @@ -673,17 +833,17 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* case TABLE_TYPE_PLUGIN: p->cb_info=(struct _plugin_table_info*)calloc(sizeof(struct _plugin_table_info),1); p->cb_info->cache_lines=dynamic_array_create(1024,1024); - ret=sscanf(not_care,"%d",&(p->valid_flag_column)); + ret=sscanf(not_care,"%d\t%d\t%[0-9,]",&(p->valid_flag_column), &p->rule_tag_column, tmp_str); if(ret==0||ret==EOF) { p->valid_flag_column=-1; } break; case TABLE_TYPE_COMPILE: - ret=sscanf(not_care,"%[a-z0-9]",user_region_encoding); + ret=sscanf(not_care,"%[a-z0-9]",tmp_str); if(ret>0) { - ret=map_str2int(string2int_map,strlwr(user_region_encoding),(int*)&(p->user_region_encoding)); + ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->user_region_encoding)); } if(ret!=1) { @@ -2657,19 +2817,20 @@ error_out: intval_rule=NULL; } -void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger) +void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger) { struct db_compile_rule_t *p_compile=(struct db_compile_rule_t*)calloc(sizeof(struct db_compile_rule_t ),1); struct _head_Maat_rule_t* p_m_rule=&(p_compile->m_rule_head); char user_region[MAX_TABLE_LINE_SIZE]={0}; + char tag_str[MAX_TABLE_LINE_SIZE]={0}; int ret=0; p_compile->declare_grp_num=0; - ret=sscanf(table_line,"%d\t%d\t%hhd\t%hhd\t%hhd\t%lld\t%s\t%d\t%d",&(p_m_rule->config_id) + ret=sscanf(table_line,"%d\t%d\t%hhd\t%hhd\t%hhd\t%s\t%s\t%d\t%d",&(p_m_rule->config_id) ,&(p_m_rule->service_id) ,&(p_m_rule->action) ,&(p_m_rule->do_blacklist) ,&(p_m_rule->do_log) - ,&(p_compile->effective_range) + ,tag_str ,user_region ,&(p_compile->is_valid) ,&(p_compile->declare_grp_num)); @@ -2677,11 +2838,26 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , "update error,invalid format of compile table %s:%s" - ,table->table_name[table->updating_name],table_line); - free(p_compile); - p_compile=NULL; + ,table->table_name[table->updating_name],table_line); table->udpate_err_cnt++; - return; + goto no_save; + } + if(n_tags>0&&strlen(tag_str)>2) + { + ret=compare_accept_tag(tag_str, tags, n_tags); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,invalid tag format of compile table %s:%s" + ,table->table_name[table->updating_name],table_line); + table->udpate_err_cnt++; + goto no_save; + } + if(ret==0) + { + table->unmatch_tag_cnt++; + goto no_save; + } } switch(table->user_region_encoding) { @@ -2702,10 +2878,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line { table->cfg_num--; } - free(p_compile->service_defined); - p_compile->service_defined=NULL; - free(p_compile); - p_compile=NULL; + goto no_save; } else { @@ -2714,13 +2887,9 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , "duplicate config of compile table %s config_id=%d" - ,table->table_name[table->updating_name],p_m_rule->config_id); - free(p_compile->service_defined); - p_compile->service_defined=NULL; - free(p_compile); - p_compile=NULL; + ,table->table_name[table->updating_name],p_m_rule->config_id); table->udpate_err_cnt++; - + goto no_save; } else { @@ -2728,7 +2897,13 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line table->cfg_num++; } } + return; +no_save: + free(p_compile->service_defined); + p_compile->service_defined=NULL; + free(p_compile); + p_compile=NULL; return; } @@ -2901,13 +3076,49 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger) q_cnt,bury_cnt); } } -void plugin_table_callback(struct _Maat_table_info_t* table,const char* table_line,void* logger) +void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line, const struct rule_tag* tags, int n_tags, void* logger) { - int i=0; + int i=0, ret=1; unsigned int len=strlen(table_line)+1; struct _plugin_table_info* p_table_cb=table->cb_info; char *p=NULL; + char* copy=NULL; + + char *token=NULL,*sub_token=NULL,*saveptr; + if(table->rule_tag_column>0&&n_tags>0) + { + copy=_maat_strdup(table_line); + for (token = copy, i=0; irule_tag_column ; token= NULL, i++) + { + sub_token= strtok_r(token,"\t", &saveptr); + if (sub_token == NULL) + break; + } + if(i==table->rule_tag_column&&strlen(sub_token)>2) + { + ret=compare_accept_tag(sub_token, tags, n_tags); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,invalid tag format of plugin table %s:%s" + ,table->table_name[table->updating_name],table_line); + table->udpate_err_cnt++; + } + if(ret==0) + { + table->unmatch_tag_cnt++; + } + } + free(copy); + copy=NULL; + if(ret!=1) + { + return; + } + } + p_table_cb->acc_line_num++; + if(p_table_cb->cb_plug_cnt>0) { for(i=0;icb_plug_cnt;i++) @@ -3202,13 +3413,13 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) update_digest_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); break; case TABLE_TYPE_COMPILE: - update_compile_rule(feather->p_table_info[table_id], line, scanner,feather->logger); + update_compile_rule(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); break; case TABLE_TYPE_GROUP: update_group_rule(feather->p_table_info[table_id], line, scanner,feather->logger); break; case TABLE_TYPE_PLUGIN: - plugin_table_callback(feather->p_table_info[table_id], line,feather->logger); + update_plugin_table(feather->p_table_info[table_id], line, feather->accept_tags, feather->n_tags, feather->logger); default: break; diff --git a/src/entry/Makefile b/src/entry/Makefile deleted file mode 100644 index 5c8f6ec..0000000 --- a/src/entry/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -#opt: OPTFLAGS = -O2 -#export OPTFLAGS - -CC = gcc -CCC = g++ -CFLAGS = -Wall -g -fPIC -#GCOV_FLAGS = -fprofile-arcs -ftest-coverage -CFLAGS += $(OPTFLAGS) -#CFLAGS += $(GCOV_FLAGS) -LDDICTATOR = -Wl,-wrap,malloc -Wl,-wrap,calloc -Wl,-wrap,free -Wl,-wrap,realloc -LDFLAGS = -lMESA_handle_logger -lMESA_htable -lpthread -lrt -lm -lrulescan -lpcre -lMESA_field_stat2 -lcrypto -lhiredis_vip -#LDFLAGS += $(LDDICTATOR) -LDFLAGS += $(GCOV_FLAGS) -MAILLIB = ../lib - -G_H_DIR =../inc_internal -H_DIR =-I$(G_H_DIR) -I../../inc -LIBMAAT = libmaatframe.a -LIBMAAT_SO = libmaatframe.so - -OBJS=config_monitor.o Maat_rule.o Maat_api.o Maat_command.o Maat_stat.o UniversalBoolMatch.o dynamic_array.o\ - cJSON.o json2iris.o map_str2int.o interval_index.o gram_index_engine.o stream_fuzzy_hash.o rbtree.o -.c.o: - $(CC) -c $(CFLAGS) -I. $(H_DIR) $< - -.cpp.o: - $(CCC) -c $(CFLAGS) -I. $(H_DIR) $< - -all: $(LIBMAAT) $(LIBMAAT_SO) - -$(LIBMAAT_SO): $(OBJS) - $(CCC) -o $(LIBMAAT_SO) -shared $(OBJS) $(LDFLAGS) - cp $(LIBMAAT_SO) ../../lib/ -$(LIBMAAT): $(OBJS) - echo making dynamic lib ... - ar cqs $(LIBMAAT) $(OBJS) - cp $(LIBMAAT) ../../lib/ - -clean: - rm -rf *.o *.gcov *.gcno $(LIBMAAT) *~ - -opt: - $(MAKE) all diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index fb79d64..c70ebc3 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -715,10 +715,11 @@ int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * l compile_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - compile_cmd[cmd_cnt].json_string="effective_range"; - compile_cmd[cmd_cnt].json_type=cJSON_Number; + + compile_cmd[cmd_cnt].json_string="tags"; + compile_cmd[cmd_cnt].json_type=cJSON_String; compile_cmd[cmd_cnt].empty_allowed=1; - compile_cmd[cmd_cnt].default_int=0; + compile_cmd[cmd_cnt].default_string="0"; cmd_cnt++; compile_cmd[cmd_cnt].json_string="user_region"; diff --git a/src/entry/stream_fuzzy_hash.c b/src/entry/stream_fuzzy_hash.c index 48ed48c..5adf5b9 100644 --- a/src/entry/stream_fuzzy_hash.c +++ b/src/entry/stream_fuzzy_hash.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include "sfh_internal.h" #include "stream_fuzzy_hash.h" #include "interval_index.h" //#define DEBUG_PRINT diff --git a/src/entry/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h similarity index 97% rename from src/entry/Maat_rule_internal.h rename to src/inc_internal/Maat_rule_internal.h index 522d1ac..11e6ae9 100644 --- a/src/entry/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -261,9 +261,11 @@ struct _Maat_table_info_t }; struct _plugin_table_info *cb_info; int valid_flag_column; //for plugin table + int rule_tag_column; //for plugin table; int user_region_encoding; //for compile table, USER_REGION_ENCODE_xx //for stat>>>>>>>> unsigned long long udpate_err_cnt; + unsigned long long unmatch_tag_cnt; unsigned long long iconv_err_cnt; int stat_line_id; mcore_long_t scan_cnt; @@ -329,6 +331,11 @@ struct GIE_aux_t GIE_handle_t* gie_handle; MESA_lqueue_head update_q; }; +struct rule_tag +{ + char* tag_name; + char* tag_val; +}; struct _Maat_scanner_t { long long version; @@ -349,7 +356,7 @@ struct _Maat_scanner_t MESA_lqueue_head region_update_q; void * expr_compiler; scan_result_t *region_rslt_buff; - MESA_lqueue_head tomb_ref;//reference of feather->garbage_q + MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q struct _region_stat_t region_counter[MAX_TABLE_NUM]; int max_thread_num; iconv_t iconv_handle[MAX_CHARSET_NUM][MAX_CHARSET_NUM];//iconv_handle[to][from] @@ -399,6 +406,9 @@ struct _Maat_feather_t pthread_mutex_t redis_write_lock; //protect redis_write_ctx long long base_rgn_seq,base_grp_seq,server_time; long long load_version_from; + + struct rule_tag *accept_tags; + int n_tags; //internal states long long new_version; int active_plugin_table_num; @@ -451,6 +461,7 @@ struct serial_rule_t //rm= Redis Maat char table_name[256]; char* table_line; }; +int parse_accept_tag(const char* value, struct rule_tag** result, void* logger); void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set); diff --git a/src/entry/UniversalBoolMatch.h b/src/inc_internal/UniversalBoolMatch.h similarity index 100% rename from src/entry/UniversalBoolMatch.h rename to src/inc_internal/UniversalBoolMatch.h diff --git a/src/entry/aligment_int64.h b/src/inc_internal/aligment_int64.h similarity index 100% rename from src/entry/aligment_int64.h rename to src/inc_internal/aligment_int64.h diff --git a/src/entry/cJSON.h b/src/inc_internal/cJSON.h similarity index 100% rename from src/entry/cJSON.h rename to src/inc_internal/cJSON.h diff --git a/src/entry/config_monitor.h b/src/inc_internal/config_monitor.h similarity index 100% rename from src/entry/config_monitor.h rename to src/inc_internal/config_monitor.h diff --git a/src/entry/dynamic_array.h b/src/inc_internal/dynamic_array.h similarity index 100% rename from src/entry/dynamic_array.h rename to src/inc_internal/dynamic_array.h diff --git a/src/inc_internal/hiredis.h b/src/inc_internal/hiredis/hiredis.h similarity index 100% rename from src/inc_internal/hiredis.h rename to src/inc_internal/hiredis/hiredis.h diff --git a/src/inc_internal/read.h b/src/inc_internal/hiredis/read.h similarity index 100% rename from src/inc_internal/read.h rename to src/inc_internal/hiredis/read.h diff --git a/src/inc_internal/sds.h b/src/inc_internal/hiredis/sds.h similarity index 100% rename from src/inc_internal/sds.h rename to src/inc_internal/hiredis/sds.h diff --git a/src/entry/interval_index.h b/src/inc_internal/interval_index.h similarity index 100% rename from src/entry/interval_index.h rename to src/inc_internal/interval_index.h diff --git a/src/entry/json2iris.h b/src/inc_internal/json2iris.h similarity index 100% rename from src/entry/json2iris.h rename to src/inc_internal/json2iris.h diff --git a/src/entry/map_str2int.h b/src/inc_internal/map_str2int.h similarity index 100% rename from src/entry/map_str2int.h rename to src/inc_internal/map_str2int.h diff --git a/src/entry/rbtree.h b/src/inc_internal/rbtree.h similarity index 100% rename from src/entry/rbtree.h rename to src/inc_internal/rbtree.h diff --git a/src/entry/rbtree_augmented.h b/src/inc_internal/rbtree_augmented.h similarity index 100% rename from src/entry/rbtree_augmented.h rename to src/inc_internal/rbtree_augmented.h diff --git a/src/entry/sfh_internal.h b/src/inc_internal/sfh_internal.h similarity index 100% rename from src/entry/sfh_internal.h rename to src/inc_internal/sfh_internal.h diff --git a/src/inc_internal/stream.h b/src/inc_internal/stream.h deleted file mode 100644 index 2a7ce9a..0000000 --- a/src/inc_internal/stream.h +++ /dev/null @@ -1,562 +0,0 @@ -#ifndef _APP_STREAM_H_ -#define _APP_STREAM_H_ - -/************************************************************************************* -Update Log: -2014-09-17 LiJia: - (1)ԭstruct ipaddrṹ, ƽ̨ṹstruct layer_addrڴֲϼ; - (2)½ӿ: - int MESA_kill_tcp_synack(struct streaminfo *stream, const void *raw_pkt); - HMDģʽµFD. - -2014-09-10 LiJia: - (1)½ӿ: - int MESA_inject_pkt(struct streaminfo *stream, const char *payload, - int payload_len, const void *raw_pkt, UCHAR snd_routedir); - Ӧò, ԭMESA_fakepacket_send_tcp()˺ܶ, ʹ. -**************************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include - -#define USE_MESA_STREAM_HASH (1) /* ʹûģʽµHASH */ - -extern int MESA_PLATFORM_VERSION_20140917; /* ƽ̨汾 */ - -#define ADDR_UNSYMMETRY (0) /* 2014-04-25, ԴXinJingİ, ʱVLANַԳ, ˺겻Ӧڴ */ - -#if 0 -#define FOR_863 (0) /* for 863Ŀûǩģ */ -#define FOR_108 (0) -#define RAW_IP_FRAG_PKT (0) /* 2013-08-21 LiJia add, ȫԭʼIPƬ */ -#endif - -#ifndef UINT8 -typedef unsigned char UINT8; -#endif -#ifndef UCHAR -typedef unsigned char UCHAR; -#endif -#ifndef UINT16 -typedef unsigned short UINT16; -#endif -#ifndef UINT32 -typedef unsigned int UINT32; -#endif -#ifndef UINT64 -typedef unsigned long long UINT64; -#endif - -//ķ -#define DIR_C2S 0x01 -#define DIR_S2C 0x02 -#define DIR_DOUBLE 0x03 - -//ķ -#define DIR_ROUTE_UP 0x00 -#define DIR_ROUTE_DOWN 0x01 - -//ַͶ -enum addr_type_t{ - __ADDR_TYPE_INIT = 0, - ADDR_TYPE_IPV4, /* 1, IPv4ַԪϢ */ - ADDR_TYPE_IPV6, /* 2, IPv6ַԪϢ */ - ADDR_TYPE_VLAN, /* 3 */ - ADDR_TYPE_MAC, /* 4 */ - ADDR_TYPE_ARP = 5, /* 5 */ - ADDR_TYPE_GRE, /* 6 */ - ADDR_TYPE_MPLS, /* 7 */ - ADDR_TYPE_PPPOE_SES, /* 8 */ - ADDR_TYPE_TCP, /* 9 */ - ADDR_TYPE_UDP = 10, /* 10 */ - ADDR_TYPE_L2TP, /* 11 */ - //ADDR_TYPE_STREAM_TUPLE4_V4, /* 12, ϵַ, IPv4ַԪϢ */ - //ADDR_TYPE_STREAM_TUPLE4_V6, /* 13, ϵַ, IPv6ַԪϢ */ - __ADDR_TYPE_IP_PAIR_V4, /* 14, IPv4ַ */ - __ADDR_TYPE_IP_PAIR_V6, /* 15, IPv6ַ */ - __ADDR_TYPE_MAX, /* 16 */ -}; - - -#define TCP_TAKEOVER_STATE_FLAG_OFF 0 -#define TCP_TAKEOVER_STATE_FLAG_ON 1 - -//Ӧò㿴״̬ -#define OP_STATE_PENDING 0 -#define OP_STATE_REMOVE_ME 1 -#define OP_STATE_CLOSE 2 -#define OP_STATE_DATA 3 - -//Ӧò㷵ؽ -#define APP_STATE_GIVEME 0x00 -#define APP_STATE_DROPME 0x01 -#define APP_STATE_FAWPKT 0x00 -#define APP_STATE_DROPPKT 0x10 - -//Ͷ -enum stream_type_t{ - STREAM_TYPE_NON = 0, /* ĸ, VLAN, IP */ - STREAM_TYPE_TCP, - STREAM_TYPE_UDP, - STREAM_TYPE_SOCKS4, - STREAM_TYPE_SOCKS5, - STREAM_TYPE_HTTP_PROXY, - STREAM_TYPE_PPPOE, -}; - -#define PROXY_STATE_SEL 0 -#define PROXY_STATE_LINK_IN 1 - - -/* ԭʼṹ */ -typedef struct { - enum addr_type_t low_layer_type; /* ԭʼײЭ, MAC(pcap), ҲIPv4(pag) */ - int raw_pkt_len; /* ԭʼܳ */ - const void *raw_pkt_data; /* ԭʼͷָ, low_layer_typeжЭ */ - struct timeval raw_pkt_ts; /* ԭʼʱ, ȫΪ0ִ֧˹(pagģʽ) */ -}raw_pkt_t; - -// Ϣ -struct proxy_node -{ - UINT16 iType; // , 0 ʾЧ - UINT16 uiPort; // ˿ - UINT16 uiUserLen; - UINT16 uiPwdLen; - UINT16 uiApendLen; - - UCHAR opstate; //ǰ״̬ - UCHAR dealstate; //״̬ - UINT32 uiIP; // IPַv4, ֽ - UCHAR *pIpv6; // IPַ, v6ַ - UCHAR *pUser; // û - UCHAR *pPwd; // - UCHAR *append; // Ϣurl - - void *apme; // Ϣurl - struct proxy_node *pnext; -} ; - - -typedef struct raw_ipfrag_list{ - void *frag_packet; - int pkt_len; - int type; /* IPv4 or IPv6 */ - struct raw_ipfrag_list *next; -}raw_ipfrag_list_t; - - -struct buf_unorder -{ - struct buf_unorder *next; - struct buf_unorder *prev; - void *data; - UINT32 len; - UINT32 tcpdatalen; - UINT32 urg_ptr; - char fin; - char urg; - char rst; - unsigned char pad; - UINT32 seq; - UINT32 ack; - raw_ipfrag_list_t *ipfrag_list; - raw_pkt_t raw_pkt; /* 洢ԭʼ */ -}; - -/*ṹ嶨壺*/ -struct half_tcpstream -{ - UCHAR *data; - UINT32 offset; /*dataеһֽTCPеƫ*/ - UINT32 count; /*ӽΪֹֽܳ*/ - UINT32 count_new; /*µֽ*/ - UINT32 count_ideal; /*ӽΪֹӦõܳ*/ - UINT32 pktcout; /*ۼƵİ*/ - UINT32 totallost; /*ۼƶ*/ - UINT32 seq; /*ڴseq*/ - UINT32 first_data_seq; /*ʼĵseq*/ - UINT32 ack_seq; /*ʹõӦ*/ - UINT16 window; /*ݻڴС*/ - UCHAR maxunorder; /*Ŀ*/ - UCHAR finstate; /*fin״̬*/ - UINT32 pktcout_new; /*µİ*/ - UINT32 unorder_cnt; - struct buf_unorder *unorderlist; /**/ - struct buf_unorder *unorderlisttail; /*βָ*/ - -}; - -/* papp */ -struct tuple4 { - u_int saddr; - u_int daddr; - u_short source; - u_short dest; -}; - -struct tuple6 -{ - UCHAR saddr[16] ; - UCHAR daddr[16] ; - UINT16 source; - UINT16 dest; -}; - -/* network-order */ -struct stream_tuple4_v4{ - UINT32 saddr; /* network order */ - UINT32 daddr; /* network order */ - UINT16 source; /* network order */ - UINT16 dest; /* network order */ -}; - - -#ifndef IPV6_ADDR_LEN -#define IPV6_ADDR_LEN (sizeof(struct in6_addr)) -#endif - -struct stream_tuple4_v6 -{ - UCHAR saddr[IPV6_ADDR_LEN] ; - UCHAR daddr[IPV6_ADDR_LEN] ; - UINT16 source; /* network order */ - UINT16 dest; /* network order */ -}; - - - -#define GRE_TAG_LEN (4) -struct layer_addr_gre -{ - UINT16 gre_id; -}; - - -#define VLAN_ID_MASK (0x0FFF) -#define VLAN_TAG_LEN (4) -struct layer_addr_vlan -{ - UINT16 vlan_id; /* network order */ -}; - -struct layer_addr_pppoe_session -{ -#if __BYTE_ORDER == __LITTLE_ENDIAN - unsigned int ver:4; - unsigned int type:4; -#endif -#if __BYTE_ORDER == __BIG_ENDIAN - unsigned int type:4; - unsigned int ver:4; -#endif - unsigned char code; - unsigned short session_id; -}; - -#ifndef MAC_ADDR_LEN -#define MAC_ADDR_LEN (6) -#endif - -struct layer_addr_mac -{ - UCHAR src_mac[MAC_ADDR_LEN]; /* network order */ - UCHAR dst_mac[MAC_ADDR_LEN]; /* network order */ -}; - -struct layer_addr_ipv4 -{ - UINT32 saddr; /* network order */ - UINT32 daddr; /* network order */ - /* 2014-04-21 lijia add, - Ϊ˿ռ䡢ԡЧ, ǿưЭδ, - IP洢ԪϢ, TCPָָֻ˿ڴ, - ȡԪʱ, Ҫget_tuple4(). - IP, ˿ϢΪ0; - */ - UINT16 source; /* network order */ - UINT16 dest; /* network order */ -}; - -struct layer_addr_ipv6 -{ - UCHAR saddr[IPV6_ADDR_LEN] ; /* network order */ - UCHAR daddr[IPV6_ADDR_LEN] ; /* network order */ - /* 2014-04-21 lijia add, - Ϊ˿ռ䡢ԡЧ, ǿưЭδ, - IP洢ԪϢ, TCPָָֻ˿ڴ, - ȡԪʱ, Ҫget_tuple4(). - IP, ˿ϢΪ0; - */ - UINT16 source;/* network order */ - UINT16 dest;/* network order */ -}; - -struct layer_addr_tcp -{ - UINT16 source; /* network order */ - UINT16 dest; /* network order */ -}; - -struct layer_addr_udp -{ - UINT16 source; /* network order */ - UINT16 dest; /* network order */ -}; - -struct layer_addr_l2tp -{ - UINT32 tunnelid; /* network order */ - UINT32 sessionid; /* network order */ -}; - -struct layer_addr -{ - UCHAR addrtype; /* ַ, enum addr_type_t */ - /* Ϊ˷Ӧòȡַ, ˴ʹ, ʡȥָǿת */ - union - { - struct stream_tuple4_v4 *tuple4_v4; - struct stream_tuple4_v6 *tuple4_v6; - struct layer_addr_ipv4 *ipv4; - struct layer_addr_ipv6 *ipv6; - struct layer_addr_vlan *vlan; - struct layer_addr_mac *mac; - struct layer_addr_gre *gre; - struct layer_addr_tcp *tcp; - struct layer_addr_udp *udp; - struct layer_addr_pppoe_session *pppoe_ses; - struct layer_addr_l2tp *l2tp; - void *paddr; - }; - UCHAR addrlen; /* ַṹ */ -}; - -/* ˽ṹںpapp, ָʱ, struct layer_addrǿת. */ -struct ipaddr -{ - UCHAR addrtype; - union - { - struct stream_tuple4_v4 *v4; - struct stream_tuple4_v6 *v6; - }paddr; - // struct tuple4 *paddr; -}; - - -/* to do: - ַ; - , ڲⲻɼ, stream.h, ⲿstream.hIJֻͼ; - */ -struct streaminfo -{ - struct layer_addr addr; //ЭַϢ - struct streaminfo *pfather;//ϲṹ - UCHAR type; // - UCHAR threadnum; // ߳ - UCHAR dir:2; /* Ч, ĵ˫0x01:c-->s; 0x02:s-->c; 0x03 c<-->s; */ - UCHAR curdir:2; /* Ч, ǰϲ߼, 0x01:c-->s; 0x02:s-->c */ - UCHAR layer_dir:2; /* Ч, ǰĵַǷĬϹ"˿ǿͻ"ͬ */ - UCHAR stream_dir:2; /* Ч, Ĵ洢ĵַǷĬϹ"˿ǿͻ"ͬ */ - UCHAR dirreverse; /* ʱǷipַת, "˿ǿͻ"෴ */ - UCHAR opstate; //ǰ״̬ - UCHAR pktstate; //ӵİ - UCHAR routedir; /* , Ч, ˹ָ, ڷʱǷǷͬ, , 0:; 1: */ - UCHAR addr_use_as_hash; /* addrǷΪHASHͱȽϵIJ, :MACַ */ - - UCHAR stream_killed_flag; /* 2014-08-22 lijia add, ģʽ, ѾKill, ֱ֮DropKill, ٸϲ */ - UCHAR __pad__[7]; //8ֽڶ - void *pproject; //ÿ̿Զʹã - void *pdetail; //ϸϢ - const void *p_layer_header; /* ָ򱾲ͷָ, ڹʱȡز */ - -#if 0 //FOR_108, ->pproject - unsigned long stream_id; -#endif - -#if 0 //RAW_IP_FRAG_PKT, ->pproject - raw_ipfrag_list_t *ipfrag_list; /* 2013-08-21 LiJia add, ԭʼƬ */ -#endif - -#if 0 //FOR_863, ->pproject - char terminal_tag[40]; /* ûǩ */ -#endif - }; - - -struct tcpdetail -{ - - void *pdata; // - UINT32 datalen; //ݳ - UINT32 lostlen; - UCHAR multisynflag:4; // multi syn - UCHAR needackflag:2; //Ҫϴack - UCHAR takeoverflag:2; - UCHAR tcpstateflag; // ڼ¼tcpĻỰSYN״̬ - UCHAR link_state; // ӵ״̬ - UCHAR creat_mod; - UINT32 regionid; //Ϣ - - struct half_tcpstream *pclient; //clientTCPϢ - struct half_tcpstream *pserver; // serverTCPϢ - UINT32 serverpkt; - UINT32 clientpkt; - UINT32 servercount; - UINT32 clientcount; - UINT64 creattime; - UINT64 lastmtime; - - UINT32 iserverseq; //ӽʱʱ洢seq - UINT32 iclientseq; //ӽʱʱ洢seq - void *apme; //Ӧò - void *pAllpktpme; //״̬tcp - struct proxy_node *pproxylist; //,ģ鸺ͷ - -}; -struct udpdetail -{ - void *pdata; // - UINT32 datalen; //ݳ - UINT32 regionid; //Ϣ - UINT32 serverpkt; - UINT32 clientpkt; - UINT32 servercount; - UINT32 clientcount; - UINT64 creattime; - UINT64 lastmtime; - void *apme; //Ӧò -}; - - -//ҵýʱsession_state״̬ -#define SESSION_STATE_PENDING 0x01 -#define SESSION_STATE_DATA 0x02 -#define SESSION_STATE_CLOSE 0x04 - -//ҵʱķֵ -#define PROT_STATE_GIVEME 0x01 -#define PROT_STATE_DROPME 0x02 -#define PROT_STATE_KILLME 0x04 - -//ҵʱ -typedef struct _plugin_session_info -{ - unsigned short plugid; //plugidƽ̨ - char session_state; //Ự״̬PENDING,DATA,CLOSE - char _pad_; // - int buflen; //ǰֶγ - long long prot_flag; //ǰֶεflagֵ - void *buf; //ǰֶ - void* app_info; //Ϣ -}stSessionInfo; -#ifdef __cplusplus -extern "C" { -#endif - - -/* - args: - [IN]: - stream:ṹָ; - [OUT] - addr_type:Ԫ, ADDR_TYPE_IPV4 or ADDR_TYPE_IPV6; - - return value: - NULL : error; - NON-NULL : ָԪṹָ; -*/ -struct layer_addr *get_stream_tuple4(struct streaminfo *this_stream); -int get_thread_count(void); - -typedef char (*STREAM_CB_FUN_T)(struct streaminfo *pstream,void **pme, int thread_seq,const void *raw_pkt); -typedef char (*IPv4_CB_FUN_T)(struct streaminfo *pstream,unsigned char routedir,int thread_seq, const void *raw_pkt); -typedef char (*IPv6_CB_FUN_T)(struct streaminfo *pstream,unsigned char routedir,int thread_seq, const void *raw_pkt); - -/* - a_* Ϣ; - f_*: ӦĸϢ; - raw_pkt: ԭʼָ, ʵΪ'raw_pkt_t'; - pme: ˽ָ룬չãʱΪNULL; - thread_seq߳; - -ֵΪĸֵ - - APP_STATE_GIVEME򱾺Ͱ - APP_STATE_DROPME򱾺Ͱ - APP_STATE_FAWPKTעݰ - APP_STATE_DROPPKTעݰ -*/ -char IPv4_ENTRY_EXAMPLE( struct streaminfo *f_stream,unsigned char routedir,int thread_seq, const void *raw_pkt); -char IPv6_ENTRY_EXAMPLE( struct streaminfo *f_stream,unsigned char routedir,int thread_seq,const void *raw_pkt); -char TCP_ENTRY_EXAMPLE(struct streaminfo *a_tcp, void **pme, int thread_seq,const void *raw_pkt); -char UDP_ENTRY_EXAMPLE(struct streaminfo *a_udp, void **pme, int thread_seq,const void *raw_pkt); - -int stream_register_tcp_allpkt (STREAM_CB_FUN_T fun); -int stream_register_tcp_takeover (STREAM_CB_FUN_T fun); - -int stream_register_tcp (STREAM_CB_FUN_T fun); -int stream_register_udp (STREAM_CB_FUN_T fun); -int stream_register_ip (IPv4_CB_FUN_T fun); -int stream_register_ipv6 (IPv6_CB_FUN_T fun); - - -void *dictator_malloc(int thread_seq,size_t size); -void dictator_free(int thread_seq,void *pbuf); -void *dictator_realloc(int thread_seq, void* pbuf, size_t size); - -char PROT_PROCESS(stSessionInfo* session_info, void **pme, int thread_seq,struct streaminfo *a_stream,void *a_packet); - -/* ӦòȡжӦûǩ */ -//int terminal_tag_probe(struct streaminfo *stream, struct mesa_tcp_hdr *this_tcphdr,void *rawippkt); -const unsigned char *get_terminal_tag(struct streaminfo *stream); -//int MESA_kill_tcp(struct streaminfo *stream, struct ip *a_packet); -int MESA_kill_tcp(struct streaminfo *stream, const void *raw_pkt); -int MESA_kill_tcp_synack(struct streaminfo *stream, const void *raw_pkt); - -/* - ARG: - stream: ṹָ; - payload: Ҫ͵ָ; - payload_len: Ҫ͵ݸس; - raw_pkt: ԭʼָ; - snd_routedir: Ҫݵķ, ԭʼΪ:stream->routedir , - ԭʼͬ, snd_dir = stream->routedir, - ԭʼ, snd_dir = stream->routedir ^ 3(1<--->2, 2<--->1ѧת); - return value: - -1: error. - >0: ͵ݰʵܳ(payload_len + ײͷ); -*/ -int MESA_inject_pkt(struct streaminfo *stream, const char *payload, int payload_len, const void *raw_pkt, UCHAR snd_routedir); - -/* 2014-07-31 LiJia add, for set one stream unorder number */ -int tcp_set_single_stream_max_unorder(struct streaminfo *stream, UCHAR dir, UCHAR unorder_num); - -raw_ipfrag_list_t *get_raw_frag_list(struct streaminfo *stream); -//void frags_list_free_one(raw_ipfrag_list_t *frags_list, int); -//raw_ipfrag_list_t *frags_list_merge(raw_ipfrag_list_t *new_merge_list, raw_ipfrag_list_t *old_list); -//raw_ipfrag_list_t *raw_ip_frag_list_attach(int thread_num); -//void raw_ip_frag_list_detach(int thread_num); - - -unsigned long get_stream_id(struct streaminfo *stream); - -void MESA_stream_list_free(struct streaminfo *heap_stream); -struct streaminfo *MESA_stream_list_dup(struct streaminfo *stack_stream, int reverse); -int MESA_stream_list_cmp(struct streaminfo *stream1, struct streaminfo *stream2); - -#ifdef __cplusplus -} -#endif - - - -#endif - diff --git a/src/inc_internal/stream_old.h b/src/inc_internal/stream_old.h deleted file mode 100644 index 682462d..0000000 --- a/src/inc_internal/stream_old.h +++ /dev/null @@ -1,336 +0,0 @@ -#ifndef _APP_STREAM_H_ -#define _APP_STREAM_H_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#define FOR_863 (0) /* for 863Ŀûǩģ */ -#define FOR_108 (0) -#define RAW_IP_FRAG_PKT (0) /* 2013-08-21 LiJia add, ȫԭʼIPƬ */ - -typedef unsigned char UINT8; -typedef unsigned long long UINT64; -typedef unsigned int UINT32; -typedef unsigned short UINT16; -typedef unsigned char UCHAR; - -//ķ -#define DIR_C2S 0x01 -#define DIR_S2C 0x02 -#define DIR_DOUBLE 0x03 - -//ķ -#define DIR_ROUTE_UP 0x00 -#define DIR_ROUTE_DOWN 0x01 - -//ַͶ -#define ADDR_TYPE_IPV4 1 -#define ADDR_TYPE_IPV6 2 -#define ADDR_TYPE_VLAN 3 - -#define TCP_TAKEOVER_STATE_FLAG_OFF 0 -#define TCP_TAKEOVER_STATE_FLAG_ON 1 - -//Ӧò㿴״̬ -#define OP_STATE_PENDING 0 -#define OP_STATE_REMOVE_ME 1 -#define OP_STATE_CLOSE 2 -#define OP_STATE_DATA 3 - -//Ӧò㷵ؽ -#define APP_STATE_GIVEME 0x00 -#define APP_STATE_DROPME 0x01 -#define APP_STATE_FAWPKT 0x00 -#define APP_STATE_DROPPKT 0x10 - -//Ͷ -#define STREAM_TYPE_TCP 1 -#define STREAM_TYPE_UDP 2 -#define STREAM_TYPE_VLAN 3 -#define STREAM_TYPE_SOCKS4 4 -#define STREAM_TYPE_SOCKS5 5 -#define STREAM_TYPE_HTTP_PROXY 6 - -#define STREAM_TYPE_MPLS 7 - -#define PROXY_STATE_SEL 0 -#define PROXY_STATE_LINK_IN 1 - - -// Ϣ -struct proxy_node -{ - UINT16 iType; // , 0 ʾЧ - UINT16 uiPort; // ʵ˿ - UINT16 uiUserLen; - UINT16 uiPwdLen; - UINT16 uiApendLen; - - UCHAR opstate; //ǰ״̬ - UCHAR dealstate; //״̬ - UINT32 uiIP; // ʵIPַv4, ֽ - UCHAR *pIpv6; // ʵIPַ, v6ַ - UCHAR *pUser; // û - UCHAR *pPwd; // - UCHAR *append; // Ϣurl - - void *apme; // Ϣurl - struct proxy_node *pnext; -} ; - - -#if RAW_IP_FRAG_PKT -typedef struct raw_ipfrag_list{ - void *frag_packet; - int pkt_len; - int type; /* IPv4 or IPv6 */ - struct raw_ipfrag_list *next; -}raw_ipfrag_list_t; -#endif - -struct buf_unorder -{ - struct buf_unorder *next; - struct buf_unorder *prev; - void *data; - UINT32 len; - UINT32 tcpdatalen; - UINT32 urg_ptr; - char fin; - char urg; - char rst; - unsigned char pad; - UINT32 seq; - UINT32 ack; -#if RAW_IP_FRAG_PKT - raw_ipfrag_list_t *ipfrag_list; -#endif -}; -/*ṹ嶨壺*/ -struct half_tcpstream -{ - UCHAR *data; - UINT32 offset; /*dataеһֽTCPеƫ*/ - UINT32 count; /*ӽΪֹֽܳ*/ - UINT32 count_new; /*µֽ*/ - UINT32 count_ideal; /*ӽΪֹӦõܳ*/ - UINT32 pktcout; /*ۼƵİ*/ - UINT32 totallost; /*ۼƶ*/ - UINT32 seq; /*ڴseq*/ - UINT32 first_data_seq; /*ʼĵseq*/ - UINT32 ack_seq; /*ʹõӦ*/ - UINT16 window; /*ݻڴС*/ - UCHAR maxunorder; /*Ŀ*/ - UCHAR finstate; /*fin״̬*/ - UINT32 pktcout_new; /*µİ*/ - UINT32 unorder_cnt; - struct buf_unorder *unorderlist; /**/ - struct buf_unorder *unorderlisttail; /*βָ*/ - -}; - -#ifndef STRUCT_TUPLE4_DEFINED -#define STRUCT_TUPLE4_DEFINED (1) -struct tuple4 { - UINT32 saddr; - UINT32 daddr; - UINT16 source; - UINT16 dest; -}; -#endif - -struct tuple6 -{ - UCHAR saddr[16] ; - UCHAR daddr[16] ; - UINT16 source; - UINT16 dest; -}; - -#define VLAN_ID_LEN 4 -struct tuplevlan -{ - UCHAR vlan_id[VLAN_ID_LEN]; -}; - -struct tupell2tp -{ - UINT32 tunnelid; - UINT32 sessionid; -}; -struct ipaddr -{ - UCHAR addrtype; //ַ ipv6 or ipv4 - union - { - struct tuple4 *v4; - struct tuple6 *v6; - }paddr; -// struct tuple4 *paddr; -}; - -struct streaminfo -{ - struct ipaddr addr; //ip˿Ϣ - struct streaminfo *pfather;//ϲṹ - UCHAR type; // - UCHAR threadnum; // ߳ - UCHAR dir:2; // ķ0x01 c-->s 0x02 s-->c 0x03 c<-->s - UCHAR curdir:2; // ǰ0x01 c-->s 0x02 s-->c - UCHAR dirreverse:4; //ʱǷipַת - UCHAR opstate; //ǰ״̬ - UCHAR pktstate; //ӵİ - UCHAR routedir; //λΪ1ʾУΪ0ʾ; ڴģʽ,7λʾ豸ID - UCHAR __pad__[2]; //8ֽڶ - void *pproject; //ÿ̿Զʹã - void *pdetail; //ϸϢ -#if FOR_108 - unsigned long stream_id; -#endif -#if RAW_IP_FRAG_PKT - raw_ipfrag_list_t *ipfrag_list; /* 2013-08-21 LiJia add, ԭʼƬ */ -#endif - -#if FOR_863 - char terminal_tag[40]; /* ûǩ */ -#endif - }; - - -struct tcpdetail -{ - - void *pdata; // - UINT32 datalen; //ݳ - UINT32 lostlen; //ݳ - - UCHAR multisynflag:4; // multi syn - UCHAR needackflag:2; //Ҫϴack - UCHAR takeoverflag:2; - UCHAR tcpstateflag; // ڼ¼tcpĻỰSYN״̬ - UCHAR link_state; // ӵ״̬ - UCHAR pad[5]; - - struct half_tcpstream *pclient; //clientTCPϢ - struct half_tcpstream *pserver; // serverTCPϢ - UINT32 serverpkt; - UINT32 clientpkt; - UINT32 servercount; - UINT32 clientcount; - UINT64 creattime; - UINT64 lastmtime; - - UINT32 iserverseq; //ӽʱʱ洢seq - UINT32 iclientseq; //ӽʱʱ洢seq - void *apme; //Ӧò - void *pAllpktpme; //״̬tcp - struct proxy_node *pproxylist; //,ģ鸺ͷ - -}; -struct udpdetail -{ - void *pdata; // - UINT32 datalen; //ݳ - UINT32 pad; - UINT32 serverpkt; - UINT32 clientpkt; - UINT32 servercount; - UINT32 clientcount; - UINT64 creattime; - UINT64 lastmtime; - void *apme; //Ӧò -}; - - -//ҵýʱsession_state״̬ -#define SESSION_STATE_PENDING 0x01 -#define SESSION_STATE_DATA 0x02 -#define SESSION_STATE_CLOSE 0x04 - -//ҵʱķֵ -#define PROT_STATE_GIVEME 0x01 -#define PROT_STATE_DROPME 0x02 -#define PROT_STATE_KILLME 0x04 - -//ҵʱ -typedef struct _plugin_session_info -{ - unsigned short plugid; //plugidƽ̨ - char session_state; //Ự״̬PENDING,DATA,CLOSE - char _pad_; // - int buflen; //ǰֶγ - long long prot_flag; //ǰֶεflagֵ - void *buf; //ǰֶ - void* app_info; //Ϣ -}stSessionInfo; - -#ifdef __cplusplus -extern "C" { -#endif - -/* -a_* Ϣ -f_*: ӦĸϢ -a_packet: ԭʼip -pme: ˽ָ룬չãʱΪNULL -thread_seqзʱǰӵ߳ - -ֵΪĸֵ - - APP_STATE_GIVEME򱾺Ͱ - APP_STATE_DROPME򱾺Ͱ - APP_STATE_FAWPKTעݰ - APP_STATE_DROPPKTעݰ -*/ -char IP_ENTRY( struct streaminfo *f_stream,unsigned char routedir,int thread_seq,struct ip * a_packet); -char IPv6_ENTRY( struct streaminfo *f_stream,unsigned char routedir,int thread_seq,struct ip6_hdr *a_packet); -char TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet); -char UDP_ENTRY(struct streaminfo *a_udp, void **pme, int thread_seq,void *a_packet); - -int stream_register_tcp_allpkt (char (*x)(struct streaminfo *pstream,void **pme, int thread_seq,void *a_packet)); -int stream_register_tcp_takeover (char (*x)(struct streaminfo *pstream,void **pme, int thread_seq,void *a_packet)); - -int stream_register_tcp (char (*x)(struct streaminfo *pstream,void **pme, int thread_seq,void *a_packet)); -int stream_register_udp (char (*x)(struct streaminfo *pstream,void **pme, int thread_seq,void *a_packet)); -//int stream_register_ip (char (*x)(struct streaminfo *pstream,void **pme, int thread_seq,void *a_packet)); -int stream_register_ip (char (*x)(struct streaminfo *f_stream,unsigned char routedir,int thread_seq,struct ip * a_packet)); -int stream_register_ipv6 (char (*x)(struct streaminfo *f_stream,unsigned char routedir,int thread_seq,struct ip6_hdr *a_packet)); - -char * printaddr (struct ipaddr *paddrinfo,int threadindex); - - -void *dictator_malloc(int thread_seq,int size); -void dictator_free(int thread_seq,void *pbuf); - -char PROT_PROCESS(stSessionInfo* session_info, void **pme, int thread_seq,struct streaminfo *a_stream,void *a_packet); - - -/* ӦòȡжӦûǩ */ -int terminal_tag_probe(struct streaminfo *stream, struct tcphdr *this_tcphdr,void *rawippkt); -const unsigned char *get_terminal_tag(struct streaminfo *stream); -int MESA_kill_tcp(struct streaminfo *stream, struct ip *a_packet); - -#if RAW_IP_FRAG_PKT -extern raw_ipfrag_list_t **G_IP_FRAG_LIST; -void frags_list_free(raw_ipfrag_list_t *frags_list); -raw_ipfrag_list_t *frags_list_merge(raw_ipfrag_list_t *new_merge_list, raw_ipfrag_list_t *old_list); -#endif - -#if FOR_108 -unsigned long papp_get_streamid(); -#endif - - -#ifdef __cplusplus -} -#endif - - - -#endif diff --git a/src/inc_internal/MESA_handle_logger.h b/src/inc_internal/view_only/MESA_handle_logger.h similarity index 100% rename from src/inc_internal/MESA_handle_logger.h rename to src/inc_internal/view_only/MESA_handle_logger.h diff --git a/src/inc_internal/MESA_htable.h b/src/inc_internal/view_only/MESA_htable.h similarity index 100% rename from src/inc_internal/MESA_htable.h rename to src/inc_internal/view_only/MESA_htable.h diff --git a/src/inc_internal/MESA_list.h b/src/inc_internal/view_only/MESA_list.h similarity index 100% rename from src/inc_internal/MESA_list.h rename to src/inc_internal/view_only/MESA_list.h diff --git a/src/inc_internal/MESA_list_queue.h b/src/inc_internal/view_only/MESA_list_queue.h similarity index 100% rename from src/inc_internal/MESA_list_queue.h rename to src/inc_internal/view_only/MESA_list_queue.h diff --git a/src/inc_internal/field_stat.h b/src/inc_internal/view_only/field_stat.h similarity index 100% rename from src/inc_internal/field_stat.h rename to src/inc_internal/view_only/field_stat.h diff --git a/src/inc_internal/rulescan.h b/src/inc_internal/view_only/rulescan.h similarity index 100% rename from src/inc_internal/rulescan.h rename to src/inc_internal/view_only/rulescan.h diff --git a/src/entry/zt_hash.h b/src/inc_internal/zt_hash.h similarity index 100% rename from src/entry/zt_hash.h rename to src/inc_internal/zt_hash.h diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 0000000..e1a6223 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,16 @@ +add_executable(maat_demo maat_demo.cpp) +target_link_libraries(maat_demo maat_frame_shared) + +add_executable(test_maatframe test_maatframe.cpp) +target_link_libraries(test_maatframe maat_frame_shared gtest) + +configure_file(table_info.conf table_info.conf COPYONLY) +configure_file(maat_json.json maat_json.json COPYONLY) +configure_file(reset_redis4maat.sh reset_redis4maat.sh COPYONLY) +file(COPY conf DESTINATION ./) +file(COPY rule DESTINATION ./) +file(COPY testdata DESTINATION ./) +file(COPY testdata_uni2ascii DESTINATION ./) + +include(GoogleTest) +gtest_discover_tests(test_maatframe) diff --git a/test/Makefile b/test/Makefile deleted file mode 100644 index f020103..0000000 --- a/test/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -LIBS=../lib/libmaatframe.so -INC=-I../inc/ -I/usr/include/MESA/ -all: - g++ -o maat_test -g -Wall maat_test.cpp $(INC) $(LIBS) -lMESA_handle_logger -clean: - rm maat_test test.log* -f diff --git a/test/maat_test.cpp b/test/maat_demo.cpp similarity index 93% rename from test/maat_test.cpp rename to test/maat_demo.cpp index ee054ea..b59bc78 100644 --- a/test/maat_test.cpp +++ b/test/maat_demo.cpp @@ -558,7 +558,74 @@ void test_offset_str_scan(Maat_feather_t feather,const char* table_name) test_offset_str_scan_with_chunk(feather,table_name,1460); return; } +void test_compile_accept_tags(Maat_feather_t feather) +{ + int ret1=0, ret2=0; + int table_id=0; + scan_status_t mid=NULL; + struct Maat_rule_t result[4]; + const char* should_hit="string bbb should hit"; + const char* should_not_hit="string aaa should not hit"; + const char* table_name="HTTP_URL"; + table_id=Maat_table_register(feather,table_name); + if(table_id==-1) + { + printf("Database table %s register failed.\n",table_name); + return; + } + + ret1=Maat_full_scan_string(feather, table_id,CHARSET_GBK, should_not_hit, strlen(should_not_hit), + result,NULL, 4, + &mid, 0); + ret2=Maat_full_scan_string(feather, table_id,CHARSET_GBK, should_hit, strlen(should_hit), + result,NULL, 4, + &mid, 0); + if(ret1<=0&&ret2>0) + { + printf("Test compile accept tags success.\n"); + } + else + { + printf("Test compile accept tags failed.\n"); + } + Maat_clean_status(&mid); + return; +} +void accept_tags_entry_cb(int table_id,const char* table_line,void* u_para) +{ + char status[32]={0}; + int entry_id=-1,seq=-1; + int is_valid=0; + sscanf(table_line,"%d\t%s\t%d\t%d",&seq,status,&entry_id,&is_valid); + printf("Test plugin accept tags loading %d %s.\n",seq, status); + return; +} + + +void test_plugin_accept_tags(Maat_feather_t feather) +{ + int table_id=0,ret=0; + const char* table_name="TEST_EFFECTIVE_RANGE_TABLE"; + table_id=Maat_table_register(feather,table_name); + if(table_id==-1) + { + printf("Database table %s register failed.\n",table_name); + } + else + { + ret=Maat_table_callback_register(feather, table_id, + NULL, + accept_tags_entry_cb, + NULL, + NULL); + if(ret<0) + { + printf("Maat callback register table %s error.\n",table_name); + } + } + return; +} void test_longer_service_define(Maat_feather_t feather, struct Maat_rule_t* rule) { int ret=0; @@ -990,12 +1057,12 @@ void maat_test_print_usage(void) { printf("Maat Test Usage:\n"); printf("\tSource:\n"); - printf("\t\t-j Test updating from %s.\n",json_path); + printf("\t\t-j Test updating from %s. https://www.sojson.com/yasuo.html\n",json_path); printf("\t\t-u Load config from %s, and monitor %s. Need manually move inc index.\n",ful_cfg_dir,inc_cfg_dir); printf("\t\t-r Read config redis %s:%u db0.\n",test_maat_redis_ip,test_maat_redis_port); printf("\tOption:\n"); printf("\t\t-d Deferred Loading config.\n"); - printf("example: ./maat_test -j -d\n"); + printf("example: ./maat_demo -j -d\n"); return; } @@ -1009,6 +1076,7 @@ int main(int argc,char* argv[]) const char* stat_file="./scan_staus.log"; const char* decrypt_key="mesa2017wy"; const char* test_digest_file="./testdata/digest_test.data"; + const char* accept_tags="{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"}]}"; int scan_interval_ms=1; int effective_interval_ms=0; @@ -1065,13 +1133,11 @@ int main(int argc,char* argv[]) Maat_set_feather_opt(feather, MAAT_OPT_STAT_ON, NULL, 0); Maat_set_feather_opt(feather, MAAT_OPT_PERF_ON, NULL, 0); Maat_set_feather_opt(feather, MAAT_OPT_SCAN_DETAIL, &scan_detail, sizeof(scan_detail)); + Maat_set_feather_opt(feather, MAAT_OPT_ACCEPT_TAGS, accept_tags, strlen(accept_tags)+1); Maat_initiate_feather(feather); - if(feather==NULL) - { - printf("Maat initial error, see %s\n",log_file); - return -1; - } + printf("Maat initiating, see %s\n",log_file); + if(deferred_load_on==1) { printf("Deferred Load ON, Waiting...\n"); @@ -1122,6 +1188,10 @@ int main(int argc,char* argv[]) Maat_clean_status(&mid); test_offset_str_scan(feather,"IMAGE_FP"); + + test_plugin_accept_tags(feather); + test_compile_accept_tags(feather); + int value=0; if(1==using_redis) { diff --git a/test/maat_json.json b/test/maat_json.json index 6c5b602..cbd7868 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -8,7 +8,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "escaped\\bdata:have\\ba\\bspace\\band\\ba\\b\\&\\bsymbol.", "is_valid": "yes", "groups": [ @@ -74,7 +73,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "anything", "is_valid": "yes", "groups": [ @@ -102,7 +100,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "anything", "is_valid": "yes", "groups": [ @@ -128,7 +125,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "anything", "is_valid": "yes", "groups": [ @@ -168,7 +164,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "anything", "is_valid": "yes", "groups": [ @@ -194,7 +189,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "anything", "is_valid": "yes", "groups": [ @@ -222,7 +216,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "utf8_中文", "is_valid": "yes", "groups": [ @@ -249,7 +242,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "utf8_维语", "is_valid": "yes", "groups": [ @@ -276,7 +268,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "utf8_维语2", "is_valid": "yes", "groups": [ @@ -303,7 +294,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "string\\bunescape", "is_valid": "yes", "groups": [ @@ -330,7 +320,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "13018_table_conjunction_test_part1\bnow_its_very_very_long0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz", "is_valid": "yes", "groups": [ @@ -357,7 +346,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "table_conjunction_test_part2", "is_valid": "yes", "groups": [ @@ -384,7 +372,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "anything", "is_valid": "yes", "groups": [ @@ -409,7 +396,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "offset_string", "is_valid": "yes", "groups": [ @@ -436,7 +422,6 @@ "action": 1, "do_blacklist": 1, "do_log": 1, - "effective_range": 0, "user_region": "offset_string", "is_valid": "yes", "groups": [ @@ -456,6 +441,60 @@ ] } ] + }, + { + "compile_id": 138, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_range": 0, + "tags":"{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\",\"上海/浦东/陆家嘴\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"联通\"]}],[{\"tag\":\"location\",\"value\":[\"北京\"]},{\"tag\":\"isp\",\"value\":[\"联通\"]}]]}", + "user_region": "Not\\baccepted\\btags", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "should&hit&aaa", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 139, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_range": 0, + "tags":"{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"移动\"]}]]}", + "user_region": "Accepted\\btags", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "should&hit&bbb", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] } ], "plugin_table": [ @@ -474,6 +513,14 @@ "2\t3355\t66\t1", "3\tcccc\t11\t1" ] + }, + { + "table_name": "TEST_EFFECTIVE_RANGE_TABLE", + "table_content": [ + "1\tSUCCESS\t99\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"移动\"]}]]}", + "2\tSUCCESS\t66\t1\t0", + "3\tFAILED\t11\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\",\"上海/浦东/陆家嘴\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"联通\"]}],[{\"tag\":\"location\",\"value\":[\"北京\"]},{\"tag\":\"isp\",\"value\":[\"联通\"]}]]}" + ] } ] } diff --git a/test/maat_json.json_iris_tmp/COMPILE.local b/test/maat_json.json_iris_tmp/COMPILE.local deleted file mode 100644 index 9704fb1..0000000 --- a/test/maat_json.json_iris_tmp/COMPILE.local +++ /dev/null @@ -1,5 +0,0 @@ -0000000004 -123 1 1 1 1 0 anything 1 -124 1 1 1 1 0 anything 1 -125 1 1 1 1 0 anything 1 -126 1 1 1 1 0 anything 1 diff --git a/test/maat_json.json_iris_tmp/CONTENT_SIZE.local b/test/maat_json.json_iris_tmp/CONTENT_SIZE.local deleted file mode 100644 index e1bc0a6..0000000 --- a/test/maat_json.json_iris_tmp/CONTENT_SIZE.local +++ /dev/null @@ -1,3 +0,0 @@ -0000000002 -3 2 100 500 1 -6 5 2014 2016 1 diff --git a/test/maat_json.json_iris_tmp/GROUP.local b/test/maat_json.json_iris_tmp/GROUP.local deleted file mode 100644 index b65def4..0000000 --- a/test/maat_json.json_iris_tmp/GROUP.local +++ /dev/null @@ -1,8 +0,0 @@ -0000000007 -0 123 1 -1 123 1 -0 124 1 -2 124 1 -3 125 1 -4 126 1 -5 126 1 diff --git a/test/maat_json.json_iris_tmp/HTTP_URL.local b/test/maat_json.json_iris_tmp/HTTP_URL.local deleted file mode 100644 index b473eae..0000000 --- a/test/maat_json.json_iris_tmp/HTTP_URL.local +++ /dev/null @@ -1,4 +0,0 @@ -0000000003 -2 1 abckkk&123 1 0 0 1 -4 3 action=search\&query=(.*) 2 0 0 1 -5 4 should_not_hit_any_rule 0 0 0 1 diff --git a/test/maat_json.json_iris_tmp/IP_CONFIG.local b/test/maat_json.json_iris_tmp/IP_CONFIG.local deleted file mode 100644 index 7936e99..0000000 --- a/test/maat_json.json_iris_tmp/IP_CONFIG.local +++ /dev/null @@ -1,3 +0,0 @@ -0000000002 -0 0 4 10.0.6.201 255.255.0.0 0 65535 0.0.0.0 255.255.255.255 0 65535 6 0 1 -1 0 4 192.168.0.1 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 6 0 1 diff --git a/test/maat_json.json_iris_tmp/QD_ENTRY_INFO.local b/test/maat_json.json_iris_tmp/QD_ENTRY_INFO.local deleted file mode 100644 index cb1bd15..0000000 --- a/test/maat_json.json_iris_tmp/QD_ENTRY_INFO.local +++ /dev/null @@ -1,4 +0,0 @@ -0000000003 -1 192.168.0.1 101 -2 192.168.0.2 101 -3 192.168.1.1 102 diff --git a/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE.local b/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE.local deleted file mode 100644 index 763d635..0000000 --- a/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE.local +++ /dev/null @@ -1,4 +0,0 @@ -0000000003 -1 3388 99 1 -2 3355 66 1 -3 cccc 11 1 diff --git a/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE_NO1.local b/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE_NO1.local deleted file mode 100644 index b278b17..0000000 --- a/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE_NO1.local +++ /dev/null @@ -1,4 +0,0 @@ -0000000003 -1 plugname1 1 -2 plugname2 1 -3 plugname3 1 diff --git a/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE_NO2.local b/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE_NO2.local deleted file mode 100644 index 62580c9..0000000 --- a/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE_NO2.local +++ /dev/null @@ -1 +0,0 @@ -0000000000 diff --git a/test/maat_json.json_iris_tmp/index/full_config_index.0000000001 b/test/maat_json.json_iris_tmp/index/full_config_index.0000000001 deleted file mode 100644 index 74047de..0000000 --- a/test/maat_json.json_iris_tmp/index/full_config_index.0000000001 +++ /dev/null @@ -1,7 +0,0 @@ -COMPILE 4 ./maat_json.json_iris_tmp/COMPILE.local -GROUP 7 ./maat_json.json_iris_tmp/GROUP.local -TEST_PLUGIN_TABLE 3 ./maat_json.json_iris_tmp/TEST_PLUGIN_TABLE.local -IP_CONFIG 2 ./maat_json.json_iris_tmp/IP_CONFIG.local -CONTENT_SIZE 2 ./maat_json.json_iris_tmp/CONTENT_SIZE.local -HTTP_URL 3 ./maat_json.json_iris_tmp/HTTP_URL.local -QD_ENTRY_INFO 3 ./maat_json.json_iris_tmp/QD_ENTRY_INFO.local diff --git a/test/table_info.conf b/test/table_info.conf index ff53531..617e8fe 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -11,7 +11,7 @@ #id name type # #For plugin table -#id name type valid_column +#id name type valid_column tag_column # #For expr/expr_plus Table #id name type src_charset dst_charset do_merge cross_cache quick_mode @@ -22,8 +22,9 @@ 3 KEYWORDS_TABLE expr UTF8 GBK/BIG5/UNICODE/UTF8/unicode_ascii_esc/unicode_ascii_aligned/unicode_ncr_dec/unicode_ncr_hex yes 0 4 IP_CONFIG ip -- 5 CONTENT_SIZE intval -- -6 QD_ENTRY_INFO plugin 4 +6 QD_ENTRY_INFO plugin 4 -- 7 FILE_DIGEST digest -- 8 HTTP_REGION expr_plus GBK GBK no 0 9 SIM_URL similar -- 10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff +11 TEST_EFFECTIVE_RANGE_TABLE plugin 4 5 -- \ No newline at end of file diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp new file mode 100644 index 0000000..ea49a8c --- /dev/null +++ b/test/test_maatframe.cpp @@ -0,0 +1,996 @@ +#include "Maat_rule.h" +#include "stream_fuzzy_hash.h" +#include "Maat_command.h" +#include +#include +#include +#include //inet_addr +#include //inet_addr +#include //inet_addr +#include +#include //fstat +#include +#include +#include +#include +#include //fstat +#include //fstat +#include +#include + +#include +const char* test_maat_redis_ip="127.0.0.1"; +unsigned short test_maat_redis_port=6379; +const char* json_path="./maat_json.json"; +const char* ful_cfg_dir="./rule/full/index/"; +const char* inc_cfg_dir="./rule/inc/index/"; +#define WAIT_FOR_EFFECTIVE_US 1000*1000 +extern int my_scandir(const char *dir, struct dirent ***namelist, + int(*filter)(const struct dirent *), + int(*compar)(const void *, const void *)); + + +Maat_feather_t g_feather=NULL; +void *logger=NULL; +int g_iThreadNum=4; +const char* table_info_path="./table_info.conf"; +int scan_interval_ms=1; +int effective_interval_ms=0; + +void Maat_read_entry_start_cb(int update_type,void* u_para) +{ + return; +} +void Maat_read_entry_cb(int table_id,const char* table_line,void* u_para) +{ + char ip_str[16]={0}; + int entry_id=-1,seq=-1; + unsigned int ip_uint=0; + int is_valid=0; + unsigned int local_ip_nr=16820416;//192.168.0.1 + sscanf(table_line,"%d\t%s\t%d\t%d",&seq,ip_str,&entry_id,&is_valid); + inet_pton(AF_INET,ip_str,&ip_uint); + if(local_ip_nr==ip_uint) + { + if(is_valid==1) + { + //printf("Load entry id %d success.\n",entry_id); + EXPECT_EQ(entry_id, 101); + } + else + { + //printf("Offload entry id %d success.\n",entry_id); + } + } + return; +} +void Maat_read_entry_finish_cb(void* u_para) +{ + Maat_feather_t feather=u_para; + long long version=0; + int ret=0,is_last_updating_table=0; + ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version)); + EXPECT_EQ(ret, 0); + ret=Maat_read_state(feather,MAAT_STATE_LAST_UPDATING_TABLE, &is_last_updating_table, sizeof(is_last_updating_table)); + EXPECT_EQ(ret, 0); + //printf("Maat Version %lld at plugin finish callback, is_last_update=%d.\n",version,is_last_updating_table); + + return; +} +void test_plugin_table(Maat_feather_t feather,const char* table_name, + Maat_start_callback_t *start,Maat_update_callback_t *update,Maat_finish_callback_t *finish, + void *u_para, + void* logger) +{ + int table_id=0,ret=0; + table_id=Maat_table_register(feather,table_name); + ASSERT_GT(table_id, 0); + + ret=Maat_table_callback_register(feather, table_id, + start, + update, + finish, + u_para); + + ASSERT_GT(ret, 0); + +} + +TEST(PluginTable, Callback) +{ + test_plugin_table(g_feather, "QD_ENTRY_INFO", + Maat_read_entry_start_cb, + Maat_read_entry_cb, + Maat_read_entry_finish_cb, + g_feather, + logger); + +} +TEST(StringScan, Full) +{ + int ret=0; + int table_id=0; + struct Maat_rule_t result[4]; + int found_pos[4]; + const char* table_name="HTTP_URL"; + scan_status_t mid=NULL; + const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; + table_id=Maat_table_register(g_feather,table_name); + ASSERT_GT(table_id, 0); + + ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), + result,found_pos, 4, &mid, 0); + + EXPECT_GE(ret, 1); + Maat_clean_status(&mid); +} + +TEST(IPScan, IPv4) +{ + int table_id=0,ret=0; + const char* table_name="IP_CONFIG"; + struct Maat_rule_t result[4]; + scan_status_t mid=NULL; + struct ipaddr ipv4_addr; + struct stream_tuple4_v4 v4_addr; + ipv4_addr.addrtype=ADDR_TYPE_IPV4; + inet_pton(AF_INET,"10.0.6.205",&(v4_addr.saddr)); + v4_addr.source=htons(50001); + inet_pton(AF_INET,"10.0.6.201",&(v4_addr.daddr)); + v4_addr.dest=htons(80); + ipv4_addr.v4=&v4_addr; + + const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; + table_id=Maat_table_register(g_feather,"HTTP_URL"); + ASSERT_GT(table_id, 0); + + ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), + result,NULL, 4, &mid, 0); + EXPECT_GE(ret, 1); + + table_id=Maat_table_register(g_feather,table_name); + + EXPECT_GT(table_id, 0); + + ret=Maat_scan_proto_addr(g_feather,table_id,&ipv4_addr,6,result,4, &mid,0); + + EXPECT_GT(ret, 0); + Maat_clean_status(&mid); + return; +} +TEST(IPScan, IPv6) + +{ + int table_id=0,ret=0; + struct Maat_rule_t result[4]; + struct ipaddr ipv6_addr; + struct stream_tuple4_v6 v6_addr; + scan_status_t mid=NULL; + + ipv6_addr.addrtype=ADDR_TYPE_IPV6; + inet_pton(AF_INET6,"2001:da8:205:1::101",&(v6_addr.saddr)); + v6_addr.source=htons(50001); + inet_pton(AF_INET6,"2001:da8:205:1::102",&(v6_addr.daddr)); + v6_addr.dest=htons(80); + ipv6_addr.v6=&v6_addr; + const char* table_name="IP_CONFIG"; + table_id=Maat_table_register(g_feather,table_name); + EXPECT_GT(table_id, 0); + + //for improving performance. + Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION,NULL, 0); + ret=Maat_scan_proto_addr(g_feather,table_id,&ipv6_addr,6,result,4, &mid,0); + EXPECT_EQ(ret, -2); + Maat_clean_status(&mid); + return; +} + +TEST(IntervalScan, Pure) +{ + int table_id=0,ret=0; + int scan_val=2015; + struct Maat_rule_t result[4]; + const char* table_name="CONTENT_SIZE"; + scan_status_t mid=NULL; + table_id=Maat_table_register(g_feather,table_name); + ASSERT_GT(table_id, 0); + + ret=Maat_scan_intval(g_feather, table_id, scan_val, result,4, &mid, 0); + EXPECT_EQ(ret, -2); + Maat_clean_status(&mid); + return; +} +TEST(DigestScan, Pure) +{ + int table_id=0,ret=0,hit_cnt=0; + struct stat digest_fstat; + unsigned long long read_size=0,scan_offset=0; + char digest_test_buff[4096]={0}; + const char* file_name="./testdata/digest_test.data"; + const char* table_name="FILE_DIGEST"; + struct Maat_rule_t result[4]; + stream_para_t sp=NULL; + scan_status_t mid=NULL; + table_id=Maat_table_register(g_feather, table_name); + ASSERT_GT(table_id, 0); + + ret=stat(file_name,&digest_fstat); + ASSERT_EQ(ret, 0); + + FILE* fp=fopen(file_name,"r"); + ASSERT_FALSE(fp==NULL); + + sp=Maat_stream_scan_digest_start(g_feather, table_id, digest_fstat.st_size, 0); + while(0==feof(fp)) + { + read_size=fread(digest_test_buff,1,sizeof(digest_test_buff),fp); + ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4, &mid); + scan_offset+=read_size; + if(ret>0) + { + hit_cnt++; + + } + } + fclose(fp); + Maat_stream_scan_string_end(&sp); + EXPECT_GE(hit_cnt, 1); + Maat_clean_status(&mid); + return; +} +TEST(StringScan, EncodedURL) +{ + const char* url_utf8="www.google.com/?q=C%23%E4%B8%AD%E5%9B%BD"; + const char* url_gb2312="www.baidu.com/?wd=C%23%D6%D0%B9%FA"; + int table_id=0,ret=0; + struct Maat_rule_t result[4]; + int found_pos[4]; + + const char* table_name="HTTP_URL"; + scan_status_t mid=NULL; + + table_id=Maat_table_register(g_feather,table_name); + ASSERT_GT(table_id, 0); + + ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url_utf8, strlen(url_utf8), + result,found_pos, 4, + &mid, 0); + EXPECT_GE(ret, 1); + Maat_clean_status(&mid); + + ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url_gb2312, strlen(url_gb2312), + result,found_pos, 4, + &mid, 0); + EXPECT_GE(ret, 1); + Maat_clean_status(&mid); + return; +} +TEST(StringScan, UnicodeEscape) +{ + const char* test_data_dir="./testdata_uni2ascii"; + struct dirent **namelist; + FILE* fp=NULL; + char file_path[256]={0}; + char buff[4096]; + size_t read_len=0; + int table_id=0,ret=0; + struct Maat_rule_t result[4]; + stream_para_t sp=NULL; + int n=0,i=0, hit_cnt=0; + const char* table_name="KEYWORDS_TABLE"; + scan_status_t mid=NULL; + + table_id=Maat_table_register(g_feather,table_name); + ASSERT_GT(table_id, 0); + + n = my_scandir(test_data_dir, &namelist, NULL, (int (*)(const void*, const void*))alphasort); + ASSERT_GT(n, 0); + + for(i=0;id_name, ".") == 0) || (strcmp(namelist[i]->d_name, "..") == 0)) + { + continue; + } + snprintf(file_path,sizeof(file_path),"%s/%s",test_data_dir,namelist[i]->d_name); + fp=fopen(file_path,"rb"); + + if(fp==NULL) + { + printf("fopen %s error.\n",file_path);; + continue; + } + sp=Maat_stream_scan_string_start(g_feather,table_id,0); + ASSERT_FALSE(sp==NULL); + + read_len=fread(buff,1,sizeof(buff),fp); + while(read_len>0) + { + + ret=Maat_stream_scan_string(&sp,CHARSET_NONE,buff,read_len + ,result, NULL, 4, &mid); + read_len=fread(buff,1,sizeof(buff),fp); + if(ret>0) + { + hit_cnt++; + } + } + Maat_stream_scan_string_end(&sp); + fclose(fp); + EXPECT_GT(hit_cnt, 0); + EXPECT_GE(result[0].config_id, 130);//130, 131 + Maat_clean_status(&mid); + } + for(i=0;i0) + { + pass_flag=1; + break; + } + } + EXPECT_EQ(pass_flag, 1); + EXPECT_EQ(result[0].config_id, 136); + + Maat_stream_scan_string_end(&sp); + free(hit_detail); + fclose(fp); + Maat_clean_status(&mid); + return; +} + +TEST(StringScan, OffsetChunk64) +{ + test_offset_str_scan_with_chunk(64); + return; +} +TEST(StringScan, OffsetChunk1460) +{ + + test_offset_str_scan_with_chunk(1460); + return; +} +void accept_tags_entry_cb(int table_id,const char* table_line,void* u_para) +{ + char status[32]={0}; + int entry_id=-1,seq=-1; + int is_valid=0; + sscanf(table_line,"%d\t%s\t%d\t%d",&seq,status,&entry_id,&is_valid); + EXPECT_STREQ(status ,"SUCCESS"); + + return; +} + + +TEST(RuleTags, Plugin) +{ + int table_id=0,ret=0; + const char* table_name="TEST_EFFECTIVE_RANGE_TABLE"; + table_id=Maat_table_register(g_feather,table_name); + ASSERT_GT(table_id, 0); + + ret=Maat_table_callback_register(g_feather, table_id, + NULL, + accept_tags_entry_cb, + NULL, + NULL); + + ASSERT_GE(ret, 0); + return; +} + +TEST(RuleTags, Compile) +{ + int ret=0; + int table_id=0; + scan_status_t mid=NULL; + struct Maat_rule_t result[4]; + const char* should_hit="string bbb should hit"; + const char* should_not_hit="string aaa should not hit"; + const char* table_name="HTTP_URL"; + table_id=Maat_table_register(g_feather,table_name); + ASSERT_GT(table_id, 0); + + ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, should_not_hit, strlen(should_not_hit), + result,NULL, 4, + &mid, 0); + + EXPECT_LE(ret, 0); + + ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, should_hit, strlen(should_hit), + result,NULL, 4, + &mid, 0); + EXPECT_EQ(ret, 1); + Maat_clean_status(&mid); + return; +} + + +TEST(StreamFuzzyHash, Pure) +{ + const size_t FILE_CHUNK_SIZE=4096; + char * file_buff=NULL,*sfh_ordered=NULL,*sfh_unorder=NULL; + int read_size=0,ret=0,chunk_num=0,i=0,idx=0; + unsigned long long *offset=NULL; + unsigned long long file_size=0,tmp=0,hash_length=0; + const char* filename="./testdata/digest_test.data"; + + FILE* fp=fopen(filename,"r"); + sfh_instance_t * fhandle = NULL; + struct stat file_info; + ret=stat(filename, &file_info); + ASSERT_TRUE(ret==0); + + file_size=file_info.st_size; + file_buff=(char*)malloc(file_size); + ret=fread(file_buff,1,file_size,fp); + ASSERT_TRUE((unsigned long long)ret==file_size); + + chunk_num=file_size/FILE_CHUNK_SIZE; + if(file_size%FILE_CHUNK_SIZE==0) + { + chunk_num=file_size/FILE_CHUNK_SIZE; + } + else + { + chunk_num=file_size/FILE_CHUNK_SIZE+1; + } + offset=(unsigned long long*)malloc(sizeof(unsigned long long)*chunk_num); + for(i=0;ifile_size) + { + read_size=file_size-offset[i]; + } + else + { + read_size=FILE_CHUNK_SIZE; + } + SFH_feed(fhandle,file_buff+offset[i],read_size,offset[i]); + } + hash_length = SFH_status(fhandle, HASH_LENGTH); + sfh_unorder=(char*)malloc(hash_length); + SFH_digest(fhandle, sfh_unorder, hash_length); + //printf("%s %u %lf %s\n",path,digest_fstat.st_size,file_entropy,digest_result_buff); + SFH_release(fhandle); + EXPECT_STREQ(sfh_ordered, sfh_unorder); + + fclose(fp); + free(file_buff); + free(sfh_ordered); + free(sfh_unorder); + free(offset); + return; +} +TEST(ScanResult, LongerServiceDefine) +{ + int ret=0; + int table_id=0; + scan_status_t mid=NULL; + + struct Maat_rule_t result[4]; + const char* scan_data="soq is using table conjunction function.http://www.3300av.com/novel/27122.txt"; + const char* table_name="HTTP_URL"; + + table_id=Maat_table_register(g_feather,table_name); + ASSERT_GT(table_id, 0); + + ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), + result, NULL, 4, + &mid, 0); + EXPECT_EQ(ret, 2); + EXPECT_EQ(result[1].config_id, 133); + EXPECT_GT(result[1].serv_def_len, 128); + + char* buff=(char*)malloc(sizeof(char)*result[1].serv_def_len); + ret=Maat_read_rule(g_feather, result+1, MAAT_RULE_SERV_DEFINE, buff, result[1].serv_def_len); + EXPECT_EQ(ret, result[1].serv_def_len); + Maat_clean_status(&mid); + free(buff); + return; +} +class MaatCmdTest : public testing::Test +{ + +protected: + + static void SetUpTestCase() + { + void *logger=NULL; + logger=MESA_create_runtime_log_handle("test_maat_redis.log",0); + + _shared_feather=Maat_feather(g_iThreadNum, table_info_path, logger); + Maat_set_feather_opt(_shared_feather,MAAT_OPT_INSTANCE_NAME,"redis", strlen("redis")+1); + Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_IP, test_maat_redis_ip, strlen(test_maat_redis_ip)+1); + Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_PORT, &test_maat_redis_port, sizeof(test_maat_redis_port)); + Maat_set_feather_opt(_shared_feather, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms)); + //Set a short intevral for testing. + Maat_set_feather_opt(_shared_feather, MAAT_OPT_EFFECT_INVERVAL_MS,&effective_interval_ms, sizeof(effective_interval_ms)); + Maat_initiate_feather(_shared_feather); + Maat_cmd_flushDB(_shared_feather); + } + static void TearDownTestCase() + { + Maat_burn_feather(_shared_feather); + + } + // Some expensive resource shared by all tests. + static Maat_feather_t _shared_feather; +}; +Maat_feather_t MaatCmdTest::_shared_feather; + +int test_add_expr_command(Maat_feather_t feather,const char* region_table,int config_id, int timeout,int label_id, const char* keywords) +{ + struct Maat_cmd_t* cmd=NULL; + struct Maat_rule_t rule; + char huge_serv_def[1024*2]; + memset(huge_serv_def,'s',sizeof(huge_serv_def)); + struct Maat_region_t region; + int group_num=1,ret=0; + memset(&rule,0,sizeof(rule)); + rule.config_id=config_id; + + strcpy(rule.service_defined,"maat_command"); + //MUST acqire by function, because Maat_cmd_t has some hidden members. + cmd=Maat_create_cmd(&rule, group_num); + cmd->expire_after=timeout; + cmd->label_id=label_id; + memset(®ion,0,sizeof(region)); + region.region_type=REGION_EXPR; + region.table_name=region_table; + region.expr_rule.district=NULL; + region.expr_rule.keywords=keywords; + region.expr_rule.expr_type=EXPR_TYPE_AND; + region.expr_rule.match_method=MATCH_METHOD_SUB; + region.expr_rule.hex_bin=UNCASE_PLAIN; + Maat_cmd_set_opt(cmd, MAAT_RULE_SERV_DEFINE, huge_serv_def, sizeof(huge_serv_def)); + Maat_add_region2cmd(cmd, 0, ®ion); + //use pipeline model. + ret=Maat_cmd_append(feather, cmd, MAAT_OP_ADD); + if(ret<0) + { + printf("Add Maat command %d failed.\n",rule.config_id); + Maat_free_cmd(cmd); + return 0; + } + //cmd has been saved in feather, so free cmd before commit is allowed. + Maat_free_cmd(cmd); + ret=Maat_cmd_commit(feather); + if(ret<0) + { + printf("Commit Maat command %d failed.\n",rule.config_id); + } + return 0; + +} +int del_command(Maat_feather_t feather,int config_id) +{ + struct Maat_cmd_t* cmd=NULL; + struct Maat_rule_t rule; + int ret=0; + memset(&rule,0,sizeof(rule)); + rule.config_id=config_id; + cmd=Maat_create_cmd(&rule, 0); + ret=Maat_cmd(feather, cmd, MAAT_OP_DEL); + if(ret<0) + { + printf("Delete Maat command %d failed.\n",rule.config_id); + } + Maat_free_cmd(cmd); + return 0; +} + +TEST_F(MaatCmdTest, Expr) +{ + const char* scan_data="Hiredis is a minimalistic C client library for the Redis database.\r\n"; + const char* table_name="HTTP_URL"; + + const char* keywords1="Hiredis"; + const char* keywords2="C Client"; + char escape_buff1[256],escape_buff2[256]; + char keywords[256]; + scan_status_t mid=NULL; + int config_id=-1, table_id=0, ret=0; + int output_ids[4]; + int output_id_cnt=0; + struct Maat_rule_t result; + int timeout=0;//seconds + int label_id=5210; + long long version_before=0,version_after=0; + Maat_feather_t feather=MaatCmdTest::_shared_feather; + + Maat_str_escape(escape_buff1, sizeof(escape_buff1),keywords1); + Maat_str_escape(escape_buff2, sizeof(escape_buff2),keywords2); + snprintf(keywords,sizeof(keywords),"%s&%s",escape_buff1,escape_buff2); + config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1); + + ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before)); + test_add_expr_command(feather,table_name,config_id, 0, label_id, keywords); + usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect + + ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_after, sizeof(version_after)); + EXPECT_EQ(ret, 0); + table_id=Maat_table_register(feather,table_name); + ASSERT_GT(table_id, 0); + ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), + &result,NULL, 1, + &mid, 0); + EXPECT_GT(ret, 0); + EXPECT_EQ(result.config_id, config_id); + + Maat_clean_status(&mid); + output_id_cnt=Maat_cmd_select(feather,label_id, output_ids, 4); + EXPECT_EQ(output_id_cnt, 1); + EXPECT_EQ(output_ids[0], config_id); + + + del_command(feather, config_id); + usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect + ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), + &result,NULL, 1, + &mid, 0); + EXPECT_EQ(ret, 0); + Maat_clean_status(&mid); + + timeout=1; + config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1); + test_add_expr_command(feather,table_name,config_id, timeout, label_id, keywords); + usleep(timeout*1000*1000+WAIT_FOR_EFFECTIVE_US); + ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), + &result,NULL, 1, + &mid, 0); + EXPECT_EQ(ret, 0); +} +TEST_F(MaatCmdTest, Lines) +{ + const int TEST_CMD_LINE_NUM=4; + const struct Maat_line_t *p_line[TEST_CMD_LINE_NUM]; + struct Maat_line_t line_rule[TEST_CMD_LINE_NUM]; + char table_line[TEST_CMD_LINE_NUM][128]; + int ret=0,i=0; + Maat_feather_t feather=MaatCmdTest::_shared_feather; + memset(&line_rule,0,sizeof(line_rule)); + for(i=0;i