commit 40f0ed7320dae6af64e75dd7d925c55afbd76249 Author: zhengchao Date: Sat Oct 10 18:30:12 2015 +0800 Began use git manage source code since MAAT_FRAME_VERSION_1_2_20150724. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..60763b1 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +#opt: OPTFLAGS = -O2 +#export OPTFLAGS + +.PHONY: all clean opt + +all: + cd src/entry/ && $(MAKE) + cd test && $(MAKE) +clean: + cd src/entry/ && $(MAKE) clean + cd test && $(MAKE) clean + +opt: + $(MAKE) all diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h new file mode 100644 index 0000000..b2f5c1c --- /dev/null +++ b/inc/Maat_rule.h @@ -0,0 +1,177 @@ + +/* +*****************Maat Network Flow Rule Manage Framework******** +* Maat is the Goddess of truth and justice in ancient Egyptian concept. +* Her feather was the measure that determined whether the souls (considered +* to reside in the heart) of the departed would reach the paradise of afterlife +* successfully. +* Author: zhengchao@iie.ac.cn,MESA +* Version 2015-07-24 Maat_set_feather_opt +* NOTE: MUST compile with G++ +* All right reserved by Institute of Infomation Engineering,Chinese Academic of Science 2014~2018 +********************************************************* +*/ +#ifndef H_MAAT_RULE_H_INCLUDE +#define H_MAAT_RULE_H_INCLUDE +#include "stream.h" +enum MAAT_CHARSET +{ + CHARSET_NONE=0, + CHARSET_GBK, + CHARSET_BIG5, + CHARSET_UNICODE, + CHARSET_UTF8, // 4 + CHARSET_BIN //5 +}; +enum MAAT_ACTION +{ + MAAT_ACTION_BLOCK=0, + MAAT_ACTION_MONIT, + MAAT_ACTION_WHITE +}; +enum MAAT_POS_TYPE +{ + MAAT_POSTYPE_EXPR=0, + MAAT_POSTYPE_REGEX +}; +typedef void* scan_status_t; +typedef void* stream_para_t; +typedef void* Maat_feather_t; +#define MAX_SERVICE_DEFINE_LEN 128 +struct Maat_rule_t +{ + int config_id; + int service_id; + char do_log; + char do_blacklist; + char action; + char resevered; + int serv_def_len; + char service_defined[MAX_SERVICE_DEFINE_LEN]; +}; +#define MAAT_RULE_UPDATE_TYPE_FULL 1 +#define MAAT_RULE_UPDATE_TYPE_INC 2 +typedef void Maat_start_callback_t(int update_type,void* u_para); +typedef void Maat_update_callback_t(int table_id,const char* table_line,void* u_para); +typedef void Maat_finish_callback_t(void* u_para); + + + + + +//--------------------HITTING DETAIL DESCRIPTION BEGIN + +#define MAAT_MAX_HIT_RULE_NUM 8 +#define MAAT_MAX_EXPR_ITEM_NUM 8 +#define MAAT_MAX_HIT_POS_NUM 8 +#define MAAT_MAX_REGEX_GROUP_NUM 8 + +//NOTE position buffer as hitting_regex_pos and hit_pos,are ONLY valid before next scan or Maat_stream_scan_string_end +struct regex_pos_t +{ + int group_num; + int hitting_regex_len; + const char* hitting_regex_pos; + int grouping_len[MAAT_MAX_REGEX_GROUP_NUM]; + const char* grouping_pos[MAAT_MAX_REGEX_GROUP_NUM]; +}; +struct str_pos_t +{ + int hit_len; + const char* hit_pos; +}; +struct sub_item_pos_t +{ + enum MAAT_POS_TYPE ruletype; + int hit_cnt; + union + { + struct regex_pos_t regex_pos[MAAT_MAX_HIT_POS_NUM]; + struct str_pos_t substr_pos[MAAT_MAX_HIT_POS_NUM]; + }; +}; + +struct Maat_region_pos_t +{ + + int region_id; + int sub_item_num; + struct sub_item_pos_t sub_item_pos[MAAT_MAX_EXPR_ITEM_NUM]; +}; + +struct Maat_hit_detail_t +{ + int config_id;//set <0 if half hit; + int hit_region_cnt; + struct Maat_region_pos_t region_pos[MAAT_MAX_HIT_RULE_NUM]; +}; +//--------------------HITTING DETAIL DESCRIPTION END +Maat_feather_t Maat_summon_feather(int max_thread_num, + const char* table_info_path, + const char* ful_cfg_dir, + const char* inc_cfg_dir, + void*logger); + +Maat_feather_t Maat_summon_feather_json(int max_thread_num, + const char* table_info_path, + const char* json_rule, + void* logger); + +void Maat_burn_feather(Maat_feather_t feather); + +//return table_id(>=0) if success,otherwise return -1; +int Maat_table_register(Maat_feather_t feather,const char* table_name); +//return 1 if success,otherwise return -1 incase invalid table_id or registed function number exceed 32; +int Maat_table_callback_register(Maat_feather_t feather,short table_id, + Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para + Maat_update_callback_t *update,//table line ,u_para + Maat_finish_callback_t *finish,//u_para + void* u_para); + + + +//Return hit rule number, return -1 when error occurs,return -2 when hit current region +//mid MUST set NULL before fist call +int Maat_scan_intval(Maat_feather_t feather,int table_id + ,unsigned int intval + ,struct Maat_rule_t*result,int rule_num + ,scan_status_t *mid,int thread_num); +int Maat_scan_addr(Maat_feather_t feather,int table_id + ,struct ipaddr* addr + ,struct Maat_rule_t*result,int rule_num + ,scan_status_t *mid,int thread_num); +int Maat_scan_proto_addr(Maat_feather_t feather,int table_id + ,struct ipaddr* addr,unsigned short int proto + ,struct Maat_rule_t*result,int rule_num + ,scan_status_t *mid,int thread_num); +int Maat_full_scan_string(Maat_feather_t feather,int table_id + ,enum MAAT_CHARSET charset,const char* data,int data_len + ,struct Maat_rule_t*result,int* found_pos,int rule_num + ,scan_status_t* mid,int thread_num); +//hite_detail could be NULL if unconcern +int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id + ,enum MAAT_CHARSET charset,const char* data,int data_len + ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num + ,int* detail_ret,scan_status_t* mid,int thread_num); + +stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,int thread_num); +int Maat_stream_scan_string(stream_para_t* stream_para + ,enum MAAT_CHARSET charset,const char* data,int data_len + ,struct Maat_rule_t*result,int* found_pos,int rule_num + ,scan_status_t* mid); +//hite_detail could be NULL if unconcern +int Maat_stream_scan_string_detail(stream_para_t* stream_para + ,enum MAAT_CHARSET charset,const char* data,int data_len + ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num + ,int* detail_ret,scan_status_t* mid); +void Maat_stream_scan_string_end(stream_para_t* stream_para); + +void Maat_clean_status(scan_status_t* mid); + +#define MAAT_OPT_SCANDIR_INTERVAL_MS 1 //opt value is interger,size=sizeof(int),1,000 milliseconds as default +#define MAAT_OPT_EFFECT_INVERVAL_MS 2 //opt value is interger,size=sizeof(int),60,000 milliseconds as default +//return -1 if failed, return 0 on success; +int Maat_set_feather_opt(Maat_feather_t feather,int type,void* value,int size); + +#endif // H_MAAT_RULE_H_INCLUDE + diff --git a/readme.txt b/readme.txt new file mode 100644 index 0000000..f8d13ae --- /dev/null +++ b/readme.txt @@ -0,0 +1,35 @@ +Author:Zheng Chao zhengchao@iie.ac.cn +2014-04-01 create this project; +2014-08-19 first online version; +2014-09-09 support expr offset; +2014-09-26 add group function; +2014-12-03 support regex grouping; +2014-12-17 write some comments in Maat_rule.h; +2014-12-22 add group_num in compile config table; +2015-01-04 make string match not case sensitive,fix garbage_bagging malloc wrong size bug; +2015-01-07 replace assert of database rule format,unescape '&' of EXPR_TYPE_OFFSET; +2015-01-22 ouput IRIS index file format error;verify string scan input data +and data len; +2015-01-28 enfore parameter check in Maat_stream_scan_string_detail to +optimize performance; +2015-02-04 support stream scan data with offset; +2015-02-20 add JSON config mode and add a demo; +2015-03-02 when use iconv_convert from gbk to big5,automaticly alternate gbk to +gb2312; +2015-04-13 1)Maat_xx_scan will return 0 if table has no config instead of -1; + 2)use my_scandir as replacement of glibc's scandir to adapt dictator + malloc wrap; + 3)if iconv take no effect,jump over this string; + 4)compile shortcut for performance; + 5)verify if region id is unique; +2015-04-20 fix Maat_stream_scan_string_detail and Maat_stream_scan_string wrong + data len when open cross caching; +2015-04-21 fix add_group_to_compile bug on compatible to none-group mode; +2015-04-29 fix false hit compile rule of one more region bug; +2015-05-06 fix add_group_to_compile return add failed status bug; +2015-05-07 1)use rwlock instead of mutex in _mi_rule; + 2)fix inc postpone update bug; +2015-05-21 1)addapt rulescan ip addr host order requirement; + 2) use readdir_r in my_scandir; + 3) fix segmentfault when have no config,again; + 4) maat_json support plugin table; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp new file mode 100644 index 0000000..8875045 --- /dev/null +++ b/src/entry/Maat_rule.cpp @@ -0,0 +1,3130 @@ +#include +#include +#include //tolower +#include +#include //inet_pton +#include //inet_pton +#include //inet_pton +#include +#include +#include +#include +#include +#include + +#include "Maat_rule.h" +#include "Maat_rule_internal.h" +#include "json2iris.h" +#include "dynamic_array.h" +#include "config_monitor.h" +#include "map_str2int.h" +#include "rulescan.h" +#include "UniversalBoolMatch.h" + +int MAAT_FRAME_VERSION_1_2_20150724=1; +const char *maat_module="MAAT Frame"; +const char* CHARSET_STRING[]={"CHARSET_NONE","GBK","BIG5","UNICODE","UTF-8"}; + +int converHextoint(char srctmp) +{ + if(isdigit(srctmp)) + { + return srctmp-'0'; + } + else + { + char temp=toupper(srctmp); + temp=temp-'A'+10; + return temp; + } +} +int hex2bin(char *hex,int hex_len,char *binary,int size) +{ + int i=0; + int resultlen=0; + int high,low; + for(i=0;iresultlen; i+=2,resultlen++) + { + high=converHextoint(hex[i]); + low=converHextoint(hex[i+1]); + binary[resultlen]=high*16+low; + } + size=resultlen; + binary[resultlen]='\0'; + return resultlen; +} +iconv_t maat_iconv_open(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from) +{ + const char *from_s=CHARSET_STRING[from]; + const char *to_s=CHARSET_STRING[to]; + iconv_t cd; + if(from==CHARSET_GBK&&to==CHARSET_BIG5) + { + from_s="gb2312"; + } + if(from>=MAX_CHARSET_NUM||to>=MAX_CHARSET_NUM) + { + return (iconv_t)-1; + } + + if(scanner->iconv_handle[to][from]==NULL) + { + scanner->iconv_handle[to][from]=iconv_open(to_s, from_s); + } + cd=scanner->iconv_handle[to][from]; + return cd; +} +int iconv_convert(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen) +{ + size_t ret; + int copy_len=0; + char* copy_buf=NULL; + if(srclen==0||src==NULL) + { + return -1; + } + iconv_t cd=maat_iconv_open(scanner,to, from); + if(cd!=(iconv_t)-1) + { + char * pInBuff=src; + size_t iInBuffLen=srclen; + + size_t iOutBuffLen=10*iInBuffLen; + char * pOutBuff=(char *)malloc(iOutBuffLen); + + char * pLeftBuff=pOutBuff; + size_t iLeftLen=iOutBuffLen; + + ret=iconv(cd, &pInBuff, &iInBuffLen, &pLeftBuff, &iLeftLen); + if(ret!=(size_t)(-1)) + { + + if(to==CHARSET_UNICODE)//jump unicode 2 bytes head 0xFF 0xFE + { + copy_len=iOutBuffLen-iLeftLen-2; + copy_buf=pOutBuff+2; + } + else + { + copy_len=iOutBuffLen-iLeftLen; + copy_buf=pOutBuff; + } + assert(copy_len<=*dstlen); + *dstlen=copy_len; + memcpy(dst,copy_buf,*dstlen); + + free(pOutBuff); + return 1; + } + else + { + free(pOutBuff); + return -1; + } + } + else + { + return -1; + } + +} + +char* strlwr(char* string) +{ + int i=0; + for(i=0;i<(int)strlen(string);i++) + { + string[i]=(char)tolower(string[i]); + } + return string; +} +char * strchr_esc(char* s,const char delim) +{ + char *token; + if(s==NULL) + return NULL; + for(token=s;*token!='\0';token++) + { + if(*token=='\\') + { + token++; + continue; + } + if(*token==delim) + break; + } + if (*token == '\0') + { + return NULL; + } + else + { + return token; + } +} +char *strtok_r_esc(char *s, const char delim, char **save_ptr) { + char *token; + + if (s == NULL) s = *save_ptr; + + /* Scan leading delimiters. */ + token=strchr_esc(s,delim); + if(token==NULL) + { + *save_ptr=token; + return s; + } + /* Find the end of the token. */ + *token='\0'; + token++; + *save_ptr=token; + + return s; +} +char *str_unescape_and(char*s) +{ + int i=0,j=0; + for(i=0,j=0;i<(int)strlen(s);i++) + { + if(s[i]=='\\'&&s[i+1]=='&') + { + s[j]='&'; + i++; + j++; + } + else{ + s[j]=s[i]; + j++; + } + } + s[j]='\0'; + return s; +} +int cnt_maskbits(struct in6_addr mask) +{ + unsigned int i=0; + int bits_cnt=0; + unsigned char* p=(unsigned char*)&mask; + for(i=0;i0;p[i]=p[i]/2) + { + bits_cnt++; + } + } + return bits_cnt; +} +inline void ipv6_ntoh(unsigned int *v6_addr) +{ + unsigned int i=0; + for(i=0;i<4;i++) + { + v6_addr[i]=ntohl(v6_addr[i]); + } + return; +} +int aligment_int_array_sum(int * array,int size) +{ + int sum=0,i=0; + int offset=0; + for(i=0;icross_cache_size=0; + sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d",&(p->table_id) + ,p->table_name + ,table_type + ,src_charset + ,dst_charset + ,merge + ,&(p->cross_cache_size)); + ret[0]=map_str2int(string2int_map,strlwr(table_type),(int*)&(p->table_type)); + ret[1]=map_str2int(string2int_map,strlwr(src_charset),(int*)&(p->src_charset)); + ret[2]=map_str2int(string2int_map,strlwr(merge),&(p->do_charset_merge)); + for(j=0;j<3;j++) + { + if(ret[j]<0) + { + fprintf(stderr,"Maat read table info %s line %d error.\n",table_info_path,i); + goto error_jump; + } + } + j=0; + for (token = dst_charset; ; token= NULL) + { + sub_token= strtok_r(token,"/", &saveptr); + if (sub_token == NULL) + break; + ret[3]=map_str2int(string2int_map,strlwr(sub_token),(int*)&(p->dst_charset[j])); + if(ret[3]>0) + { + if(p->dst_charset[j]==p->src_charset) + { + p->src_charset_in_dst=TRUE; + } + j++; + } + else + { + fprintf(stderr,"Maat read table info %s line %d error.\n",table_info_path,i); + goto error_jump; + } + + } + if(p->table_id>=num) + { + fprintf(stderr,"Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,num); + goto error_jump; + } + if(p_table_info[p->table_id]!=NULL) + { + fprintf(stderr,"Maat read table info %s line %d error:duplicated table id %d.\n",table_info_path,i,p->table_id); + goto error_jump; + } + if(p->table_type==TABLE_TYPE_PLUGIN) + { + p->cb_info=(struct _plugin_table_info*)calloc(sizeof(struct _plugin_table_info),1); + p->cb_info->cache_lines=dynamic_array_create(1024,1024); + } + p_table_info[p->table_id]=p; + table_cnt++; + continue; +error_jump: + free(p); + p=NULL; + } + fclose(fp); + map_destroy(string2int_map); + return table_cnt; +} +struct _Maat_group_rule_t* create_group_rule(int group_id) +{ + struct _Maat_group_rule_t* group=(struct _Maat_group_rule_t*)malloc(sizeof(struct _Maat_group_rule_t)); + group->group_id=group_id; + group->region_cnt=0; + group->region_boundary=0; + group->ref_cnt=0; + group->region_rules=dynamic_array_create(1,8); + group->compile_shortcut=NULL; + pthread_mutex_init(&(group->mutex), NULL); + return group; +} +void destroy_group_rule(struct _Maat_group_rule_t* group) +{ + + if(group->ref_cnt>0||group->region_cnt>0) + { + return; + } + + dynamic_array_destroy(group->region_rules,free); + group->region_cnt=0; + group->region_boundary=0; + group->region_rules=NULL; + group->ref_cnt=0; + group->group_id=-1; + pthread_mutex_destroy(&(group->mutex)); + free(group); + +} +void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bool_expr_t* a_set) +{ + int i=0,j=0; + a_set->bool_expr_id=(void*)compile_rule; + struct _Maat_group_rule_t*group=NULL; + assert(MAAT_MAX_EXPR_ITEM_NUM<=MAX_ITEMS_PER_BOOL_EXPR); + for(i=0,j=0;igroups,i); + if(group==NULL) + { + continue; + } + a_set->bool_item_ids[j]=group->group_id; + j++; + } + assert(j==compile_rule->group_cnt); + a_set->bool_item_num=j; +} +void walk_compile_hash(const uchar * key, uint size, void * data, void * user) +{ + universal_bool_expr_t* one_set=NULL; + struct _Maat_compile_rule_t* compile_rule=(struct _Maat_compile_rule_t*)data; + MESA_lqueue_head update_q=(MESA_lqueue_head)user; + if(compile_rule->db_c_rule==NULL) + { + return; + } + //make sure compile rule's each group has loadded. + if((compile_rule->group_cnt==compile_rule->db_c_rule->declare_grp_num + ||compile_rule->db_c_rule->declare_grp_num==0)//for compatible old version + &&compile_rule->group_cnt>0) + { + one_set=(universal_bool_expr_t*)malloc(sizeof(universal_bool_expr_t)); + //reading compile rule is safe in update thread, mutex lock called when modified + make_group_set(compile_rule, one_set); + MESA_lqueue_join_tail(update_q,&one_set, sizeof(void*));//put the pointer into queue + } + return; +} +void* create_bool_matcher(MESA_htable_handle compile_hash,int thread_num,void* logger) +{ + void* bool_matcher=NULL; + MESA_lqueue_head update_q=MESA_lqueue_create(0,0);; + long data_size=0; + unsigned int mem_size=0; + MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; + data_size=sizeof(void*); + universal_bool_expr_t* one_set=NULL; + universal_bool_expr_t* set_array=NULL; + int i=0; + MESA_htable_iterate(compile_hash, walk_compile_hash, update_q); + + const long q_cnt=MESA_lqueue_get_count(update_q); + set_array=(universal_bool_expr_t*)malloc(sizeof(universal_bool_expr_t)*q_cnt); + for(i=0;icompile_id=compile_id; + p->group_cnt=0; + p->groups=dynamic_array_create(1, 1); + pthread_rwlock_init(&(p->rwlock), NULL); + return p; +} +void destroy_compile_rule(struct _Maat_compile_rule_t * p) +{ + int i=0; + struct _Maat_compile_rule_t* p_group=NULL; + assert(p->group_cnt==0); + for(i=0;igroups,i); + assert(p_group==NULL); + } + p->compile_id=-1; + dynamic_array_destroy(p->groups,NULL); + if(p->db_c_rule!=NULL) + { + free(p->db_c_rule); + } + pthread_rwlock_destroy(&(p->rwlock)); + free(p); +} +scan_rule_t* create_rs_str_rule(unsigned int sub_type,enum MAAT_MATCH_METHOD match_method,int is_case_sensitive,const char* string,int len,int l_offset,int r_offset) +{ + scan_rule_t* p_rule=(scan_rule_t* )calloc(sizeof(scan_rule_t),1); + p_rule->rule_type=RULETYPE_STR; + p_rule->sub_type=sub_type; + + p_rule->string_rule.case_sensitive=is_case_sensitive; + p_rule->string_rule.match_mode=0; + p_rule->string_rule.l_offset=-1; + p_rule->string_rule.r_offset=-1; + switch(match_method) + { + case MATCH_METHOD_FULL: + p_rule->string_rule.match_mode=1; + break; + case MATCH_METHOD_LEFT: + p_rule->string_rule.l_offset=-2; + break; + case MATCH_METHOD_RIGHT: + p_rule->string_rule.r_offset=-2; + break; + case MATCH_METHOD_SUB: + p_rule->string_rule.l_offset=l_offset; + p_rule->string_rule.r_offset=r_offset; + break; + default: + assert(0); + break; + } + p_rule->string_rule.len=len; + p_rule->string_rule.str=(char*)calloc(sizeof(char),len); + memcpy(p_rule->string_rule.str,string,len); + return p_rule; + +} +void destroy_rs_str_rule(scan_rule_t* p_rule) +{ + free(p_rule->string_rule.str); + free(p_rule); +} +scan_rule_t* create_rs_ip_rule(unsigned int sub_type,struct db_ip_rule_t *db_ip_rule) +{ + scan_rule_t *p_rule=(scan_rule_t*)calloc(sizeof(scan_rule_t),1); + if(db_ip_rule->addr_type==4) + { + p_rule->rule_type=RULETYPE_IPv4; + memcpy(&(p_rule->ipv4_rule),&(db_ip_rule->ipv4_rule),sizeof(p_rule->ipv4_rule)); + } + else + { + p_rule->rule_type=RULETYPE_IPv6; + memcpy(&(p_rule->ipv6_rule),&(db_ip_rule->ipv6_rule),sizeof(p_rule->ipv6_rule)); + } + p_rule->sub_type=sub_type; + return p_rule; +} +void destroy_rs_ip_rule(scan_rule_t* p) +{ + free(p); +} +scan_rule_t* create_rs_intval_rule(unsigned int sub_type,struct db_intval_rule_t *intval_rule) +{ + scan_rule_t *p_rule=(scan_rule_t*)calloc(sizeof(scan_rule_t),1); + p_rule->rule_type=RULETYPE_INT; + p_rule->sub_type=sub_type; + p_rule->interval_rule.lb=intval_rule->intval.lb; + p_rule->interval_rule.ub=intval_rule->intval.ub; + return p_rule; +} +void destroy_rs_intval_rule(scan_rule_t* p) +{ + free(p); +} + +struct op_expr_t* create_op_expr(unsigned int expr_id,int operation,void* u_para) +{ + struct op_expr_t* op_expr=NULL; + op_expr=(struct op_expr_t*)calloc(sizeof(struct op_expr_t),1); + op_expr->p_expr=(boolean_expr_t*)calloc(sizeof(boolean_expr_t),1); + op_expr->p_expr->expr_id=expr_id; + op_expr->p_expr->operation=operation; + op_expr->p_expr->rnum=0; + op_expr->p_expr->rules=NULL; + op_expr->p_expr->tag=u_para; + return op_expr; +} +void destroy_op_expr(struct op_expr_t* op_expr) +{ + unsigned int i=0; + for(i=0;ip_expr->rnum;i++) + { + switch(op_expr->p_rules[i]->rule_type) + { + case RULETYPE_STR: + case RULETYPE_REG: + destroy_rs_str_rule(op_expr->p_rules[i]); + break; + case RULETYPE_IPv4: + case RULETYPE_IPv6: + destroy_rs_ip_rule(op_expr->p_rules[i]); + break; + case RULETYPE_INT: + destroy_rs_intval_rule(op_expr->p_rules[i]); + break; + default: + assert(0); + break; + } + op_expr->p_rules[i]=NULL; + } + free(op_expr->p_expr); + op_expr->p_expr=NULL; + free(op_expr); +} +void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule) +{ + int idx=op_expr->p_expr->rnum; + op_expr->p_rules[idx]=p_rule; + op_expr->p_expr->rnum++; + return; +} + +struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread_num,MESA_lqueue_head tomb) +{ + + MESA_htable_create_args_t hargs; + memset(&hargs,0,sizeof(hargs)); + hargs.thread_safe=1; + hargs.hash_slot_size = 1024*1024; + hargs.max_elem_num = 0; + hargs.eliminate_type = HASH_ELIMINATE_ALGO_LRU; + hargs.expire_time = 0; + hargs.key_comp = NULL; + hargs.key2index = NULL; + hargs.recursive = 1; +// hargs.data_free = _void_destroy_compile_rule; + hargs.data_free = EMPTY_FREE; + + hargs.data_expire_with_condition = NULL; + + struct _Maat_scanner_t* scanner=NULL; + scanner=(struct _Maat_scanner_t*)calloc(sizeof(struct _Maat_scanner_t),1); + + scanner->compile_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(scanner->compile_hash,0); + + + hargs.data_free=EMPTY_FREE; + scanner->group_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(scanner->group_hash,0); + + + scanner->region_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(scanner->group_hash,0); + + scanner->version=version; + scanner->cfg_num=0; + scanner->max_thread_num=scan_thread_num; + //optimized for CPU cache_alignment 64 + scanner->ref_cnt=(int*)calloc(CPU_CACHE_ALIGMENT,scan_thread_num); + scanner->region_update_q=MESA_lqueue_create(0,0); + scanner->region=rulescan_initialize(scan_thread_num); + scanner->tomb_ref=tomb; + scanner->region_rslt_buff=(scan_result_t*)malloc(sizeof(scan_result_t)*MAX_SCANNER_HIT_NUM*scan_thread_num); + + return scanner; +} + + + +void destroy_maat_scanner(struct _Maat_scanner_t*scanner) +{ + long q_cnt=0,data_size=0; + int i=0,j=0,q_ret=0; + struct op_expr_t* op_expr=NULL; + rulescan_destroy(scanner->region); + MESA_htable_destroy(scanner->compile_hash,NULL); + MESA_htable_destroy(scanner->group_hash, NULL); + MESA_htable_destroy(scanner->region_hash, NULL); + + destroy_bool_matcher((void*)scanner->expr_compiler); + q_cnt=MESA_lqueue_get_count(scanner->region_update_q); + for(i=0;iregion_update_q,&op_expr,&data_size); + assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); + destroy_op_expr(op_expr); + } + MESA_lqueue_destroy(scanner->region_update_q, lqueue_destroy_cb, NULL); + free(scanner->region_rslt_buff); + scanner->region_rslt_buff=NULL; + free(scanner->ref_cnt); + scanner->ref_cnt=NULL; + for(i=0;iiconv_handle[i][j]!=NULL) + { + iconv_close(scanner->iconv_handle[i][j]); + } + } + } + free(scanner); + return; +} + + +inline unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge) +{ + unsigned int sub_type=0; + if(do_charset_merge==TRUE) + { + sub_type=table_id<<4|CHARSET_NONE; + } + else + { + sub_type=table_id<<4|charset; + } + assert(sub_typerules); + free(p); + return; +} + + +void batch_update(rule_scanner_t scanner,MESA_lqueue_head expr_queue,void*logger) +{ + long i=0,data_size=0; + int j=0,ret=0; + unsigned int failed_ids[MAX_FAILED_NUM]; + char failed_info[512],*p=NULL; + MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; + memset(failed_ids,0,sizeof(failed_ids)); + memset(failed_info,0,sizeof(failed_info)); + const long q_cnt=MESA_lqueue_get_count(expr_queue); + if(q_cnt==0) + { + return; + } + boolean_expr_t* to_update_expr=(boolean_expr_t*)calloc(sizeof(boolean_expr_t),q_cnt); + struct op_expr_t* op_expr=NULL; + for(i=0;ip_expr,sizeof(boolean_expr_t)); + //make a whole memory chunk + to_update_expr[i].rules=(scan_rule_t*)calloc(sizeof(scan_rule_t),op_expr->p_expr->rnum); + for(j=0;j<(int)op_expr->p_expr->rnum;j++) + { + memcpy(&(to_update_expr[i].rules[j]),op_expr->p_rules[j],sizeof(scan_rule_t)); + if(to_update_expr[i].rules[j].rule_type==RULETYPE_REG||to_update_expr[i].rules[j].rule_type==RULETYPE_STR) + { + to_update_expr[i].rules[j].string_rule.str=(char*)calloc(sizeof(char),to_update_expr[i].rules[j].string_rule.len); + memcpy(to_update_expr[i].rules[j].string_rule.str + ,op_expr->p_rules[j]->string_rule.str + ,to_update_expr[i].rules[j].string_rule.len); + } + } + destroy_op_expr(op_expr); + op_expr=NULL; + } + ret=rulescan_update(scanner, to_update_expr,q_cnt, failed_ids,MAX_FAILED_NUM); + + if(ret!=1) + { + p=failed_info; + for(i=0;i10;i++) + { + p+=snprintf(p,sizeof(failed_info)-(p-failed_info),"%d,",failed_ids[i+1]); + } + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "rulescan_update error,when batch update %ld rules,regex error %u.",q_cnt,failed_ids[0]); + } + for(i=0;iregion_id=region_id; + region_rule->expr_id=expr_id; + region_rule->region_type=region_type; + pthread_mutex_lock(&(group->mutex)); + dynamic_array_write(group->region_rules,group->region_boundary,region_rule); + group->region_cnt++; + group->region_boundary++; + pthread_mutex_unlock(&(group->mutex)); + return group; +} +unsigned int del_region_from_group(struct _Maat_group_rule_t* group,int region_id,unsigned int *output_expr_id,int output_size) +{ + int i=0,j=0; + struct _Maat_region_rule_t* region_rule=NULL; + pthread_mutex_lock(&(group->mutex)); + for(i=0;iregion_boundary;i++) + { + region_rule=(struct _Maat_region_rule_t*)dynamic_array_read(group->region_rules, i); + if(region_rule==NULL) + { + continue; + } + if(region_rule->region_id==region_id) + { + dynamic_array_write(group->region_rules, i, NULL); + output_expr_id[j]=region_rule->expr_id; + j++; + assert(j<=output_size); + + free(region_rule); + region_rule=NULL; + + group->region_cnt--; + } + } + pthread_mutex_unlock(&(group->mutex)); + return j; +} + +int add_group_to_compile(struct _Maat_compile_rule_t*a_compile_rule,struct _Maat_group_rule_t* a_rule_group) +{ + int i=0,ret=-1; + struct _Maat_group_rule_t* p=NULL; + pthread_rwlock_wrlock(&(a_compile_rule->rwlock)); + if(a_compile_rule->db_c_rule!=NULL + &&a_compile_rule->group_cnt>=a_compile_rule->db_c_rule->declare_grp_num + &&a_compile_rule->db_c_rule->declare_grp_num!=0) + { + ret=-1; + } + else + { + for(i=0;igroups,i); + if(p==NULL) + { + dynamic_array_write(a_compile_rule->groups,i, a_rule_group); + a_compile_rule->group_cnt++; + a_rule_group->ref_cnt++; + //variable compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish. + if(a_rule_group->ref_cnt==1&&a_compile_rule->group_cnt==1) + { + a_rule_group->compile_shortcut=a_compile_rule; + } + else + { + a_rule_group->compile_shortcut=NULL; + } + ret=1; + break; + } + else + { + if(p->group_id==a_rule_group->group_id)//duplicate group + { + ret=-1; + + } + } + } + if(i==MAAT_MAX_EXPR_ITEM_NUM) + { + ret=-1; + } + //update group's shortcut when compile has more than one group. + if(a_compile_rule->group_cnt!=1) + { + for(i=0;igroups,i); + if(p!=NULL) + { + p->compile_shortcut=NULL; + } + } + } + } + pthread_rwlock_unlock(&(a_compile_rule->rwlock)); + + return ret; +} +struct _Maat_group_rule_t* del_group_from_compile(struct _Maat_compile_rule_t*a_compile_rule,int group_id) +{ + int i=0; + struct _Maat_group_rule_t* group_rule=NULL; + pthread_rwlock_wrlock(&(a_compile_rule->rwlock)); + for(i=0;igroups,i); + if(group_rule==NULL) + { + continue; + } + if(group_rule->group_id==group_id) + { + group_rule->ref_cnt--; + dynamic_array_write(a_compile_rule->groups,i,NULL); + a_compile_rule->group_cnt--; + pthread_rwlock_unlock(&(a_compile_rule->rwlock)); + return group_rule; + } + } + pthread_rwlock_unlock(&(a_compile_rule->rwlock)); + return NULL; +} +int MAAT_MAGIC=0xaaaa; +int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_name,int is_valid,void*logger) +{ + int ret=-1; + if(is_valid==TRUE) + { + ret=HASH_add_by_id(region_hash,region_id,&MAAT_MAGIC); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "region id %d of table %s is not unique.",region_id,table_name); + return -1; + } + + } + else + { + ret=HASH_delete_by_id(region_hash,region_id); + if(ret==-1) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "region delete error,id %d in table %s does not exisit." + ,region_id + ,table_name); + return -1; + } + } + return 1; +} +int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger) +{ + unsigned int i=0,j=0; + char* p=NULL,*saveptr=NULL,*region_string=NULL; + int region_str_len=0,ret=0,k=0; + int expr_id=0; + + scan_rule_t*p_rule=NULL; + struct _Maat_group_rule_t* group_rule=NULL; + enum MAAT_CHARSET dst_charset=CHARSET_NONE; + char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM]; + int key_left_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1},key_right_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1}; + for(i=0;igroup_hash, db_rule->group_id); + if(group_rule==NULL) + { + group_rule=create_group_rule(db_rule->group_id); + HASH_add_by_id(scanner->group_hash, db_rule->group_id, group_rule); + } + switch(db_rule->expr_type) + { + case EXPR_TYPE_AND: + if(db_rule->match_method!=MATCH_METHOD_SUB) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "Table %s region cfg %d is EXPR_TYPE_AND,but match method is not MATCH_METHOD_SUB,force fixed.", + table->table_name,db_rule->region_id); + db_rule->match_method=MATCH_METHOD_SUB; + + } + for(i=0,p=db_rule->keywords;;i++,p=NULL) + { + if(i>=MAAT_MAX_EXPR_ITEM_NUM) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "Table %s region cfg %d too many expr.",table->table_name,db_rule->region_id); + return -1; + } + sub_key_array[i]=strtok_r_esc(p,'&',&saveptr); + if(sub_key_array[i]==NULL) + { + break; + } + sub_key_array[i]=str_unescape_and(sub_key_array[i]); + } + sub_expr_cnt=i; + table->expr_rule_cnt++; + break; + case EXPR_TYPE_OFFSET: + for(i=0,p=db_rule->keywords;;i++,p=NULL) + { + if(i>=MAAT_MAX_EXPR_ITEM_NUM) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "Table %s region cfg %d too many expr.",table->table_name,db_rule->region_id); + return -1; + } + sub_key_array[i]=strtok_r_esc(p,'&',&saveptr); + if(sub_key_array[i]==NULL) + { + break; + } + sscanf(sub_key_array[i],"%d-%d:",&(key_left_offset[i]),&(key_right_offset[i])); + if(!(key_left_offset[i]>=0&&key_right_offset[i]>0&&key_left_offset[i]table_name,db_rule->region_id); + return -1; + } + sub_key_array[i]=(char*)memchr(sub_key_array[i],':',strlen(sub_key_array[i])); + if(sub_key_array[i]==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "Table %s region cfg %d invalid keywords format.",table->table_name,db_rule->region_id); + return -1; + } + sub_key_array[i]++;//jump over ':' + sub_key_array[i]=str_unescape_and(sub_key_array[i]); + } + sub_expr_cnt=i; + table->expr_rule_cnt++; + break; + case EXPR_TYPE_REGEX://it's easy,no need to charset convert + expr_id=scanner->exprid_generator++; + u_para=add_region_to_group(group_rule,db_rule->region_id,expr_id,TABLE_TYPE_EXPR); + if(u_para==NULL) + { + return -1; + } + op_expr=create_op_expr(expr_id + ,0 + ,u_para); + for(i=0,p=db_rule->keywords;;i++,p=NULL) + { + if(i>=MAAT_MAX_EXPR_ITEM_NUM) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "Table %s region cfg %d too many expr.",table->table_name,db_rule->region_id); + return -1; + } + sub_key_array[i]=strtok_r_esc(p,'&',&saveptr); + if(sub_key_array[i]==NULL) + { + break; + } + sub_key_array[i]=str_unescape_and(sub_key_array[i]); + p_rule=create_rs_str_rule(make_sub_type(table->table_id,CHARSET_NONE,0) + ,MATCH_METHOD_SUB//not care db_rule->match_method + ,db_rule->is_case_sensitive + ,sub_key_array[i] + ,strlen(sub_key_array[i]) + ,-1 + ,-1); + p_rule->rule_type=RULETYPE_REG; + op_expr_add_rule(op_expr, p_rule); + + } + MESA_lqueue_join_tail(scanner->region_update_q,&op_expr, sizeof(void*)); + table->regex_rule_cnt++; + return 0;//yes,we returned. + break; + case EXPR_TYPE_STRING: + sub_expr_cnt=1; + sub_key_array[0]=db_rule->keywords; + sub_key_array[0]=str_unescape_and(sub_key_array[0]); + table->expr_rule_cnt++; + break; + default: + break; + } + for(k=0;ktable_name,db_rule->region_id); + //this sub string will jump over before iconv_convert + } + } + if(db_rule->is_hexbin==FALSE) + { + for(j=0;jdst_charset[j]; + if(dst_charset==CHARSET_NONE) + { + break; + } + expr_id=scanner->exprid_generator++; + u_para=add_region_to_group(group_rule, db_rule->region_id,expr_id, TABLE_TYPE_EXPR); + if(u_para==NULL)//duplicate + { + return -1; + } + op_expr=create_op_expr(expr_id + ,0 //add + ,u_para + ); + for(k=0;ksrc_charset!=dst_charset)//need convert + { + + ret=iconv_convert(scanner,table->src_charset, dst_charset, + sub_key_array[k],strlen(sub_key_array[k]), + region_string, ®ion_str_len); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "Table %s region cfg %d charset convert from %s to %s failed.",table->table_name + ,db_rule->region_id + ,CHARSET_STRING[table->src_charset] + ,CHARSET_STRING[dst_charset]); + free(region_string); + continue; + } + //if convert take no effect + if(region_str_len==(int)strlen(sub_key_array[k])&& + 0==memcmp(sub_key_array[k],region_string,region_str_len)&& + TRUE==table->src_charset_in_dst) + { + free(region_string); + continue; + } + } + else + { + memcpy(region_string,sub_key_array[k],strlen(sub_key_array[k])); + region_str_len=strlen(sub_key_array[k]); + } + p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,table->do_charset_merge) + ,db_rule->match_method + ,db_rule->is_case_sensitive + ,region_string + ,region_str_len + ,key_left_offset[k] + ,key_right_offset[k]); + op_expr_add_rule(op_expr, p_rule); + free(region_string); + region_string=NULL; + } + MESA_lqueue_join_tail(scanner->region_update_q,&op_expr, sizeof(void*)); + } + + } + else + { + expr_id=scanner->exprid_generator++; + u_para=add_region_to_group(group_rule, db_rule->region_id,expr_id, TABLE_TYPE_EXPR); + if(u_para==NULL) + { + return -1; + } + op_expr=create_op_expr(expr_id + ,0 //add + ,u_para + ); + for(k=0;ktable_id,dst_charset,table->do_charset_merge) + ,db_rule->match_method + ,db_rule->is_case_sensitive + ,region_string + ,region_str_len + ,key_left_offset[k] + ,key_right_offset[k]); + op_expr_add_rule(op_expr,p_rule); + free(region_string); + region_string=NULL; + } + MESA_lqueue_join_tail(scanner->region_update_q,&op_expr, sizeof(void*)); + } + return 0; +} +int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger) +{ + struct _Maat_group_rule_t* group_rule=NULL; + scan_rule_t* p_rule=NULL; + struct op_expr_t* op_expr=NULL; + struct _Maat_group_rule_t* u_para=NULL; + int expr_id=0; + + group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id); + if(group_rule==NULL) + { + group_rule=create_group_rule(db_ip_rule->group_id); + HASH_add_by_id(scanner->group_hash, db_ip_rule->group_id, group_rule); + } + + expr_id=scanner->exprid_generator++; + u_para=add_region_to_group(group_rule,db_ip_rule->region_id,expr_id,TABLE_TYPE_IP); + if(u_para==NULL) + { + return -1; + } + op_expr=create_op_expr(expr_id + ,0 + ,u_para + ); + p_rule=create_rs_ip_rule(make_sub_type(table->table_id,CHARSET_NONE,0) + ,db_ip_rule); + op_expr_add_rule(op_expr,p_rule); + MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); + return 0; +} +int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger) +{ + struct _Maat_group_rule_t* group_rule=NULL; + scan_rule_t* p_rule=NULL; + struct op_expr_t* op_expr=NULL; + struct _Maat_group_rule_t* u_para=NULL; + int expr_id=0; + + group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id); + if(group_rule==NULL) + { + group_rule=create_group_rule(intval_rule->group_id); + HASH_add_by_id(scanner->group_hash, intval_rule->group_id, group_rule); + } + expr_id=scanner->exprid_generator++; + u_para=add_region_to_group(group_rule,intval_rule->region_id,expr_id,TABLE_TYPE_INTVAL); + if(u_para==NULL) + { + return -1; + } + op_expr=create_op_expr(expr_id + ,0 + ,u_para + ); + p_rule=create_rs_intval_rule(make_sub_type(table->table_id,CHARSET_NONE,0) + ,intval_rule); + op_expr_add_rule(op_expr,p_rule); + MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); + return 0; +} +int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id,struct _Maat_scanner_t *maat_scanner,void* logger) +{ + int i=0; + unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0}; + int expr_num=0; + struct _Maat_group_rule_t* group_rule=NULL; + struct op_expr_t* op_expr=NULL; + + group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(maat_scanner->group_hash, group_id); + if(group_rule==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,table %s group id %u not exist,while delete region id %d." + ,table->table_name + ,group_id + ,region_id); + return -1; + } + assert(group_id==group_rule->group_id); + expr_num=del_region_from_group(group_rule,region_id, expr_id, sizeof(expr_id)/sizeof(unsigned int)); + if(expr_num==0) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + "region delete error,id %d table %s region not in group id %d." + ,region_id + ,table->table_name + ,group_id); + return -1; + } + for(i=0;iregion_update_q,&op_expr, sizeof(void*)); + } + + if(group_rule->region_cnt==0&&group_rule->region_cnt==0) + { + HASH_delete_by_id(maat_scanner->group_hash,group_id); + garbage_bagging(GARBAGE_GROUP_RULE, group_rule, maat_scanner->tomb_ref); + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + "last region rule of group id %d in table %s region id %d has been delete." + ,group_id + ,table->table_name + ,region_id); + } + return 1; +} +int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) +{ + struct _Maat_group_rule_t* group_rule=NULL; + struct _Maat_compile_rule_t*compile_rule=NULL; + int ret=0; + group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); + if(group_rule==NULL) + { + group_rule=create_group_rule(db_group_rule->group_id); + ret=HASH_add_by_id(scanner->group_hash, db_group_rule->group_id,group_rule); + assert(ret>=0); + } + + compile_rule=(struct _Maat_compile_rule_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id); + if(compile_rule==NULL) + { + compile_rule=create_compile_rule(db_group_rule->compile_id); + ret=HASH_add_by_id(scanner->compile_hash,db_group_rule->compile_id, compile_rule); + assert(ret>=0); + } + ret=add_group_to_compile(compile_rule,group_rule); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, + "update error,add %s group %d to compile %d error,compile rule is full or duplicate group." + ,table->table_name + ,db_group_rule->group_id + ,db_group_rule->compile_id); + return -1; + } + return 0; + +} +void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) +{ + struct _Maat_compile_rule_t*compile_rule=NULL; + struct _Maat_group_rule_t* group_rule=NULL; + compile_rule=(struct _Maat_compile_rule_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id); + if(compile_rule==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,delete %s group rule error : compile id %d does not exisit." + ,table->table_name + ,db_group_rule->compile_id); + return; + } + group_rule=del_group_from_compile(compile_rule, db_group_rule->group_id); + if(group_rule==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,delete %s group rule error : group id %d not in compile id %d." + ,table->table_name + ,db_group_rule->group_id + ,db_group_rule->compile_id); + return; + } + if(compile_rule->group_cnt==0&&compile_rule->db_c_rule==NULL) + { + HASH_delete_by_id(scanner->compile_hash, db_group_rule->compile_id); + garbage_bagging(GARBAGE_COMPILE_RULE, compile_rule, scanner->tomb_ref); + } + if(group_rule->ref_cnt==0&&group_rule->region_cnt==0) + { + HASH_delete_by_id(scanner->group_hash,db_group_rule->group_id); + garbage_bagging(GARBAGE_GROUP_RULE, group_rule, scanner->tomb_ref); + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + "table %s group id %d been eternal delete." + ,table->table_name + ,db_group_rule->group_id); + } + return; +} +int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) +{ + struct _Maat_compile_rule_t *compile_rule=NULL; + struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head); + compile_rule=(struct _Maat_compile_rule_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id); + if(compile_rule==NULL) + { + compile_rule=create_compile_rule(p_maat_rule_head->config_id); + HASH_add_by_id(scanner->compile_hash,p_maat_rule_head->config_id,compile_rule); + } + if(compile_rule->db_c_rule!=NULL)//duplicate config + { + return -1; + } + compile_rule->db_c_rule=db_compile_rule; + return 0; + +} +int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) +{ + struct _Maat_compile_rule_t *compile_rule=NULL; + compile_rule=(struct _Maat_compile_rule_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id); + if(compile_rule==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,delete %s compile rule error : congfig id %d does not exisit." + ,table->table_name + ,db_compile_rule->m_rule_head.config_id); + return -1; + } + pthread_rwlock_wrlock(&(compile_rule->rwlock)); + free(compile_rule->db_c_rule->service_defined); + free(compile_rule->db_c_rule); + compile_rule->db_c_rule=NULL; + pthread_rwlock_unlock(&(compile_rule->rwlock)); + + if(compile_rule->group_cnt==0) + { + HASH_delete_by_id(scanner->compile_hash, compile_rule->compile_id); + garbage_bagging(GARBAGE_COMPILE_RULE,compile_rule, scanner->tomb_ref); + } + return 1; +} +void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger) +{ + struct db_group_rule_t db_group_rule; + int ret=0; + ret=sscanf(table_line,"%d\t%d\t%d",&(db_group_rule.group_id) + ,&(db_group_rule.compile_id) + ,&(db_group_rule.is_valid)); + assert(ret==3); + if(db_group_rule.is_valid==FALSE) + { + del_group_rule(table, &db_group_rule,scanner,logger); + table->cfg_num--; + } + else + { + ret=add_group_rule(table,&db_group_rule, scanner,logger); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + "duplicate config of group table %s group_id %d compile_id %d.",table->table_name + ,db_group_rule.group_id + ,db_group_rule.compile_id); + + } + else + { + //no need to free db_group_rule,it was saved in scanner->compile_hash + table->cfg_num++; + } + } + + return; +} +void compatible_group_udpate(struct _Maat_table_info_t* table,int region_id,int compile_id,int is_valid,struct _Maat_scanner_t *scanner,void* logger) +{ + char virtual_group_line[256]; + snprintf(virtual_group_line,sizeof(virtual_group_line), + "%d\t%d\t%d",region_id,compile_id,is_valid); + update_group_rule(table, virtual_group_line,scanner,logger); + return; +} + +void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +{ + struct db_str_rule_t* maat_str_rule=(struct db_str_rule_t*)malloc(sizeof(struct db_str_rule_t)); + int ret=0,db_hexbin=0; + ret=sscanf(table_line,"%d\t%d\t%s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id) + ,&(maat_str_rule->group_id) + ,maat_str_rule->keywords + ,(int*)&(maat_str_rule->expr_type) + ,(int*)&(maat_str_rule->match_method) + ,&db_hexbin + ,&(maat_str_rule->is_valid)); + if(ret!=7) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,invalid format of expr table %s:%s",table->table_name,table_line); + free(maat_str_rule); + maat_str_rule=NULL; + return; + } + switch(db_hexbin) + { + case 0: + maat_str_rule->is_hexbin=FALSE; + maat_str_rule->is_case_sensitive=FALSE; + break; + case 1: + maat_str_rule->is_hexbin=TRUE; + maat_str_rule->is_case_sensitive=FALSE; + break; + case 2: + maat_str_rule->is_hexbin=FALSE; + maat_str_rule->is_case_sensitive=TRUE; + break; + default: + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,invalid hexbin value of expr table %s:%s",table->table_name,table_line); + goto error_out; + } + ret=sync_region(scanner->region_hash + ,maat_str_rule->region_id + ,table->table_name + ,maat_str_rule->is_valid,logger); + if(ret<0) + { + goto error_out; + } + if(group_mode_on==FALSE)//for compatible old version + { + compatible_group_udpate(table + ,maat_str_rule->region_id + ,maat_str_rule->group_id + ,maat_str_rule->is_valid + ,scanner + ,logger); + maat_str_rule->group_id=maat_str_rule->region_id; + } + + if(maat_str_rule->is_valid==FALSE) + { + ret=del_region_rule(table,maat_str_rule->region_id,maat_str_rule->group_id, scanner, logger); + if(ret>0) + { + table->cfg_num--; + } + } + else + { + + ret=add_expr_rule(table, maat_str_rule,scanner, logger); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + "duplicate config of expr table %s region_id=%d",table->table_name,maat_str_rule->region_id); + + } + else + { + table->cfg_num++; + } + } +error_out: + free(maat_str_rule); + maat_str_rule=NULL; +} +void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +{ + struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); + char src_ip[40],mask_src_ip[40],dst_ip[40],mask_dst_ip[40]; + + unsigned short i_src_port,i_sport_mask,i_dst_port,i_dport_mask; + struct in6_addr v6_src_mask,v6_dst_mask; + int protocol=0,direction=0; + int ret=0; + int ret_array[8]={1},i=0; + ret=sscanf(table_line,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t%d" + ,&(ip_rule->region_id) + ,&(ip_rule->group_id) + ,&(ip_rule->addr_type) + ,src_ip + ,mask_src_ip + ,&i_src_port + ,&i_sport_mask + ,dst_ip + ,mask_dst_ip + ,&i_dst_port + ,&i_dport_mask + ,&protocol + ,&direction + ,&(ip_rule->is_valid)); + if(ret!=14||(ip_rule->addr_type!=4&&ip_rule->addr_type!=6) + ||protocol>65535||protocol<0 + ||(direction!=0&&direction!=1)) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,invalid format of ip table %s:%s",table->table_name,table_line); + goto error_out; + } + if(ip_rule->addr_type==4) + { + ret_array[0]=inet_pton(AF_INET,src_ip,&(ip_rule->ipv4_rule.saddr)); + ip_rule->ipv4_rule.saddr=ntohl(ip_rule->ipv4_rule.saddr); + ret_array[1]=inet_pton(AF_INET,mask_src_ip,&(ip_rule->ipv4_rule.smask)); + ip_rule->ipv4_rule.smask=ntohl(ip_rule->ipv4_rule.smask); + + ret_array[2]=inet_pton(AF_INET,dst_ip,&(ip_rule->ipv4_rule.daddr)); + ip_rule->ipv4_rule.daddr=ntohl(ip_rule->ipv4_rule.daddr); + ret_array[3]=inet_pton(AF_INET,mask_dst_ip,&(ip_rule->ipv4_rule.dmask)); + ip_rule->ipv4_rule.dmask=ntohl(ip_rule->ipv4_rule.dmask); + + ip_rule->ipv4_rule.min_sport=i_src_port&i_sport_mask; + ip_rule->ipv4_rule.max_sport=(i_src_port&i_sport_mask)+(~i_sport_mask); + + ip_rule->ipv4_rule.min_dport=i_dst_port&i_dport_mask; + ip_rule->ipv4_rule.max_dport=(i_dst_port&i_dport_mask)+(~i_dport_mask); + + ip_rule->ipv4_rule.proto=protocol; + ip_rule->ipv4_rule.direction=direction; + } + else + { + ret_array[0]=inet_pton(AF_INET6,src_ip,&(ip_rule->ipv6_rule.saddr)); + ipv6_ntoh(ip_rule->ipv6_rule.saddr); + ret_array[1]=inet_pton(AF_INET6,mask_src_ip,&(v6_src_mask)); + ip_rule->ipv6_rule.smask_bits=128-cnt_maskbits(v6_src_mask); + + ret_array[2]=inet_pton(AF_INET6,dst_ip,&(ip_rule->ipv6_rule.daddr)); + ipv6_ntoh(ip_rule->ipv6_rule.daddr); + ret_array[3]=inet_pton(AF_INET6,mask_dst_ip,&(v6_dst_mask)); + ip_rule->ipv6_rule.dmask_bits=128-cnt_maskbits(v6_dst_mask); + + ip_rule->ipv6_rule.min_sport=i_src_port&i_sport_mask; + ip_rule->ipv6_rule.max_sport=(i_src_port&i_sport_mask)+(~i_sport_mask); + + ip_rule->ipv6_rule.min_dport=i_dst_port&i_dport_mask; + ip_rule->ipv6_rule.max_dport=(i_dst_port&i_dport_mask)+~(i_dport_mask); + + ip_rule->ipv6_rule.proto=protocol; + ip_rule->ipv6_rule.direction=direction; + } + for(i=0;i<4;i++) + { + if(ret_array[i]<=0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,invalid format of ip table %s:%s",table->table_name,table_line); + goto error_out; + } + } + ret=sync_region(scanner->region_hash + ,ip_rule->region_id + ,table->table_name + ,ip_rule->is_valid,logger); + if(ret<0) + { + goto error_out; + } + if(group_mode_on==FALSE)//for compatible old version + { + compatible_group_udpate(table + ,ip_rule->region_id + ,ip_rule->group_id + ,ip_rule->is_valid + ,scanner + ,logger); + ip_rule->group_id=ip_rule->region_id; + } + if(ip_rule->is_valid==FALSE) + { + ret=del_region_rule(table,ip_rule->region_id,ip_rule->group_id, scanner, logger); + if(ret>0) + { + table->cfg_num--; + } + } + else + { + + ret=add_ip_rule(table, ip_rule,scanner, logger); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + "duplicate config of ip table %s config_id=%d",table->table_name,ip_rule->region_id); + + } + else + { + table->cfg_num++; + } + } +error_out: + free(ip_rule); + ip_rule=NULL; +} + +void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +{ + struct db_intval_rule_t* intval_rule=(struct db_intval_rule_t*)calloc(sizeof(struct db_intval_rule_t),1); + int ret=0; + ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id) + ,&(intval_rule->group_id) + ,&(intval_rule->intval.lb) + ,&(intval_rule->intval.ub) + ,&(intval_rule->is_valid)); + + if(ret!=5||intval_rule->intval.ubintval.lb) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,invalid format of interval table %s:%s",table->table_name,table_line); + goto error_out; + } + ret=sync_region(scanner->region_hash + ,intval_rule->region_id + ,table->table_name + ,intval_rule->is_valid,logger); + if(ret<0) + { + goto error_out; + } + if(group_mode_on==FALSE)//for compatible old version + { + compatible_group_udpate(table + ,intval_rule->region_id + ,intval_rule->group_id + ,intval_rule->is_valid + ,scanner + ,logger); + intval_rule->group_id=intval_rule->region_id; + } + + if(intval_rule->is_valid==FALSE) + { + ret=del_region_rule(table,intval_rule->region_id,intval_rule->group_id, scanner, logger); + if(ret>0) + { + table->cfg_num--; + } + } + else + { + ret=add_intval_rule(table, intval_rule,scanner,logger); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + "duplicate config of intval table %s config_id=%d",table->table_name,intval_rule->region_id); + } + else + { + table->cfg_num++; + } + } +error_out: + free(intval_rule); + intval_rule=NULL; +} + +void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger) +{ + struct db_compile_rule_t *p_compile=(struct db_compile_rule_t*)calloc(sizeof(struct db_compile_rule_t ),1); + struct _head_Maat_rule_t* p_m_rule=&(p_compile->m_rule_head); + char user_region[128*2]={0}; + int ret=0; + p_compile->declare_grp_num=0; + ret=sscanf(table_line,"%d\t%d\t%hhd\t%hhd\t%hhd\t%lld\t%s\t%d\t%d",&(p_m_rule->config_id) + ,&(p_m_rule->service_id) + ,&(p_m_rule->action) + ,&(p_m_rule->do_blacklist) + ,&(p_m_rule->do_log) + ,&(p_compile->effective_range) + ,user_region + ,&(p_compile->is_valid) + ,&(p_compile->declare_grp_num)); + if((ret!=8&&ret!=9)||strlen(user_region)>MAX_SERVICE_DEFINE_LEN||p_compile->declare_grp_num>MAAT_MAX_EXPR_ITEM_NUM) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "update error,invalid format of compile table %s:%s",table->table_name,table_line); + free(p_compile); + p_compile=NULL; + return; + } + + p_m_rule->serv_def_len=strlen(user_region)+1; + p_compile->service_defined=(char*)malloc(p_m_rule->serv_def_len*sizeof(char)); + memcpy(p_compile->service_defined,user_region,p_m_rule->serv_def_len); + if(p_compile->is_valid==FALSE) + { + ret=del_compile_rule(table,p_compile,scanner, logger); + if(ret>0) + { + table->cfg_num--; + } + free(p_compile->service_defined); + p_compile->service_defined=NULL; + free(p_compile); + p_compile=NULL; + } + else + { + ret=add_compile_rule(table, p_compile, scanner,logger); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + "duplicate config of compile table %s config_id=%d",table->table_name,p_m_rule->config_id); + free(p_compile->service_defined); + p_compile->service_defined=NULL; + free(p_compile); + p_compile=NULL; + + } + else + { + //no need to free p_compile,it was saved in scanner->compile_hash + table->cfg_num++; + } + } + + return; +} + +void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q) +{ + struct _maat_garbage_t* bag=(struct _maat_garbage_t*)malloc(sizeof(struct _maat_garbage_t)); + bag->raw=p; + bag->type=type; + bag->create_time=time(NULL); + bag->ok_times=0; + MESA_lqueue_join_tail(garbage_q,&bag,sizeof(void*)); + return; +} +void garbage_bury(MESA_lqueue_head garbage_q,void *logger) +{ + MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; + _maat_garbage_t* bag=NULL; + long data_size=0; + const long q_cnt=MESA_lqueue_get_count(garbage_q); + int i=0,bury_cnt=0; + int ref_cnt=0; + int have_timeout=0; + time_t now=time(NULL); + for(i=0;icreate_time<10) + { + MESA_lqueue_join_tail(garbage_q,&bag,sizeof(void*)); + continue; + } + have_timeout=1; + switch(bag->type) + { + case GARBAGE_COMPILE_RULE: + destroy_compile_rule(bag->compile_rule); + break; + case GARBAGE_GROUP_RULE: + destroy_group_rule(bag->group_rule); + break; + case GARBAGE_SCANNER: + ref_cnt=aligment_int_array_sum(bag->scanner->ref_cnt,bag->scanner->max_thread_num); + + if(ref_cnt==0) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, + "scanner %p version %d has no reference peacefully destroyed.",bag->scanner,bag->scanner->version); + } + else + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, + "scanner %p version %d force destroyed,ref_cnt %d.", + bag->scanner,bag->scanner->version,ref_cnt); + + } + destroy_maat_scanner(bag->scanner); + break; + case GARBAGE_BOOL_MATCHER: + destroy_bool_matcher(bag->bool_matcher); + break; + default: + assert(0); + } + free(bag); + bag=NULL; + bury_cnt++; + } + if(q_cnt>0&&have_timeout==1) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, + "Garbage queue size %ld, bury %d", + q_cnt,bury_cnt); + } +} +void plugin_table_callback(struct _Maat_table_info_t* table,const char* table_line,void* logger) +{ + int i=0; + unsigned int len=strlen(table_line)+1; + struct _plugin_table_info* p_table_cb=table->cb_info; + for(i=0;icb_plug_cnt;i++) + { + p_table_cb->cb_plug[i].update(table->table_id,table_line,p_table_cb->cb_plug[i].u_para); + } + char *p=(char*)calloc(len,1); + memcpy(p,table_line,len); + p_table_cb->cache_size+=len; + dynamic_array_write(p_table_cb->cache_lines,p_table_cb->line_num,p); + p_table_cb->line_num++; +} +void maat_start_cb(unsigned int new_version,int update_type,void*u_para) +{ + struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; + struct _Maat_table_info_t* p_table=NULL; + struct _plugin_table_info* p_table_cb=NULL; + int i=0,j=0; + if(update_type==CM_UPDATE_TYPE_FULL) + { + feather->update_tmp_scanner=create_maat_scanner(new_version,feather->scan_thread_num,feather->garbage_q); + MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, + "Full config version %u -> %u update start", + feather->maat_version,new_version); + } + else + { + + MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, + "Inc config version %u -> %u update start", + feather->maat_version,new_version); + feather->maat_version=new_version; + } + for(i=0;itable_cnt;i++) + { + p_table=feather->p_table_info[i]; + if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN) + { + continue; + } + p_table_cb=p_table->cb_info; + for(j=0;jcb_plug_cnt;j++) + { + p_table_cb->cb_plug[j].start(update_type,p_table_cb->cb_plug[j].u_para); + } + } + return; +} +void do_increase_update(struct _Maat_feather_t *feather) +{ + void *tmp1=NULL,*tmp2=NULL; + tmp1=create_bool_matcher(feather->scanner->compile_hash, + feather->scan_thread_num, + feather->logger); + tmp2=feather->scanner->expr_compiler; + + //assume pinter = operation is thread safe + feather->scanner->expr_compiler=tmp1; + garbage_bagging(GARBAGE_BOOL_MATCHER, tmp2, feather->garbage_q); + + batch_update(feather->scanner->region, + feather->scanner->region_update_q, + feather->logger); + feather->scanner->last_update_time=time(NULL); + + return; + +} +void maat_finish_cb(void* u_para) +{ + struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; + struct _Maat_table_info_t* p_table=NULL; + struct _plugin_table_info* p_table_cb=NULL; + int i=0,j=0,total=0; + for(i=0;ip_table_info[i]; + if(p_table!=NULL) + { + total+=p_table->cfg_num; + } + } + for(i=0;ip_table_info[i]; + + if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN) + { + continue; + } + p_table_cb=p_table->cb_info; + for(j=0;jcb_plug_cnt;j++) + { + p_table_cb->cb_plug[j].finish(p_table_cb->cb_plug[j].u_para); + } + } + if(feather->update_tmp_scanner!=NULL) + { + feather->update_tmp_scanner->cfg_num=total; + feather->update_tmp_scanner->expr_compiler=create_bool_matcher(feather->update_tmp_scanner->compile_hash, + feather->scan_thread_num, + feather->logger); + batch_update(feather->update_tmp_scanner->region, + feather->update_tmp_scanner->region_update_q, + feather->logger); + MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, + "Full config version %u load %d entries complete.", + feather->update_tmp_scanner->version,feather->update_tmp_scanner->cfg_num); + feather->update_tmp_scanner->last_update_time=time(NULL); + } + else + { + feather->scanner->cfg_num=total; + feather->scanner->version=feather->maat_version; + if(time(NULL)-feather->scanner->last_update_time>60) + { + do_increase_update(feather); + MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, + "Inc config version %u build complete,%d entries in total.", + feather->scanner->version,feather->scanner->cfg_num); + } + else + { + MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, + "Postpone config version %u %d entries load to rulescan.", + feather->scanner->version,feather->scanner->cfg_num); + } + } + return; +} +void maat_update_cb(const char* table_name,const char* line,void *u_para) +{ + struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; + int ret=-1; + int table_id=-1; + _Maat_scanner_t* scanner=NULL; + if(feather->update_tmp_scanner!=NULL) + { + scanner=feather->update_tmp_scanner; + } + else + { + scanner=feather->scanner; + } + ret=map_str2int(feather->map_tablename2id,table_name,&table_id); + if(ret<0) + { + MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module ,"update warning,unknown table name %s",table_name); + return; + } + switch(feather->p_table_info[table_id]->table_type) + { + case TABLE_TYPE_EXPR: + update_expr_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); + break; + case TABLE_TYPE_IP: + update_ip_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); + break; + case TABLE_TYPE_INTVAL: + update_intval_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); + break; + case TABLE_TYPE_COMPILE: + update_compile_rule(feather->p_table_info[table_id], line, scanner,feather->logger); + break; + case TABLE_TYPE_GROUP: + update_group_rule(feather->p_table_info[table_id], line, scanner,feather->logger); + break; + case TABLE_TYPE_PLUGIN: + plugin_table_callback(feather->p_table_info[table_id], line,feather->logger); + break; + default: + break; + + } + return; +} +void *thread_rule_monitor(void *arg) +{ + struct _Maat_feather_t *feather=(struct _Maat_feather_t *)arg; + const char* inc_cfg_dir=(const char*)feather->inc_dir; + struct _Maat_scanner_t* old_scanner=NULL; + long expr_wait_q_cnt=0; + while(feather->still_working) + { + usleep(feather->scan_interval_ms*1000); + config_monitor_traverse(feather->maat_version, + inc_cfg_dir, + maat_start_cb, + maat_update_cb, + maat_finish_cb, + feather, + feather->logger); + if(feather->update_tmp_scanner!=NULL) + { + old_scanner=feather->scanner; + //__sync_lock_test_and_set not work in some OS. + //feather->scanner=__sync_lock_test_and_set(&(feather->scanner),feather->update_tmp_scanner); + feather->scanner=feather->update_tmp_scanner; + if(old_scanner!=NULL) + { + assert(feather->scanner->version>old_scanner->version); + assert(old_scanner->tomb_ref==feather->garbage_q); + garbage_bagging(GARBAGE_SCANNER, old_scanner, feather->garbage_q); + } + feather->update_tmp_scanner=NULL; + feather->maat_version=feather->scanner->version; + } + if(feather->scanner!=NULL) + { + expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); + if(expr_wait_q_cnt>0&&time(NULL)-feather->scanner->last_update_time>feather->effect_interval_ms*1000) + { + do_increase_update(feather); + MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, + "Actual udpate config version %u %d entries load to rulescan after postpone.", + feather->scanner->version,feather->scanner->cfg_num); + } + } + garbage_bury(feather->garbage_q,feather->logger); + } + + MESA_htable_destroy(feather->map_tablename2id,free); + destroy_maat_scanner(feather->scanner); + garbage_bury(feather->garbage_q,feather->logger); + MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL); + + int i=0,j=0; + struct dynamic_array_t* d_array=NULL; + char* lines=NULL; + for(i=0;ip_table_info[i]==NULL) + { + continue; + } + if(feather->p_table_info[i]->table_type==TABLE_TYPE_PLUGIN) + { + d_array=feather->p_table_info[i]->cb_info->cache_lines; + for(j=0;jp_table_info[i]->cb_info->line_num;j++) + { + lines=(char*)dynamic_array_read(d_array, j); + free(lines); + } + } + free(feather->p_table_info[i]); + feather->p_table_info[i]=NULL; + } + free(feather); + return NULL; +} +//return 1 if insert a unique id +//return 0 if id is duplicated +//return -1 if set is full +int insert_set_id(unsigned int **set,int* size,int cnt,unsigned int id) +{ + int i=0; + for(i=0;ibool_item_num);j++) + { + if(hitted_id[i]==compile_hit->bool_item_ids[j]) + { + region_pos[k]=i; + k++; + } + } + } + return k; +} +int region_compile(struct _scan_status_t *_mid,scan_result_t* region_hit,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size) +{ + + int scan_ret=0,result_cnt=0; + int ret=0,i=0,j=0; + int r_in_c_cnt=0; + void* expr_compiler=_mid->feather->scanner->expr_compiler; + int shortcut_avilable_cnt=0; + struct _Maat_group_rule_t* group_rule=NULL; + struct _Maat_compile_rule_t* array_mi_rule[MAX_SCANNER_HIT_NUM]; + struct _Maat_compile_rule_t* _mi_rule=NULL; + int region_pos[MAX_SCANNER_HIT_NUM]; + + _mid->cur_hit_cnt=0; + for(i=0;igroup_id<0) + { + continue; + } + if(group_rule->compile_shortcut!=NULL&&group_rule->ref_cnt==1&&shortcut_avilable_cntcompile_shortcut); + shortcut_avilable_cnt++; + } + _mid->cur_hit_id[_mid->cur_hit_cnt]=(int)group_rule->group_id; + _mid->cur_hit_cnt++; + ret=insert_set_id(&(_mid->hitted_group_id), + &(_mid->hit_group_size), + _mid->hit_group_cnt, + (unsigned int)group_rule->group_id); + _mid->hit_group_cnt+=ret; + } + if(shortcut_avilable_cnt==region_hit_num||shortcut_avilable_cnt==MAX_SCANNER_HIT_NUM) + { + //short cut for rules contains one group + scan_ret=shortcut_avilable_cnt; + } + else + { + scan_ret=boolexpr_match(expr_compiler,_mid->thread_num, + _mid->hitted_group_id,_mid->hit_group_cnt, + (void **)array_mi_rule, MAX_SCANNER_HIT_NUM); + } + for(i=0,j=0;idb_c_rule==NULL) + { + continue; + } + + if(0==pthread_rwlock_tryrdlock(&(_mi_rule->rwlock))) + { + make_group_set(_mi_rule,&(rs_result[result_cnt].group_set)); + r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set),_mid->cur_hit_id,_mid->cur_hit_cnt, + region_pos, MAX_SCANNER_HIT_NUM); + if(r_in_c_cnt>0)//compile config hitted becasue of new reigon + { + memcpy(&(result[result_cnt]),&(_mi_rule->db_c_rule->m_rule_head),sizeof(struct _head_Maat_rule_t)); + memcpy(result[result_cnt].service_defined + ,_mi_rule->db_c_rule->service_defined + ,_mi_rule->db_c_rule->m_rule_head.serv_def_len); + rs_result[result_cnt].compile_id=_mi_rule->compile_id; + result_cnt++; + } + pthread_rwlock_unlock(&(_mi_rule->rwlock)); + + } + + } + return result_cnt; +} + +int exprid2region_id(struct _Maat_group_rule_t* group_rule,int expr_id) +{ + int i=0,region_id=-1;; + struct _Maat_region_rule_t* region_rule=NULL; + assert(group_rule->group_id>=0); + pthread_mutex_lock(&(group_rule->mutex)); + for(i=0;iregion_boundary;i++) + { + region_rule=(struct _Maat_region_rule_t*)dynamic_array_read(group_rule->region_rules, i); + if(region_rule==NULL) + { + continue; + } + if(region_rule->expr_id==expr_id) + { + region_id=region_rule->region_id; + } + } + pthread_mutex_unlock(&(group_rule->mutex)); + return region_id; +} + + +int fill_regex_pos(struct regex_pos_t *regex_pos,int size,rule_result_t *rs_result,const char* buff) +{ + int i=0,j=0; + int group_num=rs_result->group_num; + unsigned int * position=rs_result->position; + unsigned int * length=rs_result->length; + regex_pos->group_num=group_num; + for(i=0;i<(int)rs_result->result_num&&iMAX_MATCH_POS_NUM) + { + break; + } + regex_pos[i].hitting_regex_pos=buff+position[(group_num+1)*i]; + regex_pos[i].hitting_regex_len=length[(group_num+1)*i]; + for(j=0;jMAAT_MAX_REGEX_GROUP_NUM) + { + break; + } + regex_pos[i].grouping_pos[j]=buff+position[(group_num+1)*i+j+1]; + regex_pos[i].grouping_len[j]=length[(group_num+1)*i+j+1]; + } + } + return i; +} +int fill_substr_pos(struct str_pos_t* str_pos,int size,rule_result_t *rs_result,const char* buff) +{ + int i=0; + unsigned int * position=rs_result->position; + unsigned int * length=rs_result->length; + for(i=0;i<(int)rs_result->result_num&&iMAX_MATCH_POS_NUM) + { + break; + } + str_pos[i].hit_pos=buff+position[i]; + str_pos[i].hit_len= length[i]; + } + return i; +} +int hit_pos_RS2Maat(struct sub_item_pos_t* maat_sub_item,int size,rule_result_t* rs_result,int rs_cnt,const char* buff) +{ + int k=0; + for(k=0;kcur_hit_id,_mid->cur_hit_cnt, + region_pos, MAX_SCANNER_HIT_NUM); + assert(r_in_c_cnt>0);//previous hitted compile was elimited in region_compile + for(j=0,k=0;jfeather=feather; + _mid->thread_num=thread_num; + _mid->cur_hit_cnt=0; + _mid->hit_group_cnt=0; + _mid->hit_group_size=4; + _mid->hitted_group_id=(unsigned int*)malloc(sizeof(unsigned int)*_mid->hit_group_size); + return _mid; +} + +int detain_last_data(char* buff,int buff_size,int detained_len,const char* data,int data_len) +{ + int to_copy_size=0,foward_offset=0; + int ret_len; + if(data_len<=buff_size-detained_len) + { + to_copy_size=data_len; + memcpy(buff+detained_len,data,data_len); + ret_len=detained_len+data_len; + } + else if(data_len>buff_size-detained_len&&data_len=buff_size + { + memcpy(buff,data+data_len-buff_size,buff_size); + ret_len=buff_size; + } + return ret_len; +} +Maat_feather_t Maat_summon_feather(int max_thread_num, + const char* table_info_path, + const char* ful_cfg_dir, + const char* inc_cfg_dir, + void* logger) +{ + + _Maat_feather_t* feather=(_Maat_feather_t*)calloc(sizeof(struct _Maat_feather_t),1); + feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path); + feather->map_tablename2id=map_create(); + int i=0; + for(i=0;ip_table_info[i]!=NULL) + { + if(feather->p_table_info[i]->table_type==TABLE_TYPE_GROUP) + { + feather->GROUP_MODE_ON=1; + } + map_register(feather->map_tablename2id,feather->p_table_info[i]->table_name,feather->p_table_info[i]->table_id); + } + } + memcpy(feather->inc_dir,inc_cfg_dir,strlen(inc_cfg_dir)); + memcpy(feather->full_dir,ful_cfg_dir,strlen(ful_cfg_dir)); + feather->logger=logger; + feather->scan_thread_num=max_thread_num; + feather->garbage_q=MESA_lqueue_create(0,0); + config_monitor_traverse(feather->maat_version, + ful_cfg_dir, + maat_start_cb, + maat_update_cb, + maat_finish_cb, + feather, + logger); + if(feather->update_tmp_scanner==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + "At initiation: no valid index file in %s",ful_cfg_dir); + } + feather->scanner=feather->update_tmp_scanner; + feather->update_tmp_scanner=NULL; + feather->still_working=1; + if(feather->scanner!=NULL) + { + feather->maat_version=feather->scanner->version; + } + feather->effect_interval_ms=60*1000; + feather->scan_interval_ms=1*1000; + pthread_t cfg_mon_t; + pthread_create(&cfg_mon_t, NULL, thread_rule_monitor, (void*)feather); + return feather; +} +Maat_feather_t Maat_summon_feather_json(int max_thread_num, + const char* table_info_path, + const char* json_rule, + void* logger) +{ + Maat_feather_t feather; + char full_index_dir[256]={0}; + int ret=-1; + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + "Maat initial with JSON file %s.",json_rule); + + ret=json2iris(json_rule, full_index_dir,sizeof(full_index_dir),logger); + if(ret<0) + { + return NULL; + } + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + "generate index file %s OK.",full_index_dir); + feather=Maat_summon_feather(max_thread_num,table_info_path, full_index_dir, full_index_dir,logger); + return feather; +} +int Maat_set_feather_opt(Maat_feather_t feather,int type,void* value,int size) +{ + _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + int intval=0; + switch(type) + { + case MAAT_OPT_EFFECT_INVERVAL_MS: + intval=*(int*)value; + if(size!=sizeof(int)||intval<=0) + { + return -1; + } + _feather->effect_interval_ms=intval; + break; + case MAAT_OPT_SCANDIR_INTERVAL_MS: + intval=*(int*)value; + if(size!=sizeof(int)||intval<0) + { + return -1; + } + _feather->scan_interval_ms=intval; + break; + default: + return -1; + } + return 0; +} +void Maat_burn_feather(Maat_feather_t feather) +{ + _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + _feather->still_working=0;//destroy will proceed in thread_rule_monitor + return; +} +int Maat_table_register(Maat_feather_t feather,const char* table_name) +{ + struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; + int table_id=-1,ret=0; + ret=map_str2int(_feather->map_tablename2id, table_name,&table_id); + if(ret>0) + { + return table_id; + } + else + { + return -1; + } +} +int Maat_table_callback_register(Maat_feather_t feather,short table_id, + Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para + Maat_update_callback_t *update,//table line ,u_para + Maat_finish_callback_t *finish,//u_para + void* u_para) +{ + struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; + int idx=0,i=0; + _Maat_table_info_t *p_table=_feather->p_table_info[table_id]; + const char* lines=NULL; + if(p_table==NULL) + { + return -1; + } + idx=p_table->cb_info->cb_plug_cnt; + if(idx==MAX_PLUGING_NUM) + { + return -1; + } + p_table->cb_info->cb_plug_cnt++; + p_table->cb_info->cb_plug[idx].start=start; + p_table->cb_info->cb_plug[idx].update=update; + p_table->cb_info->cb_plug[idx].finish=finish; + p_table->cb_info->cb_plug[idx].u_para=u_para; + if(p_table->cb_info->line_num>0) + { + start(MAAT_RULE_UPDATE_TYPE_FULL,u_para); + for(i=0;icb_info->line_num;i++) + { + lines=(const char*)dynamic_array_read(p_table->cb_info->cache_lines,i); + if(lines==NULL) + { + break; + } + update(table_id,lines,u_para); + } + finish(u_para); + } + return 1; +} +int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id + ,enum MAAT_CHARSET charset,const char* data,int data_len + ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num + ,int* detail_ret,scan_status_t* mid,int thread_num) +{ + int region_ret=0,compile_ret=0,hit_region_cnt=0; + unsigned int sub_type=0; + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct _scan_status_t* _mid=NULL; + + scan_result_t *region_result=NULL; + _compile_result_t compile_result[rule_num];//dynamic array + + struct _Maat_table_info_t *p_table=NULL; + p_table=_feather->p_table_info[table_id]; + if(data==NULL||data_len==0) + { + return 0; + } + if(p_table->cfg_num==0) + { + return 0; + } + if(p_table==NULL||p_table->table_type!=TABLE_TYPE_EXPR) + { + return -1; + } + if(p_table->do_charset_merge==1) + { + sub_type=make_sub_type(table_id,CHARSET_NONE,0); + } + else + { + sub_type=make_sub_type(table_id,charset,0); + } + + scan_data_t scan_data; + scan_data.text_data.text=data; + scan_data.text_data.tlen=data_len; + scan_data.text_data.toffset=0; + _Maat_scanner_t* my_scanner=NULL; + my_scanner=_feather->scanner; + if(my_scanner==NULL) + { + return 0; + } + assert(thread_num<_feather->scan_thread_num); + region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; + int offset=(CPU_CACHE_ALIGMENT/sizeof(int))*thread_num; + my_scanner->ref_cnt[offset]++; + if(p_table->expr_rule_cnt>0) + { + scan_data.rule_type=RULETYPE_STR; + scan_data.sub_type=sub_type; + region_ret=rulescan_search(my_scanner->region, thread_num, &scan_data, region_result, MAX_SCANNER_HIT_NUM); + if(region_ret>0) + { + hit_region_cnt+=region_ret; + } + } + if(p_table->regex_rule_cnt>0) + { + scan_data.rule_type=RULETYPE_REG; + scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0); + region_ret=rulescan_search(my_scanner->region, thread_num, &scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt); + if(region_ret>0) + { + hit_region_cnt+=region_ret; + } + + } + if(hit_region_cnt>0) + { + if(*mid==NULL) + { + _mid=_Maat_make_status(_feather,thread_num); + *mid=_mid; + } + else + { + _mid=(struct _scan_status_t*)(*mid); + } + compile_ret=region_compile(_mid,region_result,hit_region_cnt,result,compile_result,rule_num); + if(hit_detail!=NULL) + { + *detail_ret=fill_region_hit_detail(data,_mid, + region_result,hit_region_cnt, + compile_result,compile_ret, + hit_detail,detail_num); + } + } + my_scanner->ref_cnt[offset]--; + if(compile_ret==0&&hit_region_cnt>0) + { + return -2; + } + return compile_ret; +} +int Maat_full_scan_string(Maat_feather_t feather,int table_id + ,enum MAAT_CHARSET charset,const char* data,int data_len + ,struct Maat_rule_t*result,int* found_pos,int rule_num + ,scan_status_t* mid,int thread_num) +{ + int detail_ret=0,compile_ret=0; + compile_ret=Maat_full_scan_string_detail(feather,table_id, + charset, data,data_len, + result, rule_num, + NULL, 0, &detail_ret,mid,thread_num); + return compile_ret; +} +int Maat_scan_intval(Maat_feather_t feather,int table_id + ,unsigned int intval + ,struct Maat_rule_t*result,int rule_num + ,scan_status_t *mid,int thread_num) +{ + int region_ret=0,compile_ret=0; + struct _scan_status_t* _mid=NULL; + scan_data_t intval_scan_data; + scan_result_t *region_result=NULL; + _compile_result_t compile_result[rule_num]; + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct _Maat_scanner_t* my_scanner=NULL; + intval_scan_data.rule_type=RULETYPE_INT; + intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); + intval_scan_data.int_data=intval; + if(_feather->p_table_info[table_id]==NULL||_feather->p_table_info[table_id]->table_type!=TABLE_TYPE_INTVAL) + { + return -1; + } + my_scanner=_feather->scanner; + if(my_scanner==NULL) + { + return 0; + } + if(_feather->p_table_info[table_id]->cfg_num==0) + { + return 0; + } + region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; + + int offset=(CPU_CACHE_ALIGMENT/sizeof(int))*thread_num; + my_scanner->ref_cnt[offset]++; + region_ret=rulescan_search(my_scanner->region, thread_num, &intval_scan_data, region_result, MAX_SCANNER_HIT_NUM); + if(region_ret<0) + { + return -1; + } + else if(region_ret>0) + { + if(*mid==NULL) + { + _mid=_Maat_make_status(_feather,thread_num); + *mid=_mid; + } + else + { + _mid=(struct _scan_status_t*)(*mid); + } + compile_ret=region_compile(_mid,region_result,region_ret,result,compile_result,rule_num); + } + + my_scanner->ref_cnt[offset]--; + if(compile_ret==0&®ion_ret>0) + { + return -2; + } + return compile_ret; + +} + +int Maat_scan_proto_addr(Maat_feather_t feather,int table_id + ,struct ipaddr* addr,unsigned short int proto + ,struct Maat_rule_t*result,int rule_num + ,scan_status_t *mid,int thread_num) +{ + int region_ret=0,compile_ret=0; + struct _scan_status_t* _mid=NULL; + scan_data_t ip_scan_data; + scan_result_t *region_result=NULL; + _compile_result_t compile_result[rule_num]; + + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct _Maat_scanner_t* my_scanner=NULL; + if(_feather->p_table_info[table_id]==NULL||_feather->p_table_info[table_id]->table_type!=TABLE_TYPE_IP) + { + return -1; + } + if(_feather->p_table_info[table_id]->cfg_num==0) + { + return 0; + } + ip_scan_data.rule_type=RULETYPE_IPv4; + ip_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); + switch(addr->addrtype) + { + case ADDR_TYPE_IPV4: + ip_scan_data.ipv4_data.saddr=ntohl(addr->paddr.v4->saddr); + ip_scan_data.ipv4_data.daddr=ntohl(addr->paddr.v4->daddr); + ip_scan_data.ipv4_data.sport=ntohs(addr->paddr.v4->source); + ip_scan_data.ipv4_data.dport=ntohs(addr->paddr.v4->dest); + ip_scan_data.ipv4_data.proto=proto; + break; + case ADDR_TYPE_IPV6: + ip_scan_data.rule_type=RULETYPE_IPv6; + memcpy(ip_scan_data.ipv6_data.saddr,addr->paddr.v6->saddr,sizeof(ip_scan_data.ipv6_data.saddr)); + ipv6_ntoh(ip_scan_data.ipv6_data.saddr); + memcpy(ip_scan_data.ipv6_data.daddr,addr->paddr.v6->daddr,sizeof(ip_scan_data.ipv6_data.daddr)); + ipv6_ntoh(ip_scan_data.ipv6_data.daddr); + ip_scan_data.ipv6_data.sport=ntohs(addr->paddr.v6->source); + ip_scan_data.ipv6_data.dport=ntohs(addr->paddr.v6->dest); + ip_scan_data.ipv6_data.proto=proto; + break; + default: + return -1; + break; + } + my_scanner=_feather->scanner; + if(my_scanner==NULL) + { + return 0; + } + region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; + int offset=(CPU_CACHE_ALIGMENT/sizeof(int))*thread_num; + my_scanner->ref_cnt[offset]++; + region_ret=rulescan_search(my_scanner->region, thread_num, &ip_scan_data, region_result, MAX_SCANNER_HIT_NUM); + if(region_ret<0) + { + return -1; + } + else if(region_ret>0) + { + if(*mid==NULL) + { + _mid=_Maat_make_status(_feather,thread_num); + *mid=_mid; + } + else + { + _mid=(struct _scan_status_t*)(*mid); + } + compile_ret=region_compile(_mid,region_result,region_ret,result,compile_result,rule_num); + } + + my_scanner->ref_cnt[offset]--; + if(compile_ret==0&®ion_ret>0) + { + return -2; + } + return compile_ret; +} +int Maat_scan_addr(Maat_feather_t feather,int table_id + ,struct ipaddr* addr + ,struct Maat_rule_t*result,int rule_num + ,scan_status_t *mid,int thread_num) +{ + int compile_ret=0; + compile_ret=Maat_scan_proto_addr(feather,table_id, + addr, 0, + result, rule_num, + mid,thread_num); + return compile_ret; +} +stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,int thread_num) +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct _Maat_scanner_t* scanner=NULL; + + struct _Maat_table_info_t *p_table=NULL; + assert(thread_num<_feather->scan_thread_num); + if(_feather->p_table_info[table_id]==NULL) + { + return NULL; + } + p_table=_feather->p_table_info[table_id]; + if(p_table==NULL||p_table->table_type!=TABLE_TYPE_EXPR) + { + return NULL; + } + struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1); + scanner=_feather->scanner; + sp->feather=_feather; + sp->version=_feather->maat_version; + sp->acc_scan_len=0; + if(scanner==NULL) + { + return sp; + } + int offset=(CPU_CACHE_ALIGMENT/sizeof(int))*thread_num; + scanner->ref_cnt[offset]++; + assert(table_id<256); + sp->table_id=table_id; + sp->thread_num=thread_num; + sp->max_cross_size=p_table->cross_cache_size; + sp->caching_size=0; + sp->scan_buff=NULL; + sp->last_cache=NULL; + if(p_table->do_charset_merge==1) + { + sp->do_merge=1; + } + if(p_table->expr_rule_cnt>0) + { + sp->do_expr=1; + } + if(p_table->regex_rule_cnt>0) + { + sp->do_regex=1; + } + sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num); + return sp; +} + +int Maat_stream_scan_string_detail(stream_para_t* stream_para + ,enum MAAT_CHARSET charset,const char* data,int data_len + ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num + ,int* detail_ret,scan_status_t* mid) +{ + struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); + struct _Maat_scanner_t* scanner=sp->feather->scanner; + + int sub_type=0; + int region_ret=0,hit_region_cnt=0,compile_ret=0; + struct _scan_status_t* _mid=(struct _scan_status_t*)(*mid); + scan_result_t *region_result; + _compile_result_t compile_result[rule_num];//dynamic array + scan_data_t region_scan_data; + if(data==NULL||data_len==0) + { + return 0; + } + if(scanner==NULL) + { + return 0; + } + if(sp->version!=sp->feather->maat_version) + { + return 0; + } + if(sp->feather->p_table_info[sp->table_id]->cfg_num==0) + { + return 0; + } + region_result=scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*sp->thread_num; + *detail_ret=0; + if(sp->do_merge==1) + { + sub_type=make_sub_type(sp->table_id,CHARSET_NONE,0); + } + else + { + sub_type=make_sub_type(sp->table_id,charset,0); + } + if(sp->max_cross_size>0&&sp->caching_size>0) + { + if(sp->scan_buff!=NULL) + { + free(sp->scan_buff); + sp->scan_buff=NULL; + } + sp->scan_buff=(char*)malloc(sp->caching_size+data_len); + memcpy(sp->scan_buff,sp->last_cache,sp->caching_size); + memcpy(sp->scan_buff+sp->caching_size,data,data_len); + region_scan_data.text_data.text=sp->scan_buff; + region_scan_data.text_data.tlen=sp->caching_size+data_len; + } + else + { + region_scan_data.text_data.text=data; + region_scan_data.text_data.tlen=data_len; + } + + if(sp->last_cache==NULL&&sp->max_cross_size>0) + { + assert(sp->caching_size==0); + sp->last_cache=(char*)malloc(sizeof(char)*sp->max_cross_size); + } + if(sp->max_cross_size>0) + { + sp->caching_size=detain_last_data(sp->last_cache,sp->max_cross_size,sp->caching_size,data,data_len); + } + + region_scan_data.text_data.toffset=(int)MIN(0xffffffff/2, sp->acc_scan_len);//longger then int + sp->acc_scan_len+=data_len; + if(sp->do_expr==1) + { + region_scan_data.rule_type=RULETYPE_STR; + region_scan_data.sub_type=sub_type; + region_ret=rulescan_searchstream(sp->rs_stream_para, ®ion_scan_data, region_result, MAX_SCANNER_HIT_NUM); + if(region_ret<0) + { + return -1; + } + else if(region_ret>0) + { + hit_region_cnt+=region_ret; + } + } + if(sp->do_regex==1) + { + region_scan_data.rule_type=RULETYPE_REG; + region_scan_data.sub_type=make_sub_type(sp->table_id,CHARSET_NONE,0); + region_ret=rulescan_searchstream(sp->rs_stream_para, ®ion_scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt); + if(region_ret<0) + { + return -1; + } + else if(region_ret>0) + { + hit_region_cnt+=region_ret; + } + } + if(hit_region_cnt>0) + { + if(*mid==NULL) + { + _mid=_Maat_make_status(sp->feather,sp->thread_num); + *mid=_mid; + } + else + { + _mid=(struct _scan_status_t*)(*mid); + } + compile_ret=region_compile(_mid,region_result,hit_region_cnt,result,compile_result,rule_num); + if(hit_detail!=NULL) + { + if(sp->scan_buff!=NULL) + { + *detail_ret=fill_region_hit_detail(sp->scan_buff,_mid, + region_result,hit_region_cnt, + compile_result,compile_ret, + hit_detail,detail_num); + } + else + { + *detail_ret=fill_region_hit_detail(data,_mid, + region_result,hit_region_cnt, + compile_result,compile_ret, + hit_detail,detail_num); + } + } + } + if(*detail_ret==0) + { + free(sp->scan_buff); + sp->scan_buff=0; + } + if(compile_ret==0&&hit_region_cnt>0) + { + return -2; + } + return compile_ret; +} +int Maat_stream_scan_string(stream_para_t* stream_para + ,enum MAAT_CHARSET charset,const char* data,int data_len + ,struct Maat_rule_t*result,int* found_pos,int rule_num + ,scan_status_t* mid) + +{ + + int compile_ret=0; + int detail_ret=0; + compile_ret=Maat_stream_scan_string_detail(stream_para, charset,data,data_len, + result,rule_num, NULL, 0,&detail_ret,mid); + return compile_ret; +} +void Maat_stream_scan_string_end(stream_para_t* stream_para) +{ + struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); + struct _Maat_scanner_t* scanner=sp->feather->scanner; + int offset=(CPU_CACHE_ALIGMENT/sizeof(int))*sp->thread_num; + + if(scanner!=NULL) + { + if(sp->version==sp->feather->maat_version) + { + scanner->ref_cnt[offset]--; + } + + rulescan_endstream(sp->rs_stream_para); + + } + if(sp->last_cache!=NULL) + { + free(sp->last_cache); + sp->last_cache=NULL; + sp->caching_size=0; + } + if(sp->scan_buff!=NULL) + { + free(sp->scan_buff); + sp->scan_buff=NULL; + } + free(sp); + *stream_para=NULL; + return; +} +void Maat_clean_status(scan_status_t* mid) +{ + struct _scan_status_t* _mid=NULL; + if(*mid==NULL) + { + return; + } + _mid=(struct _scan_status_t*)(*mid); + free(_mid->hitted_group_id); + free(_mid); + *mid=NULL; + return; +} + diff --git a/src/entry/Maat_rule_internal.h b/src/entry/Maat_rule_internal.h new file mode 100644 index 0000000..736b9b6 --- /dev/null +++ b/src/entry/Maat_rule_internal.h @@ -0,0 +1,288 @@ +#include "Maat_rule.h" + +#include +#include +#include "dynamic_array.h" +#include "UniversalBoolMatch.h" +#include "rulescan.h" + +#include +#include + +#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411) +#define atomic_inc(x) __sync_add_and_fetch((x),1) +#define atomic_dec(x) __sync_sub_and_fetch((x),1) +#define atomic_add(x,y) __sync_add_and_fetch((x),(y)) +#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y)) +typedef int atomic_t; +#define ATOMIC_INIT(i) { (i) } +#define atomic_read(x) __sync_add_and_fetch((x),0) +#define atomic_set(x,y) __sync_lock_test_and_set((x),y) +#else +#include +#endif +#define CPU_CACHE_ALIGMENT 64 +#define TRUE 1 +#define FALSE 0 + +#define MAX_TABLE_NUM 256 +#define MAX_CHARSET_NUM 6 +#define MAX_TABLE_NAME_LEN 256 +#define MAX_TABLE_LINE_SIZE (1024*4) +#define MAX_EXPR_KEYLEN 1024 +#define MAX_PLUGING_NUM 32 + +#define MAX_SCANNER_HIT_NUM 64 + +#define MAX_GROUP_CACHE 128 + +#define MAX_FAILED_NUM 128 + +#ifndef MAX +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#endif + +#ifndef MIN +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif + +typedef void* rule_scanner_t; +enum MAAT_TABLE_TYPE +{ + TABLE_TYPE_EXPR=0, + TABLE_TYPE_IP, + TABLE_TYPE_COMPILE, + TABLE_TYPE_PLUGIN, + TABLE_TYPE_INTVAL, + TABLE_TYPE_GROUP + +}; +enum MAAT_EXPR_TYPE +{ + EXPR_TYPE_STRING=0, + EXPR_TYPE_AND, + EXPR_TYPE_REGEX, + EXPR_TYPE_OFFSET +}; +enum MAAT_MATCH_METHOD +{ + MATCH_METHOD_SUB=0, + MATCH_METHOD_RIGHT, + MATCH_METHOD_LEFT, + MATCH_METHOD_FULL +}; +struct db_str_rule_t +{ + int region_id; + int group_id; + char keywords[MAX_EXPR_KEYLEN]; + enum MAAT_EXPR_TYPE expr_type; + enum MAAT_MATCH_METHOD match_method; + int is_hexbin; + int is_case_sensitive; + int is_valid; + +}; +struct db_ip_rule_t +{ + int region_id; + int group_id; + int addr_type; + union + { + //ip address use network order + //port use host order + ipv4_rule_t ipv4_rule; + ipv6_rule_t ipv6_rule; + }; + int is_valid; +}; +struct db_intval_rule_t +{ + int region_id; + int group_id; + interval_rule_t intval; + int is_valid; +}; +struct _head_Maat_rule_t +{ + int config_id; + int service_id; + char do_log; + char do_blacklist; + char action; + char resevered; + int serv_def_len; +}; +struct db_compile_rule_t +{ +// Maat_rule_t m_rule_head; + struct _head_Maat_rule_t m_rule_head;// fix len of Maat_rule_t + char* service_defined; + long long effective_range; + int is_valid; + int declare_grp_num; +}; +struct db_group_rule_t +{ + int group_id; + int compile_id; + int is_valid; +}; +struct op_expr_t +{ + boolean_expr_t* p_expr; + scan_rule_t* p_rules[MAAT_MAX_EXPR_ITEM_NUM]; +}; + +struct _Maat_region_rule_t +{ + int region_id; + int expr_id; + enum MAAT_TABLE_TYPE region_type; +}; +struct _Maat_group_rule_t +{ + int group_id; + int region_boundary; + int region_cnt; + int ref_cnt; + dynamic_array_t *region_rules; + void* compile_shortcut; + pthread_mutex_t mutex; +}; +struct _Maat_compile_rule_t +{ + struct db_compile_rule_t *db_c_rule; + dynamic_array_t *groups; + int group_cnt; + int compile_id;//equal to db_c_rule->m_rule.config_id + pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read +}; +struct _compile_result_t +{ + int compile_id; + universal_bool_expr_t group_set; +}; +struct _callback_plugin +{ + Maat_start_callback_t *start; + Maat_update_callback_t *update; + Maat_finish_callback_t *finish; + void* u_para; +}; + +struct _plugin_table_info +{ + int cb_plug_cnt; + struct _callback_plugin cb_plug[MAX_PLUGING_NUM]; + dynamic_array_t *cache_lines; + int line_num; + long cache_size; +}; +struct _Maat_table_info_t +{ + unsigned short table_id; + char table_name[MAX_TABLE_NAME_LEN]; + enum MAAT_TABLE_TYPE table_type; + enum MAAT_CHARSET src_charset; + enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; + int src_charset_in_dst; + int do_charset_merge; + int cfg_num; + int cross_cache_size; + int expr_rule_cnt; //expr_type=0,1,3 + int regex_rule_cnt; //expr_type=2 + struct _plugin_table_info *cb_info; +}; +struct _scan_status_t +{ + struct _Maat_feather_t* feather; + int thread_num; + int cur_hit_cnt; + int hit_group_cnt; + int hit_group_size; + unsigned int cur_hit_id[MAX_SCANNER_HIT_NUM]; + unsigned int *hitted_group_id; +}; +enum maat_garbage_type +{ + GARBAGE_SCANNER=0, + GARBAGE_GROUP_RULE, + GARBAGE_COMPILE_RULE, + GARBAGE_BOOL_MATCHER +}; +struct iconv_handle_t +{ + int is_initialized; + iconv_t cd; +}; +struct _stream_para_t +{ + struct _Maat_feather_t* feather; + int version; + int thread_num; + int max_cross_size; + int caching_size; + unsigned short table_id; + char do_merge; + char do_expr:4; + char do_regex:4; + char* last_cache; + char* scan_buff; + void* rs_stream_para; + long acc_scan_len; + +}; +struct _Maat_scanner_t +{ + int version; + time_t last_update_time; + int *ref_cnt; //optimized for cache_alignment 64 + rule_scanner_t region; + MESA_htable_handle region_hash; + MESA_htable_handle group_hash; + MESA_htable_handle compile_hash; + unsigned int cfg_num; + unsigned int exprid_generator; + MESA_lqueue_head region_update_q; + void * expr_compiler; + scan_result_t *region_rslt_buff; + MESA_lqueue_head tomb_ref;//reference of feather->garbage_q + int max_thread_num; + iconv_t iconv_handle[MAX_CHARSET_NUM][MAX_CHARSET_NUM];//iconv_handle[to][from] +}; +struct _Maat_feather_t +{ + struct _Maat_scanner_t *scanner; + struct _Maat_scanner_t *update_tmp_scanner; + MESA_lqueue_head garbage_q; + int table_cnt; + struct _Maat_table_info_t *p_table_info[MAX_TABLE_NUM]; + MESA_htable_handle map_tablename2id; + void* logger; + int maat_version; + int scan_thread_num; + char inc_dir[MAX_TABLE_NAME_LEN]; + char full_dir[MAX_TABLE_NAME_LEN]; + int GROUP_MODE_ON; + int still_working; + int scan_interval_ms; + int effect_interval_ms; +}; +struct _maat_garbage_t +{ + enum maat_garbage_type type; + time_t create_time; + int ok_times; + union + { + struct _Maat_scanner_t* scanner; + struct _Maat_group_rule_t* group_rule; + struct _Maat_compile_rule_t* compile_rule; + void* bool_matcher; + void * raw; + }; +}; +void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); +void garbage_bury(MESA_lqueue_head garbage_q,void *logger); diff --git a/src/entry/Makefile b/src/entry/Makefile new file mode 100644 index 0000000..32f35c8 --- /dev/null +++ b/src/entry/Makefile @@ -0,0 +1,37 @@ +#opt: OPTFLAGS = -O2 +#export OPTFLAGS + +CC = g++ +CCC = g++ +CFLAGS = -Wall -g -fPIC +CFLAGS += $(OPTFLAGS) +LDFLAGS = -lMESA_handle_logger -lMESA_htable -lpthread +MAILLIB = ../lib + +G_H_DIR =../inc_internal +H_DIR =-I$(G_H_DIR) -I../../inc +LIBMAAT = libmaatframe.a +LIBMAAT_SO = libmaatframe.so + +OBJS=config_monitor.o Maat_rule.o UniversalBoolMatch.o dynamic_array.o cJSON.o json2iris.o map_str2int.o +.c.o: + $(CC) -c $(CFLAGS) -I. $(H_DIR) $< + +.cpp.o: + $(CCC) -c $(CFLAGS) -I. $(H_DIR) $< + +all: $(LIBMAAT) $(LIBMAAT_SO) + +$(LIBMAAT_SO): $(OBJS) + $(CCC) -o $(LIBMAAT_SO) -shared $(OBJS) $(LDFLAGS) + cp $(LIBMAAT_SO) ../../lib/ +$(LIBMAAT): $(OBJS) + echo making dynamic lib ... + ar cqs $(LIBMAAT) $(OBJS) + cp $(LIBMAAT) ../../lib/ + +clean: + rm -rf *.o $(LIBMAAT) *~ + +opt: + $(MAKE) all diff --git a/src/entry/UniversalBoolMatch.cpp b/src/entry/UniversalBoolMatch.cpp new file mode 100644 index 0000000..477cc6d --- /dev/null +++ b/src/entry/UniversalBoolMatch.cpp @@ -0,0 +1,259 @@ +#include "UniversalBoolMatch.h" +#include +#include +#include +using namespace std; +#include +#include + +static const unsigned int MAX_ARRAY_SIZE=65536; + +struct thread_local_data_t +{ + unsigned int mapped_ids[MAX_ARRAY_SIZE]; + unsigned int used_cells[MAX_ARRAY_SIZE]; + unsigned char * bitmap; + unsigned int * matched_bitmap; +}; + +struct boolexpr_matcher_t +{ + unsigned int max_thread_num; + unsigned int bool_expr_num; + unsigned int multi_expr_num; + void ** bool_expr_ids; + unsigned char * multi_expr_size; + unsigned int bool_item_id_num; + unsigned int min_item_id; + unsigned int max_item_id; + unsigned int * bool_item_ids; + unsigned int * mapped_ptr; + unsigned int * mapped_ids; + unsigned int theta; + unsigned int L[65537]; + thread_local_data_t * thread_data; +}; + +void * boolexpr_initialize(universal_bool_expr_t * bool_exprs, unsigned int bool_expr_num, unsigned int max_thread_num, unsigned int * mem_size) +{ + if(bool_exprs==NULL || bool_expr_num==0 || max_thread_num==0) return NULL; + + for(unsigned int i=0; iMAX_ITEMS_PER_BOOL_EXPR) + { + return NULL; + } + } + + int I=-1, J=(int)bool_expr_num; + while(I1) I++; + if(I==J) break; + J--; + while(J>I && bool_exprs[J].bool_item_num==1) J--; + if(J==I) break; + swap(bool_exprs[I], bool_exprs[J]); + } + + for(int k=0; k<(int)bool_expr_num; k++) + { + if((k=I && bool_exprs[k].bool_item_num>1)) + { + printf("[%s:%d]: fatal error!\n", __FILE__, __LINE__); + return NULL; + } + } + + unsigned int mem_bytes=0; + + boolexpr_matcher_t * matcher=new boolexpr_matcher_t; + mem_bytes+=sizeof(boolexpr_matcher_t); + + matcher->max_thread_num=max_thread_num; + matcher->bool_expr_num=bool_expr_num; + matcher->multi_expr_num=I; + + matcher->bool_expr_ids=new void *[bool_expr_num]; + mem_bytes+=bool_expr_num*sizeof(void *); + + matcher->multi_expr_size=new unsigned char[matcher->multi_expr_num+1]; + mem_bytes+=(matcher->multi_expr_num+1)*sizeof(unsigned char); + + matcher->thread_data=new thread_local_data_t[max_thread_num]; + mem_bytes+=max_thread_num*sizeof(thread_local_data_t); + + for(unsigned int i=0; ithread_data[i].bitmap=new unsigned char[matcher->multi_expr_num+1]; + mem_bytes+=(matcher->multi_expr_num+1)*sizeof(unsigned char); + + unsigned int size=(bool_expr_num-matcher->multi_expr_num); + size=(size>>5)+1; + matcher->thread_data[i].matched_bitmap=new unsigned int[size]; + mem_bytes+=size*sizeof(unsigned int); + } + + map< unsigned int, vector > M; + unsigned int count=0; + for(unsigned int i=0; ibool_expr_ids[i] =bool_exprs[i].bool_expr_id; + if(imulti_expr_num) + { + matcher->multi_expr_size[i]=bool_exprs[i].bool_item_num; + } + count+=bool_exprs[i].bool_item_num; + for(unsigned int j=0; jbool_item_id_num=(unsigned int)M.size(); + matcher->bool_item_ids=new unsigned int[M.size()]; + matcher->mapped_ptr =new unsigned int[M.size()+1]; + matcher->mapped_ids =new unsigned int[count]; + mem_bytes+=(2*(unsigned int)M.size()+1+count)*sizeof(unsigned int); + + matcher->mapped_ptr[0]=0; + map< unsigned int, vector >::const_iterator it=M.begin(); + for(unsigned int k=0; kbool_item_ids[k]=it->first; + copy(it->second.begin(), it->second.end(), matcher->mapped_ids+matcher->mapped_ptr[k]); + matcher->mapped_ptr[k+1]=matcher->mapped_ptr[k]+(unsigned int)it->second.size(); + } + + matcher->min_item_id=matcher->bool_item_ids[0]; + matcher->max_item_id=matcher->bool_item_ids[M.size()-1]; + for(unsigned int k=0; kbool_item_ids[k]-=matcher->min_item_id; + } + + unsigned long long ONE=1; + unsigned int theta=0; + while((ONE<<(theta+16))<=matcher->bool_item_ids[M.size()-1]) theta++; + matcher->theta=theta; + + matcher->L[0]=0; + for(unsigned int i=1; i<65536; i++) + { + matcher->L[i]=(unsigned int)(lower_bound(matcher->bool_item_ids, matcher->bool_item_ids+M.size(), i*(1U<bool_item_ids); + } + matcher->L[65536]=(unsigned int)M.size(); + + M.clear(); + + *mem_size=mem_bytes; + return matcher; +} + +int boolexpr_match(void * instance, unsigned int thread_id, unsigned int * item_ids, unsigned int item_num, void ** result, unsigned int size) +{ + if(instance==NULL) return -1; + + boolexpr_matcher_t * matcher=(boolexpr_matcher_t *)instance; + if(thread_id>=matcher->max_thread_num) return -1; + + unsigned int * mapped_ids=matcher->thread_data[thread_id].mapped_ids; + unsigned int ids_num=0; + for(unsigned int i=0; imin_item_id || item_ids[i]>matcher->max_item_id) continue; + + unsigned int id=item_ids[i]-matcher->min_item_id; + unsigned int k=id>>matcher->theta; + + int l=matcher->L[k], h=(int)matcher->L[k+1]-1; + if(hbool_item_ids[m]) h=m-1; + else l=m+1; + } + if(h<(int)matcher->L[k] || matcher->bool_item_ids[h]!=id) continue; + + for(unsigned int j=matcher->mapped_ptr[h]; jmapped_ptr[h+1]; j++) + { + if(ids_num==MAX_ARRAY_SIZE) return -1; + mapped_ids[ids_num++]=matcher->mapped_ids[j]; + } + } + + unsigned int * used_cells=matcher->thread_data[thread_id].used_cells; + unsigned int used_num=0; + for(unsigned int i=0; i>3); + } + + unsigned char * bitmap=matcher->thread_data[thread_id].bitmap; + unsigned int * matched_bitmap=matcher->thread_data[thread_id].matched_bitmap; + for(unsigned int i=0; imulti_expr_num) + { + bitmap[used_cells[i]]=0; + } + else + { + unsigned int j=used_cells[i]-matcher->multi_expr_num; + matched_bitmap[j>>5]&=~(1U<<(j&31)); + } + } + + unsigned int r=0; + + for(unsigned int i=0; i>3); + if(xmulti_expr_num) + { + unsigned int y=(mapped_ids[i]&7); + if((bitmap[x]&(1U<multi_expr_size[x])-1) + { + if(rbool_expr_ids[x]; + } + } + } + else + { + unsigned int j=x-matcher->multi_expr_num; + if((matched_bitmap[j>>5]&(1U<<(j&31)))==0) + { + if(rbool_expr_ids[x]; + matched_bitmap[j>>5]|=(1U<<(j&31)); + } + } + } + + return r; +} + +void boolexpr_destroy(void * instance) +{ + if(instance!=NULL) + { + boolexpr_matcher_t * matcher=(boolexpr_matcher_t *)instance; + delete [] matcher->bool_expr_ids; + delete [] matcher->multi_expr_size; + delete [] matcher->bool_item_ids; + delete [] matcher->mapped_ptr; + delete [] matcher->mapped_ids; + for(unsigned int i=0; imax_thread_num; i++) + { + delete [] matcher->thread_data[i].bitmap; + } + delete [] matcher->thread_data; + delete matcher; + } +} diff --git a/src/entry/UniversalBoolMatch.h b/src/entry/UniversalBoolMatch.h new file mode 100644 index 0000000..95cd6c5 --- /dev/null +++ b/src/entry/UniversalBoolMatch.h @@ -0,0 +1,46 @@ +/* + * + * Copyright (c) 2014 + * String Algorithms Research Group + * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) + * National Engineering Laboratory for Information Security Technologies (NELIST) + * All rights reserved + * + * Written by: LIU YANBING (liuyanbing@iie.ac.cn) + * Last modification: 2014-12-09 + * + * This code is the exclusive and proprietary property of IIE-CAS and NELIST. + * Usage for direct or indirect commercial advantage is not allowed without + * written permission from the authors. + * + */ + +#ifndef H_UNIVERSAL_BOOL_MATCH_H +#define H_UNIVERSAL_BOOL_MATCH_H + +#ifdef __cplusplus +extern "C" +{ +#endif + + #define MAX_ITEMS_PER_BOOL_EXPR 8 + + typedef struct _universal_bool_expr_t + { + void * bool_expr_id; + unsigned int bool_item_num; + unsigned int bool_item_ids[MAX_ITEMS_PER_BOOL_EXPR]; + }universal_bool_expr_t; + + /*注意：本函数调用会交换bool_exprs中元素的位置*/ + void * boolexpr_initialize(universal_bool_expr_t * bool_exprs, unsigned int bool_expr_num, unsigned int max_thread_num, unsigned int * mem_size); + + int boolexpr_match(void * instance, unsigned int thread_id, unsigned int * item_ids, unsigned int item_num, void ** result, unsigned int size); + + void boolexpr_destroy(void * instance); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/entry/cJSON.c b/src/entry/cJSON.c new file mode 100644 index 0000000..35452cb --- /dev/null +++ b/src/entry/cJSON.c @@ -0,0 +1,596 @@ +/* + Copyright (c) 2009 Dave Gamble + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +/* cJSON */ +/* JSON parser in C. */ + +#include +#include +#include +#include +#include +#include +#include +#include "cJSON.h" + +static const char *ep; + +const char *cJSON_GetErrorPtr(void) {return ep;} + +static int cJSON_strcasecmp(const char *s1,const char *s2) +{ + if (!s1) return (s1==s2)?0:1;if (!s2) return 1; + for(; tolower(*s1) == tolower(*s2); ++s1, ++s2) if(*s1 == 0) return 0; + return tolower(*(const unsigned char *)s1) - tolower(*(const unsigned char *)s2); +} + +static void *(*cJSON_malloc)(size_t sz) = malloc; +static void (*cJSON_free)(void *ptr) = free; + +static char* cJSON_strdup(const char* str) +{ + size_t len; + char* copy; + + len = strlen(str) + 1; + if (!(copy = (char*)cJSON_malloc(len))) return 0; + memcpy(copy,str,len); + return copy; +} + +void cJSON_InitHooks(cJSON_Hooks* hooks) +{ + if (!hooks) { /* Reset hooks */ + cJSON_malloc = malloc; + cJSON_free = free; + return; + } + + cJSON_malloc = (hooks->malloc_fn)?hooks->malloc_fn:malloc; + cJSON_free = (hooks->free_fn)?hooks->free_fn:free; +} + +/* Internal constructor. */ +static cJSON *cJSON_New_Item(void) +{ + cJSON* node = (cJSON*)cJSON_malloc(sizeof(cJSON)); + if (node) memset(node,0,sizeof(cJSON)); + return node; +} + +/* Delete a cJSON structure. */ +void cJSON_Delete(cJSON *c) +{ + cJSON *next; + while (c) + { + next=c->next; + if (!(c->type&cJSON_IsReference) && c->child) cJSON_Delete(c->child); + if (!(c->type&cJSON_IsReference) && c->valuestring) cJSON_free(c->valuestring); + if (c->string) cJSON_free(c->string); + cJSON_free(c); + c=next; + } +} + +/* Parse the input text to generate a number, and populate the result into item. */ +static const char *parse_number(cJSON *item,const char *num) +{ + double n=0,sign=1,scale=0;int subscale=0,signsubscale=1; + + if (*num=='-') sign=-1,num++; /* Has sign? */ + if (*num=='0') num++; /* is zero */ + if (*num>='1' && *num<='9') do n=(n*10.0)+(*num++ -'0'); while (*num>='0' && *num<='9'); /* Number? */ + if (*num=='.' && num[1]>='0' && num[1]<='9') {num++; do n=(n*10.0)+(*num++ -'0'),scale--; while (*num>='0' && *num<='9');} /* Fractional part? */ + if (*num=='e' || *num=='E') /* Exponent? */ + { num++;if (*num=='+') num++; else if (*num=='-') signsubscale=-1,num++; /* With sign? */ + while (*num>='0' && *num<='9') subscale=(subscale*10)+(*num++ - '0'); /* Number? */ + } + + n=sign*n*pow(10.0,(scale+subscale*signsubscale)); /* number = +/- number.fraction * 10^+/- exponent */ + + item->valuedouble=n; + item->valueint=(int)n; + item->type=cJSON_Number; + return num; +} + +/* Render the number nicely from the given item into a string. */ +static char *print_number(cJSON *item) +{ + char *str; + double d=item->valuedouble; + if (fabs(((double)item->valueint)-d)<=DBL_EPSILON && d<=INT_MAX && d>=INT_MIN) + { + str=(char*)cJSON_malloc(21); /* 2^64+1 can be represented in 21 chars. */ + if (str) sprintf(str,"%d",item->valueint); + } + else + { + str=(char*)cJSON_malloc(64); /* This is a nice tradeoff. */ + if (str) + { + if (fabs(floor(d)-d)<=DBL_EPSILON && fabs(d)<1.0e60)sprintf(str,"%.0f",d); + else if (fabs(d)<1.0e-6 || fabs(d)>1.0e9) sprintf(str,"%e",d); + else sprintf(str,"%f",d); + } + } + return str; +} + +static unsigned parse_hex4(const char *str) +{ + unsigned h=0; + if (*str>='0' && *str<='9') h+=(*str)-'0'; else if (*str>='A' && *str<='F') h+=10+(*str)-'A'; else if (*str>='a' && *str<='f') h+=10+(*str)-'a'; else return 0; + h=h<<4;str++; + if (*str>='0' && *str<='9') h+=(*str)-'0'; else if (*str>='A' && *str<='F') h+=10+(*str)-'A'; else if (*str>='a' && *str<='f') h+=10+(*str)-'a'; else return 0; + h=h<<4;str++; + if (*str>='0' && *str<='9') h+=(*str)-'0'; else if (*str>='A' && *str<='F') h+=10+(*str)-'A'; else if (*str>='a' && *str<='f') h+=10+(*str)-'a'; else return 0; + h=h<<4;str++; + if (*str>='0' && *str<='9') h+=(*str)-'0'; else if (*str>='A' && *str<='F') h+=10+(*str)-'A'; else if (*str>='a' && *str<='f') h+=10+(*str)-'a'; else return 0; + return h; +} + +/* Parse the input text into an unescaped cstring, and populate item. */ +static const unsigned char firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; +static const char *parse_string(cJSON *item,const char *str) +{ + const char *ptr=str+1;char *ptr2;char *out;int len=0;unsigned uc,uc2; + if (*str!='\"') {ep=str;return 0;} /* not a string! */ + + while (*ptr!='\"' && *ptr && ++len) if (*ptr++ == '\\') ptr++; /* Skip escaped quotes. */ + + out=(char*)cJSON_malloc(len+1); /* This is how long we need for the string, roughly. */ + if (!out) return 0; + + ptr=str+1;ptr2=out; + while (*ptr!='\"' && *ptr) + { + if (*ptr!='\\') *ptr2++=*ptr++; + else + { + ptr++; + switch (*ptr) + { + case 'b': *ptr2++='\b'; break; + case 'f': *ptr2++='\f'; break; + case 'n': *ptr2++='\n'; break; + case 'r': *ptr2++='\r'; break; + case 't': *ptr2++='\t'; break; + case 'u': /* transcode utf16 to utf8. */ + uc=parse_hex4(ptr+1);ptr+=4; /* get the unicode char. */ + + if ((uc>=0xDC00 && uc<=0xDFFF) || uc==0) break; /* check for invalid. */ + + if (uc>=0xD800 && uc<=0xDBFF) /* UTF16 surrogate pairs. */ + { + if (ptr[1]!='\\' || ptr[2]!='u') break; /* missing second-half of surrogate. */ + uc2=parse_hex4(ptr+3);ptr+=6; + if (uc2<0xDC00 || uc2>0xDFFF) break; /* invalid second-half of surrogate. */ + uc=0x10000 + (((uc&0x3FF)<<10) | (uc2&0x3FF)); + } + + len=4;if (uc<0x80) len=1;else if (uc<0x800) len=2;else if (uc<0x10000) len=3; ptr2+=len; + + switch (len) { + case 4: *--ptr2 =((uc | 0x80) & 0xBF); uc >>= 6; + case 3: *--ptr2 =((uc | 0x80) & 0xBF); uc >>= 6; + case 2: *--ptr2 =((uc | 0x80) & 0xBF); uc >>= 6; + case 1: *--ptr2 =(uc | firstByteMark[len]); + } + ptr2+=len; + break; + default: *ptr2++=*ptr; break; + } + ptr++; + } + } + *ptr2=0; + if (*ptr=='\"') ptr++; + item->valuestring=out; + item->type=cJSON_String; + return ptr; +} + +/* Render the cstring provided to an escaped version that can be printed. */ +static char *print_string_ptr(const char *str) +{ + const char *ptr;char *ptr2,*out;int len=0;unsigned char token; + + if (!str) return cJSON_strdup(""); + ptr=str;while ((token=*ptr) && ++len) {if (strchr("\"\\\b\f\n\r\t",token)) len++; else if (token<32) len+=5;ptr++;} + + out=(char*)cJSON_malloc(len+3); + if (!out) return 0; + + ptr2=out;ptr=str; + *ptr2++='\"'; + while (*ptr) + { + if ((unsigned char)*ptr>31 && *ptr!='\"' && *ptr!='\\') *ptr2++=*ptr++; + else + { + *ptr2++='\\'; + switch (token=*ptr++) + { + case '\\': *ptr2++='\\'; break; + case '\"': *ptr2++='\"'; break; + case '\b': *ptr2++='b'; break; + case '\f': *ptr2++='f'; break; + case '\n': *ptr2++='n'; break; + case '\r': *ptr2++='r'; break; + case '\t': *ptr2++='t'; break; + default: sprintf(ptr2,"u%04x",token);ptr2+=5; break; /* escape and print */ + } + } + } + *ptr2++='\"';*ptr2++=0; + return out; +} +/* Invote print_string_ptr (which is useful) on an item. */ +static char *print_string(cJSON *item) {return print_string_ptr(item->valuestring);} + +/* Predeclare these prototypes. */ +static const char *parse_value(cJSON *item,const char *value); +static char *print_value(cJSON *item,int depth,int fmt); +static const char *parse_array(cJSON *item,const char *value); +static char *print_array(cJSON *item,int depth,int fmt); +static const char *parse_object(cJSON *item,const char *value); +static char *print_object(cJSON *item,int depth,int fmt); + +/* Utility to jump whitespace and cr/lf */ +static const char *skip(const char *in) {while (in && *in && (unsigned char)*in<=32) in++; return in;} + +/* Parse an object - create a new root, and populate. */ +cJSON *cJSON_ParseWithOpts(const char *value,const char **return_parse_end,int require_null_terminated) +{ + const char *end=0; + cJSON *c=cJSON_New_Item(); + ep=0; + if (!c) return 0; /* memory fail */ + + end=parse_value(c,skip(value)); + if (!end) {cJSON_Delete(c);return 0;} /* parse failure. ep is set. */ + + /* if we require null-terminated JSON without appended garbage, skip and then check for a null terminator */ + if (require_null_terminated) {end=skip(end);if (*end) {cJSON_Delete(c);ep=end;return 0;}} + if (return_parse_end) *return_parse_end=end; + return c; +} +/* Default options for cJSON_Parse */ +cJSON *cJSON_Parse(const char *value) {return cJSON_ParseWithOpts(value,0,0);} + +/* Render a cJSON item/entity/structure to text. */ +char *cJSON_Print(cJSON *item) {return print_value(item,0,1);} +char *cJSON_PrintUnformatted(cJSON *item) {return print_value(item,0,0);} + +/* Parser core - when encountering text, process appropriately. */ +static const char *parse_value(cJSON *item,const char *value) +{ + if (!value) return 0; /* Fail on null. */ + if (!strncmp(value,"null",4)) { item->type=cJSON_NULL; return value+4; } + if (!strncmp(value,"false",5)) { item->type=cJSON_False; return value+5; } + if (!strncmp(value,"true",4)) { item->type=cJSON_True; item->valueint=1; return value+4; } + if (*value=='\"') { return parse_string(item,value); } + if (*value=='-' || (*value>='0' && *value<='9')) { return parse_number(item,value); } + if (*value=='[') { return parse_array(item,value); } + if (*value=='{') { return parse_object(item,value); } + + ep=value;return 0; /* failure. */ +} + +/* Render a value to text. */ +static char *print_value(cJSON *item,int depth,int fmt) +{ + char *out=0; + if (!item) return 0; + switch ((item->type)&255) + { + case cJSON_NULL: out=cJSON_strdup("null"); break; + case cJSON_False: out=cJSON_strdup("false");break; + case cJSON_True: out=cJSON_strdup("true"); break; + case cJSON_Number: out=print_number(item);break; + case cJSON_String: out=print_string(item);break; + case cJSON_Array: out=print_array(item,depth,fmt);break; + case cJSON_Object: out=print_object(item,depth,fmt);break; + } + return out; +} + +/* Build an array from input text. */ +static const char *parse_array(cJSON *item,const char *value) +{ + cJSON *child; + if (*value!='[') {ep=value;return 0;} /* not an array! */ + + item->type=cJSON_Array; + value=skip(value+1); + if (*value==']') return value+1; /* empty array. */ + + item->child=child=cJSON_New_Item(); + if (!item->child) return 0; /* memory fail */ + value=skip(parse_value(child,skip(value))); /* skip any spacing, get the value. */ + if (!value) return 0; + + while (*value==',') + { + cJSON *new_item; + if (!(new_item=cJSON_New_Item())) return 0; /* memory fail */ + child->next=new_item;new_item->prev=child;child=new_item; + value=skip(parse_value(child,skip(value+1))); + if (!value) return 0; /* memory fail */ + } + + if (*value==']') return value+1; /* end of array */ + ep=value;return 0; /* malformed. */ +} + +/* Render an array to text */ +static char *print_array(cJSON *item,int depth,int fmt) +{ + char **entries; + char *out=0,*ptr,*ret;int len=5; + cJSON *child=item->child; + int numentries=0,i=0,fail=0; + + /* How many entries in the array? */ + while (child) numentries++,child=child->next; + /* Explicitly handle numentries==0 */ + if (!numentries) + { + out=(char*)cJSON_malloc(3); + if (out) strcpy(out,"[]"); + return out; + } + /* Allocate an array to hold the values for each */ + entries=(char**)cJSON_malloc(numentries*sizeof(char*)); + if (!entries) return 0; + memset(entries,0,numentries*sizeof(char*)); + /* Retrieve all the results: */ + child=item->child; + while (child && !fail) + { + ret=print_value(child,depth+1,fmt); + entries[i++]=ret; + if (ret) len+=strlen(ret)+2+(fmt?1:0); else fail=1; + child=child->next; + } + + /* If we didn't fail, try to malloc the output string */ + if (!fail) out=(char*)cJSON_malloc(len); + /* If that fails, we fail. */ + if (!out) fail=1; + + /* Handle failure. */ + if (fail) + { + for (i=0;itype=cJSON_Object; + value=skip(value+1); + if (*value=='}') return value+1; /* empty array. */ + + item->child=child=cJSON_New_Item(); + if (!item->child) return 0; + value=skip(parse_string(child,skip(value))); + if (!value) return 0; + child->string=child->valuestring;child->valuestring=0; + if (*value!=':') {ep=value;return 0;} /* fail! */ + value=skip(parse_value(child,skip(value+1))); /* skip any spacing, get the value. */ + if (!value) return 0; + + while (*value==',') + { + cJSON *new_item; + if (!(new_item=cJSON_New_Item())) return 0; /* memory fail */ + child->next=new_item;new_item->prev=child;child=new_item; + value=skip(parse_string(child,skip(value+1))); + if (!value) return 0; + child->string=child->valuestring;child->valuestring=0; + if (*value!=':') {ep=value;return 0;} /* fail! */ + value=skip(parse_value(child,skip(value+1))); /* skip any spacing, get the value. */ + if (!value) return 0; + } + + if (*value=='}') return value+1; /* end of array */ + ep=value;return 0; /* malformed. */ +} + +/* Render an object to text. */ +static char *print_object(cJSON *item,int depth,int fmt) +{ + char **entries=0,**names=0; + char *out=0,*ptr,*ret,*str;int len=7,i=0,j; + cJSON *child=item->child; + int numentries=0,fail=0; + /* Count the number of entries. */ + while (child) numentries++,child=child->next; + /* Explicitly handle empty object case */ + if (!numentries) + { + out=(char*)cJSON_malloc(fmt?depth+4:3); + if (!out) return 0; + ptr=out;*ptr++='{'; + if (fmt) {*ptr++='\n';for (i=0;ichild;depth++;if (fmt) len+=depth; + while (child) + { + names[i]=str=print_string_ptr(child->string); + entries[i++]=ret=print_value(child,depth,fmt); + if (str && ret) len+=strlen(ret)+strlen(str)+2+(fmt?2+depth:0); else fail=1; + child=child->next; + } + + /* Try to allocate the output string */ + if (!fail) out=(char*)cJSON_malloc(len); + if (!out) fail=1; + + /* Handle failure */ + if (fail) + { + for (i=0;ichild;int i=0;while(c)i++,c=c->next;return i;} +cJSON *cJSON_GetArrayItem(cJSON *array,int item) {cJSON *c=array->child; while (c && item>0) item--,c=c->next; return c;} +cJSON *cJSON_GetObjectItem(cJSON *object,const char *string) {cJSON *c=object->child; while (c && cJSON_strcasecmp(c->string,string)) c=c->next; return c;} + +/* Utility for array list handling. */ +static void suffix_object(cJSON *prev,cJSON *item) {prev->next=item;item->prev=prev;} +/* Utility for handling references. */ +static cJSON *create_reference(cJSON *item) {cJSON *ref=cJSON_New_Item();if (!ref) return 0;memcpy(ref,item,sizeof(cJSON));ref->string=0;ref->type|=cJSON_IsReference;ref->next=ref->prev=0;return ref;} + +/* Add item to array/object. */ +void cJSON_AddItemToArray(cJSON *array, cJSON *item) {cJSON *c=array->child;if (!item) return; if (!c) {array->child=item;} else {while (c && c->next) c=c->next; suffix_object(c,item);}} +void cJSON_AddItemToObject(cJSON *object,const char *string,cJSON *item) {if (!item) return; if (item->string) cJSON_free(item->string);item->string=cJSON_strdup(string);cJSON_AddItemToArray(object,item);} +void cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item) {cJSON_AddItemToArray(array,create_reference(item));} +void cJSON_AddItemReferenceToObject(cJSON *object,const char *string,cJSON *item) {cJSON_AddItemToObject(object,string,create_reference(item));} + +cJSON *cJSON_DetachItemFromArray(cJSON *array,int which) {cJSON *c=array->child;while (c && which>0) c=c->next,which--;if (!c) return 0; + if (c->prev) c->prev->next=c->next;if (c->next) c->next->prev=c->prev;if (c==array->child) array->child=c->next;c->prev=c->next=0;return c;} +void cJSON_DeleteItemFromArray(cJSON *array,int which) {cJSON_Delete(cJSON_DetachItemFromArray(array,which));} +cJSON *cJSON_DetachItemFromObject(cJSON *object,const char *string) {int i=0;cJSON *c=object->child;while (c && cJSON_strcasecmp(c->string,string)) i++,c=c->next;if (c) return cJSON_DetachItemFromArray(object,i);return 0;} +void cJSON_DeleteItemFromObject(cJSON *object,const char *string) {cJSON_Delete(cJSON_DetachItemFromObject(object,string));} + +/* Replace array/object items with new ones. */ +void cJSON_ReplaceItemInArray(cJSON *array,int which,cJSON *newitem) {cJSON *c=array->child;while (c && which>0) c=c->next,which--;if (!c) return; + newitem->next=c->next;newitem->prev=c->prev;if (newitem->next) newitem->next->prev=newitem; + if (c==array->child) array->child=newitem; else newitem->prev->next=newitem;c->next=c->prev=0;cJSON_Delete(c);} +void cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem){int i=0;cJSON *c=object->child;while(c && cJSON_strcasecmp(c->string,string))i++,c=c->next;if(c){newitem->string=cJSON_strdup(string);cJSON_ReplaceItemInArray(object,i,newitem);}} + +/* Create basic types: */ +cJSON *cJSON_CreateNull(void) {cJSON *item=cJSON_New_Item();if(item)item->type=cJSON_NULL;return item;} +cJSON *cJSON_CreateTrue(void) {cJSON *item=cJSON_New_Item();if(item)item->type=cJSON_True;return item;} +cJSON *cJSON_CreateFalse(void) {cJSON *item=cJSON_New_Item();if(item)item->type=cJSON_False;return item;} +cJSON *cJSON_CreateBool(int b) {cJSON *item=cJSON_New_Item();if(item)item->type=b?cJSON_True:cJSON_False;return item;} +cJSON *cJSON_CreateNumber(double num) {cJSON *item=cJSON_New_Item();if(item){item->type=cJSON_Number;item->valuedouble=num;item->valueint=(int)num;}return item;} +cJSON *cJSON_CreateString(const char *string) {cJSON *item=cJSON_New_Item();if(item){item->type=cJSON_String;item->valuestring=cJSON_strdup(string);}return item;} +cJSON *cJSON_CreateArray(void) {cJSON *item=cJSON_New_Item();if(item)item->type=cJSON_Array;return item;} +cJSON *cJSON_CreateObject(void) {cJSON *item=cJSON_New_Item();if(item)item->type=cJSON_Object;return item;} + +/* Create Arrays: */ +cJSON *cJSON_CreateIntArray(const int *numbers,int count) {int i;cJSON *n=0,*p=0,*a=cJSON_CreateArray();for(i=0;a && ichild=n;else suffix_object(p,n);p=n;}return a;} +cJSON *cJSON_CreateFloatArray(const float *numbers,int count) {int i;cJSON *n=0,*p=0,*a=cJSON_CreateArray();for(i=0;a && ichild=n;else suffix_object(p,n);p=n;}return a;} +cJSON *cJSON_CreateDoubleArray(const double *numbers,int count) {int i;cJSON *n=0,*p=0,*a=cJSON_CreateArray();for(i=0;a && ichild=n;else suffix_object(p,n);p=n;}return a;} +cJSON *cJSON_CreateStringArray(const char **strings,int count) {int i;cJSON *n=0,*p=0,*a=cJSON_CreateArray();for(i=0;a && ichild=n;else suffix_object(p,n);p=n;}return a;} + +/* Duplication */ +cJSON *cJSON_Duplicate(cJSON *item,int recurse) +{ + cJSON *newitem,*cptr,*nptr=0,*newchild; + /* Bail on bad ptr */ + if (!item) return 0; + /* Create new item */ + newitem=cJSON_New_Item(); + if (!newitem) return 0; + /* Copy over all vars */ + newitem->type=item->type&(~cJSON_IsReference),newitem->valueint=item->valueint,newitem->valuedouble=item->valuedouble; + if (item->valuestring) {newitem->valuestring=cJSON_strdup(item->valuestring); if (!newitem->valuestring) {cJSON_Delete(newitem);return 0;}} + if (item->string) {newitem->string=cJSON_strdup(item->string); if (!newitem->string) {cJSON_Delete(newitem);return 0;}} + /* If non-recursive, then we're done! */ + if (!recurse) return newitem; + /* Walk the ->next chain for the child. */ + cptr=item->child; + while (cptr) + { + newchild=cJSON_Duplicate(cptr,1); /* Duplicate (with recurse) each item in the ->next chain */ + if (!newchild) {cJSON_Delete(newitem);return 0;} + if (nptr) {nptr->next=newchild,newchild->prev=nptr;nptr=newchild;} /* If newitem->child already set, then crosswire ->prev and ->next and move on */ + else {newitem->child=newchild;nptr=newchild;} /* Set newitem->child and move to it */ + cptr=cptr->next; + } + return newitem; +} + +void cJSON_Minify(char *json) +{ + char *into=json; + while (*json) + { + if (*json==' ') json++; + else if (*json=='\t') json++; // Whitespace characters. + else if (*json=='\r') json++; + else if (*json=='\n') json++; + else if (*json=='/' && json[1]=='/') while (*json && *json!='\n') json++; // double-slash comments, to end of line. + else if (*json=='/' && json[1]=='*') {while (*json && !(*json=='*' && json[1]=='/')) json++;json+=2;} // multiline comments. + else if (*json=='\"'){*into++=*json++;while (*json && *json!='\"'){if (*json=='\\') *into++=*json++;*into++=*json++;}*into++=*json++;} // string literals, which are \" sensitive. + else *into++=*json++; // All other characters. + } + *into=0; // and null-terminate. +} diff --git a/src/entry/cJSON.h b/src/entry/cJSON.h new file mode 100644 index 0000000..867b7c3 --- /dev/null +++ b/src/entry/cJSON.h @@ -0,0 +1,143 @@ +/* + Copyright (c) 2009 Dave Gamble + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +#ifndef cJSON__h +#define cJSON__h + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* cJSON Types: */ +#define cJSON_False 0 +#define cJSON_True 1 +#define cJSON_NULL 2 +#define cJSON_Number 3 +#define cJSON_String 4 +#define cJSON_Array 5 +#define cJSON_Object 6 + +#define cJSON_IsReference 256 + +/* The cJSON structure: */ +typedef struct cJSON { + struct cJSON *next,*prev; /* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */ + struct cJSON *child; /* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */ + + int type; /* The type of the item, as above. */ + + char *valuestring; /* The item's string, if type==cJSON_String */ + int valueint; /* The item's number, if type==cJSON_Number */ + double valuedouble; /* The item's number, if type==cJSON_Number */ + + char *string; /* The item's name string, if this item is the child of, or is in the list of subitems of an object. */ +} cJSON; + +typedef struct cJSON_Hooks { + void *(*malloc_fn)(size_t sz); + void (*free_fn)(void *ptr); +} cJSON_Hooks; + +/* Supply malloc, realloc and free functions to cJSON */ +extern void cJSON_InitHooks(cJSON_Hooks* hooks); + + +/* Supply a block of JSON, and this returns a cJSON object you can interrogate. Call cJSON_Delete when finished. */ +extern cJSON *cJSON_Parse(const char *value); +/* Render a cJSON entity to text for transfer/storage. Free the char* when finished. */ +extern char *cJSON_Print(cJSON *item); +/* Render a cJSON entity to text for transfer/storage without any formatting. Free the char* when finished. */ +extern char *cJSON_PrintUnformatted(cJSON *item); +/* Delete a cJSON entity and all subentities. */ +extern void cJSON_Delete(cJSON *c); + +/* Returns the number of items in an array (or object). */ +extern int cJSON_GetArraySize(cJSON *array); +/* Retrieve item number "item" from array "array". Returns NULL if unsuccessful. */ +extern cJSON *cJSON_GetArrayItem(cJSON *array,int item); +/* Get item "string" from object. Case insensitive. */ +extern cJSON *cJSON_GetObjectItem(cJSON *object,const char *string); + +/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */ +extern const char *cJSON_GetErrorPtr(void); + +/* These calls create a cJSON item of the appropriate type. */ +extern cJSON *cJSON_CreateNull(void); +extern cJSON *cJSON_CreateTrue(void); +extern cJSON *cJSON_CreateFalse(void); +extern cJSON *cJSON_CreateBool(int b); +extern cJSON *cJSON_CreateNumber(double num); +extern cJSON *cJSON_CreateString(const char *string); +extern cJSON *cJSON_CreateArray(void); +extern cJSON *cJSON_CreateObject(void); + +/* These utilities create an Array of count items. */ +extern cJSON *cJSON_CreateIntArray(const int *numbers,int count); +extern cJSON *cJSON_CreateFloatArray(const float *numbers,int count); +extern cJSON *cJSON_CreateDoubleArray(const double *numbers,int count); +extern cJSON *cJSON_CreateStringArray(const char **strings,int count); + +/* Append item to the specified array/object. */ +extern void cJSON_AddItemToArray(cJSON *array, cJSON *item); +extern void cJSON_AddItemToObject(cJSON *object,const char *string,cJSON *item); +/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */ +extern void cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item); +extern void cJSON_AddItemReferenceToObject(cJSON *object,const char *string,cJSON *item); + +/* Remove/Detatch items from Arrays/Objects. */ +extern cJSON *cJSON_DetachItemFromArray(cJSON *array,int which); +extern void cJSON_DeleteItemFromArray(cJSON *array,int which); +extern cJSON *cJSON_DetachItemFromObject(cJSON *object,const char *string); +extern void cJSON_DeleteItemFromObject(cJSON *object,const char *string); + +/* Update array items. */ +extern void cJSON_ReplaceItemInArray(cJSON *array,int which,cJSON *newitem); +extern void cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem); + +/* Duplicate a cJSON item */ +extern cJSON *cJSON_Duplicate(cJSON *item,int recurse); +/* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will +need to be released. With recurse!=0, it will duplicate any children connected to the item. +The item->next and ->prev pointers are always zero on return from Duplicate. */ + +/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */ +extern cJSON *cJSON_ParseWithOpts(const char *value,const char **return_parse_end,int require_null_terminated); + +extern void cJSON_Minify(char *json); + +/* Macros for creating things quickly. */ +#define cJSON_AddNullToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateNull()) +#define cJSON_AddTrueToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateTrue()) +#define cJSON_AddFalseToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateFalse()) +#define cJSON_AddBoolToObject(object,name,b) cJSON_AddItemToObject(object, name, cJSON_CreateBool(b)) +#define cJSON_AddNumberToObject(object,name,n) cJSON_AddItemToObject(object, name, cJSON_CreateNumber(n)) +#define cJSON_AddStringToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateString(s)) + +/* When assigning an integer value, it needs to be propagated to valuedouble too. */ +#define cJSON_SetIntValue(object,val) ((object)?(object)->valueint=(object)->valuedouble=(val):(val)) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/entry/config_monitor.cpp b/src/entry/config_monitor.cpp new file mode 100644 index 0000000..fe5f6cc --- /dev/null +++ b/src/entry/config_monitor.cpp @@ -0,0 +1,321 @@ +#include "MESA_handle_logger.h" +#include "config_monitor.h" +#include +#include +#include +#include + +const char* module_config_monitor="CONFIG_MONITOR"; + +#define CM_UPDATE_TYPE_ERR -1 +#define CM_UPDATE_TYPE_NONE 0 + +#define CM_MAX_TABLE_NUM 256 +#define MAX_CONFIG_FN_LEN 256 +#define MAX_CONFIG_LINE 1024*4 +//#define USING_DICTATOR 1 +extern "C" void __real_free(void*p); +struct cm_table_info_t +{ + char table_name[MAX_CONFIG_FN_LEN]; + char cfg_path[MAX_CONFIG_FN_LEN]; + int cfg_num; +}; +//replacement of glibc scandir, to adapt dictator malloc wrap +#define ENLARGE_STEP 1024 +int my_scandir(const char *dir, struct dirent ***namelist, + int(*filter)(const struct dirent *), + int(*compar)(const void *, const void *)) +{ + DIR * od; + int n = 0; + int DIR_ENT_SIZE=ENLARGE_STEP; + struct dirent ** list = NULL; + struct dirent * p; + struct dirent entry,*result; + + if((dir == NULL) || (namelist == NULL)) + return -1; + + od = opendir(dir); + if(od == NULL) + return -1; + + list = (struct dirent **)malloc(DIR_ENT_SIZE*sizeof(struct dirent *)); + + + while(0==readdir_r(od,&entry,&result)) + { + if(result==NULL) + { + break; + } + if( filter && !filter(&entry)) + continue; + + p = (struct dirent *)malloc(sizeof(struct dirent)); + memcpy((void *)p,(void *)(&entry),sizeof(struct dirent)); + list[n] = p; + + n++; + if(n >= DIR_ENT_SIZE) + { + DIR_ENT_SIZE+=ENLARGE_STEP; + list=(struct dirent **)realloc((void*)list,DIR_ENT_SIZE*sizeof(struct dirent *)); + + } + } + + closedir(od); + + *namelist = list; + + if(compar) + qsort((void *)*namelist,n,sizeof(struct dirent *),compar); + + return n; + +} + int filter_fn(const struct dirent * ent) + { + if(ent->d_type != DT_REG) + return 0; + + return (strncmp(ent->d_name,"full_config_index",strlen("full_config_index")) == 0|| + strncmp(ent->d_name,"inc_config_index",strlen("inc_config_index")) == 0); + } +int get_new_idx_path(unsigned int current_version,const char*file_dir,void* logger,char*** idx_path,int*idx_num) +{ + struct dirent **namelist; + int n=0,i=0,sscanf_ret; + + char update_str[32]={0}; + unsigned int latest_ful_version=0,latest_inc_version=0; + unsigned int config_seq=0; + int *inc_file_idx; + int full_file_idx=0,inc_idx_num=0,path_len=0; + + int update_type=CM_UPDATE_TYPE_NONE; + + + n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort); + if (n < 0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"scan dir error"); + update_type=CM_UPDATE_TYPE_ERR; + return update_type; + } + inc_file_idx=(int*)calloc(sizeof(int),n); + inc_idx_num=0; + for(i=0;id_name, ".") == 0) || (strcmp(namelist[i]->d_name, "..") == 0)) + { + continue; + } + if(strlen(namelist[i]->d_name)>32) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor + ,"config file %s filename too long,should like full_config_index.0000000001" + ,namelist[i]->d_name); + continue; + } + sscanf_ret=sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%u",update_str,&config_seq); + if(sscanf_ret!=2) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor + ,"config file %s filename error,should like full_config_index.0000000001" + ,namelist[i]->d_name); + continue; + } + if(strncasecmp(update_str,"full",strlen(update_str))==0) + { + if(config_seq>latest_ful_version) + { + latest_ful_version=config_seq; + full_file_idx=i; + } + } + else if(strncasecmp(update_str,"inc",strlen(update_str))==0) + { + if(config_seq>current_version) + { + inc_file_idx[inc_idx_num]=i; + inc_idx_num++; + if(config_seq>latest_inc_version) + { + latest_inc_version=config_seq; + } + + } + } + else + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor + ,"config file %s,not full or inc config" + ,namelist[i]->d_name); + } + } + //full update + if(latest_ful_version>current_version) + { + *idx_path=(char**)malloc(sizeof(char**)); + path_len=strlen(file_dir)+strlen(namelist[full_file_idx]->d_name)+1+1; + (*idx_path)[0]=(char*)malloc(path_len); + snprintf((*idx_path)[0],path_len,"%s/%s",file_dir,namelist[full_file_idx]->d_name); + *idx_num=1; + update_type=CM_UPDATE_TYPE_FULL; + } + //inc update,it's possible that do inc after full update in this function,but we'll process it at next call. + else if(latest_inc_version>current_version) + { + + *idx_path=(char**)malloc(sizeof(char**)*inc_idx_num); + for(i=0;id_name)+1+1; + (*idx_path)[i]=(char*)malloc(path_len); + snprintf((*idx_path)[i],path_len,"%s/%s",file_dir,namelist[inc_file_idx[i]]->d_name); + + } + *idx_num=inc_idx_num; + update_type=CM_UPDATE_TYPE_INC; + + } + else + { + update_type=CM_UPDATE_TYPE_NONE; + } + free(inc_file_idx); + for(i=0;i0)//jump over empty line + { + i++; + } + if(i==size) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor ,"Too much lines in %s",path); + break; + } + } + fclose(fp); + return i; +} + +int cm_read_table_file(struct cm_table_info_t* index, + void (*update)(const char*,const char*,void*), + void* u_para, + void* logger) +{ + int cfg_num=0,i=0; + char line[MAX_CONFIG_LINE]={0},*ret_str=NULL; + FILE*fp=fopen(index->cfg_path,"r"); + if(fp==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error,open %s failed.",index->cfg_path); + return -1; + } + fscanf(fp,"%d\n",&cfg_num); + if(cfg_num!=index->cfg_num) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor ,"file %s config num not matched",index->cfg_path); + fclose(fp); + return -1; + } + for(i=0;icfg_path,i,cfg_num); + break; + } + if(line[sizeof(line)-1]!='\0') + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor , + "update error,line size more than %u at of file %s:%d", + sizeof(line),index->cfg_path,i); + continue; + } + update(index->table_name,line,u_para); + } + fclose(fp); + return 0; +} +const char* path2filename(const char*path) +{ + int i=0; + for(i=strlen(path);i>0;i--) + { + if(path[i]=='/') + { + break; + } + } + return path+i+1; +} +void config_monitor_traverse(unsigned int version,const char*idx_dir, + void (*start)(unsigned int ,int ,void*), + void (*update)(const char* ,const char*,void* ), + void (*finish)(void*), + void* u_para, + void* logger) +{ + + int update_type=CM_UPDATE_TYPE_NONE; + unsigned int new_version=0; + char**idx_path_array=NULL; + const char* table_filename=NULL; + char str_not_care[256]={0}; + int idx_num=0,table_num=0,i=0,j=0; + struct cm_table_info_t table_array[CM_MAX_TABLE_NUM]; + update_type=get_new_idx_path(version, idx_dir,logger, &idx_path_array, &idx_num); + if(update_type==CM_UPDATE_TYPE_FULL||update_type==CM_UPDATE_TYPE_INC) + { + + for(i=0;i +#include +int dynamic_array_VERSION_20141202=0; +struct dynamic_array_t* dynamic_array_create(int size,int step) +{ + struct dynamic_array_t* d_array=(struct dynamic_array_t*)calloc(sizeof(struct dynamic_array_t),1); + d_array->array=(void**)calloc(sizeof(void*),size); + d_array->size=size; + d_array->enlarge_step=step; + return d_array; +} +void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(void *)) +{ + int i; + if(free_data!=NULL) + { + for(i=0;isize;i++) + { + free_data(d_array->array[i]); + } + } + free(d_array->array); + free(d_array); +} +void* dynamic_array_read(struct dynamic_array_t* d_array,int i) +{ + if(isize) + { + return d_array->array[i]; + } + else + { + return NULL; + } +} +void dynamic_array_write(struct dynamic_array_t* d_array,int i,void* data) +{ + int new_size=0; + if(isize) + { + d_array->array[i]=data; + } + else + { + new_size=i+d_array->enlarge_step; + d_array->array=(void**)realloc(d_array->array,new_size*sizeof(void*)); + memset(d_array->array+d_array->size,0,(new_size-d_array->size)*sizeof(void*)); + d_array->size=new_size; + d_array->array[i]=data; + return; + } +} diff --git a/src/entry/dynamic_array.h b/src/entry/dynamic_array.h new file mode 100644 index 0000000..7583c92 --- /dev/null +++ b/src/entry/dynamic_array.h @@ -0,0 +1,13 @@ +#ifndef _DYNAMIC_ARRAY_H_INCLUDE_ +#define _DYNAMIC_ARRAY_H_INCLUDE_ +struct dynamic_array_t +{ + void ** array; + int size; + int enlarge_step; +}; +struct dynamic_array_t* dynamic_array_create(int size,int step); +void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(void *)); +void* dynamic_array_read(struct dynamic_array_t* d_array,int i); +void dynamic_array_write(struct dynamic_array_t* d_array,int i,void* data); +#endif //_DYNAMIC_ARRAY_H_INCLUDE_ diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp new file mode 100644 index 0000000..f33e7d8 --- /dev/null +++ b/src/entry/json2iris.cpp @@ -0,0 +1,919 @@ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cJSON.h" +#include "map_str2int.h" +#include "Maat_rule_internal.h" +const char* maat_json="maat json"; +const int json_version=1; +#define MAX_PATH_LINE 256 +#define MAX_COLUMN_NUM 32 +struct group_info_t +{ + int group_id; +}; +struct iris_table_t +{ + char table_name[MAX_PATH_LINE]; + char table_path[MAX_PATH_LINE]; + int line_count; +}; +struct iris_description_t +{ + int group_cnt; + int region_cnt; + + char tmp_iris_dir[MAX_PATH_LINE]; + char tmp_iris_index_dir[MAX_PATH_LINE]; + char index_path[MAX_PATH_LINE]; + + struct iris_table_t group_table; + struct iris_table_t compile_table; + MESA_htable_handle group_name_map; + MESA_htable_handle iris_table_map; + MESA_htable_handle str2int_map; + +}; +struct traslate_command_t +{ + const char* json_string; + char* json_value; + int json_type; + int str2int_flag; + int empty_allowed; + const char* default_string; + int default_int; +}; +int set_iris_descriptor(const char* json_file,cJSON *json,struct iris_description_t *iris_cfg,void * logger) +{ + cJSON* j=NULL; + char* table_name=NULL; + memset(iris_cfg,0,sizeof(struct iris_description_t)); + snprintf(iris_cfg->tmp_iris_dir,sizeof(iris_cfg->tmp_iris_dir),"%s_iris_tmp",json_file); + snprintf(iris_cfg->tmp_iris_index_dir,sizeof(iris_cfg->tmp_iris_index_dir),"%s_iris_tmp/index",json_file); + snprintf(iris_cfg->index_path,sizeof(iris_cfg->index_path),"%s/full_config_index.%010d",iris_cfg->tmp_iris_index_dir,json_version); + + j=cJSON_GetObjectItem(json, "compile_table"); + if(j==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile_table not defined."); + return -1; + } + table_name=j->valuestring; + memcpy(iris_cfg->compile_table.table_name,table_name,strlen(table_name)); + snprintf(iris_cfg->compile_table.table_path,sizeof(iris_cfg->compile_table.table_path), + "%s/%s.local",iris_cfg->tmp_iris_dir,iris_cfg->compile_table.table_name); + j=cJSON_GetObjectItem(json, "group_table"); + if(j==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "group_table not defined."); + return -1; + } + table_name=j->valuestring; + memcpy(iris_cfg->group_table.table_name,table_name,strlen(table_name)); + snprintf(iris_cfg->group_table.table_path,sizeof(iris_cfg->group_table.table_name), + "%s/%s.local",iris_cfg->tmp_iris_dir,iris_cfg->group_table.table_name); + + MESA_htable_create_args_t hargs; + memset(&hargs,0,sizeof(hargs)); + hargs.thread_safe=1; + hargs.hash_slot_size = 1024; + hargs.max_elem_num = 0; + hargs.eliminate_type = HASH_ELIMINATE_ALGO_LRU; + hargs.expire_time = 0; + hargs.key_comp = NULL; + hargs.key2index = NULL; + hargs.recursive = 0; + hargs.data_free = free; + hargs.data_expire_with_condition = NULL; + + iris_cfg->group_name_map=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(iris_cfg->group_name_map, 0); + + iris_cfg->iris_table_map=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(iris_cfg->iris_table_map, 0); + + iris_cfg->str2int_map=map_create(); + + map_register(iris_cfg->str2int_map, "yes",1); + map_register(iris_cfg->str2int_map, "no",0); + + map_register(iris_cfg->str2int_map, "ip",TABLE_TYPE_IP); + map_register(iris_cfg->str2int_map, "string",TABLE_TYPE_EXPR); + map_register(iris_cfg->str2int_map, "intval",TABLE_TYPE_INTVAL); + + map_register(iris_cfg->str2int_map, "ipv4",4); + map_register(iris_cfg->str2int_map, "ipv6",6); + + map_register(iris_cfg->str2int_map, "double",0); + map_register(iris_cfg->str2int_map, "single",1); + + map_register(iris_cfg->str2int_map, "none",0); + map_register(iris_cfg->str2int_map, "and",1); + map_register(iris_cfg->str2int_map, "regex",2); + map_register(iris_cfg->str2int_map, "offset",3); + + map_register(iris_cfg->str2int_map, "sub",0); + map_register(iris_cfg->str2int_map, "right",1); + map_register(iris_cfg->str2int_map, "left",2); + map_register(iris_cfg->str2int_map, "complete",3); + + map_register(iris_cfg->str2int_map, "uncase plain",0); + map_register(iris_cfg->str2int_map, "hexbin",1); + map_register(iris_cfg->str2int_map, "case plain",2); + + return 0; +} +void clear_iris_descriptor(struct iris_description_t *iris_cfg) +{ + if(iris_cfg->group_name_map!=NULL) + { + MESA_htable_destroy(iris_cfg->group_name_map, NULL); + } + if(iris_cfg->iris_table_map!=NULL) + { + MESA_htable_destroy(iris_cfg->iris_table_map, NULL); + } + return; +} +int create_tmp_dir(struct iris_description_t *p) +{ + + + if((access(p->tmp_iris_dir,F_OK))<0) + { + if((mkdir(p->tmp_iris_dir, 0777)) < 0) + { + return -1; + } + } + if((access(p->tmp_iris_index_dir,F_OK))<0) + { + if((mkdir(p->tmp_iris_index_dir, 0777)) < 0) + { + return -1; + } + } + return 0; +} +int set_file_rulenum(const char* path,int rulenum,void* logger) +{ + FILE* fp=NULL; + if(rulenum==0) + { + fp=fopen(path,"w"); + } + else + { + fp=fopen(path,"r+"); + } + if(fp==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "fopen %s failed %s at set rule num.",path,strerror(errno)); + return -1; + } + fprintf(fp,"%010d\n",rulenum); + fclose(fp); + return 0; +} +int direct_write_rule(cJSON* json,MESA_htable_handle str2int,struct traslate_command_t*cmd,int cmd_cnt,const char* path,void* logger) +{ + int i=0,ret=-1; + cJSON* item=NULL; + cJSON dummy; + char *p=NULL; + int int_value=0; + FILE* fp=NULL; + for(i=0;itype!=cmd[i].json_type) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "%s not defined or wrong format.",cmd[i].json_string); + goto error_out; + } + if(cmd[i].str2int_flag==1) + { + p=item->valuestring; + ret=map_str2int(str2int, p, &int_value); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "%s's value %s is not valid format.",cmd[i].json_string,p); + free(p); + goto error_out; + } + cmd[i].json_value=(char*)malloc(21);/* 2^64+1 can be represented in 21 chars. */ + snprintf(cmd[i].json_value,21,"%d",int_value); + + } + else + { + switch(item->type) + { + case cJSON_Number: + cmd[i].json_value=cJSON_Print(item); + break; + case cJSON_String: + cmd[i].json_value=(char*)calloc(strlen(item->valuestring)+1,1); + memcpy(cmd[i].json_value,item->valuestring,strlen(item->valuestring)); + break; + default://impossible ,already checked + assert(0); + break; + } + } + } + fp=fopen(path,"a"); + if(fp==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "fopen %s error %s.",path,strerror(errno)); + goto error_out; + } + for(i=0;istr2int_map,json_cmd, cmd_cnt,path,logger); +} +int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +{ + struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; + int cmd_cnt=0; + memset(json_cmd,0,sizeof(json_cmd)); + + json_cmd[cmd_cnt].json_string="region_id"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="group_id"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="keywords"; + json_cmd[cmd_cnt].json_type=cJSON_String; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="expr_type"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].str2int_flag=1; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="match_method"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].str2int_flag=1; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="format"; + json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].str2int_flag=1; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="is_valid"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + +} +int write_intval_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +{ + struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; + int cmd_cnt=0; + memset(json_cmd,0,sizeof(json_cmd)); + + json_cmd[cmd_cnt].json_string="region_id"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="group_id"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="low_boundary"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="up_boundary"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="is_valid"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + +} +struct iris_table_t* query_table_info(iris_description_t* p_iris,const char* table_name) +{ + struct iris_table_t* table_info=NULL; + table_info=(struct iris_table_t*)MESA_htable_search(p_iris->iris_table_map, (const unsigned char*)table_name,strlen(table_name)); + if(table_info==NULL) + { + table_info=(struct iris_table_t*)calloc(sizeof(struct iris_table_t),1); + table_info->line_count=0; + memcpy(table_info->table_name,table_name,MIN(sizeof(table_info->table_name)-1, strlen(table_name))); + snprintf(table_info->table_path,sizeof(table_info->table_path),"%s/%s.local",p_iris->tmp_iris_dir,table_info->table_name); + MESA_htable_add(p_iris->iris_table_map,(const unsigned char*)table_info->table_name,strlen(table_info->table_name),table_info); + } + return table_info; +} +int write_plugin_table(cJSON* plug_table_json,int sequence,iris_description_t* p_iris,void* logger) +{ + cJSON* item=NULL,*table_content=NULL,*each_line=NULL; + struct iris_table_t* table_info=NULL; + const char* table_name=NULL,*line_content=NULL; + int ret=0,i=0,line_cnt=0; + FILE*fp=NULL; + + item=cJSON_GetObjectItem(plug_table_json,"table_name"); + if(item==NULL||item->type!=cJSON_String) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "The %d plugin_table's table_name not defined or format error.",sequence); + return -1; + } + table_name= item->valuestring; + table_info=query_table_info(p_iris, table_name); + table_content=cJSON_GetObjectItem(plug_table_json,"table_content"); + if(table_content==NULL||table_content->type!=cJSON_Array) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "%d plugin_table's table_content not defined or format error." + ,sequence); + return -1; + } + line_cnt=cJSON_GetArraySize(table_content); + if(table_info->line_count==0) + { + ret=set_file_rulenum(table_info->table_path,0,logger); + if(ret<0) + { + return -1; + } + } + fp=fopen(table_info->table_path,"a"); + if(fp==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "fopen %s error %s.",table_info->table_path,strerror(errno)); + return -1; + } + + for(i=0;itype!=cJSON_String) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "plugin_table %s's line %d format error.",table_info->table_name,i+1); + continue; + } + line_content=each_line->valuestring; + fprintf(fp,"%s\n",line_content); + table_info->line_count++; + } + fclose(fp); + set_file_rulenum(table_info->table_path,table_info->line_count,logger); + return 0; +} +int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_description_t* p_iris,void* logger) +{ + cJSON* item=NULL,*table_content=NULL; + int ret=0; + int region_id=0; + const char* table_name=NULL,*table_type_str=NULL; + enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_EXPR; + struct iris_table_t* table_info=NULL; + + item=cJSON_GetObjectItem(region_json,"table_name"); + if(item==NULL||item->type!=cJSON_String) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d's region table_name not defined or format error.",compile_id); + return -1; + } + table_name=item->valuestring; + table_info=query_table_info( p_iris, table_name); + item=cJSON_GetObjectItem(region_json,"table_type"); + if(item==NULL||item->type!=cJSON_String) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d table name %s's table_type not defined or format error." + ,compile_id,table_name); + return -1; + } + table_type_str=item->valuestring; + ret=map_str2int(p_iris->str2int_map,table_type_str,(int*)&(table_type)); + if(ret!=1) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d table name %s's table_type %s invalid." + ,compile_id,table_name,table_type_str); + return -1; + } + table_content=cJSON_GetObjectItem(region_json,"table_content"); + if(table_content==NULL||table_content->type!=cJSON_Object) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d table name %s's table_content not defined or format error." + ,compile_id,table_name); + return -1; + } + if(table_info->line_count==0) + { + ret=set_file_rulenum(table_info->table_path,0,logger); + if(ret<0) + { + return -1; + } + } + region_id=p_iris->region_cnt++; + cJSON_AddNumberToObject(table_content, "region_id", region_id); + cJSON_AddNumberToObject(table_content, "group_id", group_id); + cJSON_AddNumberToObject(table_content, "is_valid", 1); + + switch(table_type) + { + case TABLE_TYPE_EXPR: + ret=write_expr_rule(table_content, p_iris, table_info->table_path, logger); + break; + case TABLE_TYPE_IP: + ret=write_ip_rule(table_content, p_iris, table_info->table_path, logger); + break; + case TABLE_TYPE_INTVAL: + ret=write_intval_rule(table_content, p_iris, table_info->table_path, logger); + break; + default: + assert(0); + break; + } + if(ret>=0) + { + table_info->line_count++; + set_file_rulenum(table_info->table_path,table_info->line_count,logger); + } + return ret; +} + +int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * logger) +{ + int compile_id=-1,cmd_cnt=0,ret=-1; + cJSON* item=NULL; + struct traslate_command_t compile_cmd[MAX_COLUMN_NUM]; + memset(compile_cmd,0,sizeof(compile_cmd)); + compile_cmd[cmd_cnt].json_string="compile_id"; + compile_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string="service"; + compile_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string="action"; + compile_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string="do_blacklist"; + compile_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string="do_log"; + compile_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string="effective_rage"; + compile_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string="user_region"; + compile_cmd[cmd_cnt].json_type=cJSON_String; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string="is_valid"; + compile_cmd[cmd_cnt].json_type=cJSON_String; + compile_cmd[cmd_cnt].str2int_flag=1; + cmd_cnt++; + + + if(p_iris->compile_table.line_count==0) + { + ret=set_file_rulenum(p_iris->compile_table.table_path, 0,logger); + if(ret<0) + { + return -1; + } + } + ret=direct_write_rule(compile, p_iris->str2int_map,compile_cmd,cmd_cnt, p_iris->compile_table.table_path,logger); + if(ret<0) + { + return -1; + } + item=cJSON_GetObjectItem(compile,"compile_id"); + if(item->type!=cJSON_Number) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile_id format not number."); + return -1; + } + compile_id=item->valueint; + p_iris->compile_table.line_count++; + set_file_rulenum(p_iris->compile_table.table_path,p_iris->compile_table.line_count,logger); + return compile_id; +} +int write_group_rule(int compile_id ,int group_id,struct iris_description_t *p_iris,void * logger) +{ + FILE*fp=NULL; + int ret=0; + + if(p_iris->group_table.line_count==0) + { + ret=set_file_rulenum(p_iris->group_table.table_path,0,logger); + if(ret<0) + { + return -1; + } + } + fp=fopen(p_iris->group_table.table_path,"a"); + if(fp==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "fopen %s error %s.",p_iris->group_table.table_path,strerror(errno)); + return -1; + } + fprintf(fp,"%d\t%d\t1\n",group_id,compile_id); + fclose(fp); + p_iris->group_table.line_count++; + ret=set_file_rulenum(p_iris->group_table.table_path,p_iris->group_table.line_count,logger); + return 0; +} +void table_idx_write_cb(const uchar * key, uint size, void * data, void * user) +{ + struct iris_table_t* p_table=(struct iris_table_t*)data; + FILE* fp=(FILE*)user; + fprintf(fp,"%s\t%d\t%s\n",p_table->table_name,p_table->line_count,p_table->table_path); +} +int write_index_file(struct iris_description_t *p_iris,void* logger) +{ + FILE*fp=NULL; + fp=fopen(p_iris->index_path,"w"); + if(fp==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "index file %s fopen error %s.",p_iris->index_path,strerror(errno)); + return -1; + } + fprintf(fp,"%s\t%d\t%s\n",p_iris->compile_table.table_name + ,p_iris->compile_table.line_count + ,p_iris->compile_table.table_path); + fprintf(fp,"%s\t%d\t%s\n",p_iris->group_table.table_name + ,p_iris->group_table.line_count + ,p_iris->group_table.table_path); + MESA_htable_iterate(p_iris->iris_table_map, table_idx_write_cb, fp); + fclose(fp); + return 0; +} +int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger) +{ + int i=0,j=0,k=0; + int compile_id=-1,compile_cnt=0,group_cnt=0,region_cnt=0,plug_table_cnt=0; + int ret=0; + cJSON *c_rules=NULL,*g_rules=NULL,*r_rules=NULL,*item=NULL,*plug_tables=NULL; + cJSON *compile_rule=NULL,*group_rule=NULL,*region_rule=NULL,*each_plug_table=NULL; + const char* group_name=NULL; + struct group_info_t *group_info=NULL; + plug_tables=cJSON_GetObjectItem(json,"plugin_table"); + if(NULL!=plug_tables) + { + plug_table_cnt=cJSON_GetArraySize(plug_tables); + for(i=0;itype!=cJSON_String) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d's group has no name.",compile_id); + return -1; + } + group_name=item->valuestring; + group_info=(struct group_info_t*)MESA_htable_search(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name)); + if(group_info!=NULL)//exist group name ,region already read + { + ret=write_group_rule(compile_id, group_info->group_id, p_iris, logger); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d write group error.",compile_id); + return -1; + } + r_rules=cJSON_GetObjectItem(group_rule,"regions"); + if(r_rules!=NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_json, + "compile rule %d's %s declared in previous compile rule, regions NOT take effect." + ,compile_id,group_name); + } + continue; + } + + group_info=(struct group_info_t*)malloc(sizeof(struct group_info_t)); + group_info->group_id=p_iris->group_cnt++; + MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info); + + r_rules=cJSON_GetObjectItem(group_rule,"regions"); + if(r_rules==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d's %s has no region.",compile_id,group_name); + return -1; + } + region_cnt=cJSON_GetArraySize(r_rules); + if(region_cnt<=0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d's %s has no region.",compile_id,group_name); + return -1; + } + for(k=0;kgroup_id, p_iris, logger); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d write region error.",compile_id); + return -1; + } + } + ret=write_group_rule(compile_id, group_info->group_id, p_iris, logger); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d write group error.",compile_id); + return -1; + } + + } + } + ret=write_index_file(p_iris,logger); + if(ret<0) + { + return -1; + } + return 0; +} +int json2iris(const char* json_file,char* iris_dir_buf,int buf_len,void* logger) +{ + FILE* json_fp=NULL; + cJSON *json=NULL; + struct stat fstat_buf; + int ret=-1; + char* json_buff=NULL; + unsigned long json_file_size=0,read_size=0; + struct iris_description_t iris_cfg; + memset(&iris_cfg,0,sizeof(iris_cfg)); + ret=stat(json_file, &fstat_buf); + if(ret!=0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "fstat file %s error.",json_file); + goto error_out; + } + + json_fp=fopen(json_file,"r"); + if(json_fp==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "fopen file %s error %s.",json_file,strerror(errno)); + goto error_out; + } + + + json_file_size=fstat_buf.st_size; + json_buff=(char*)calloc(1,json_file_size); + read_size=fread(json_buff,1,json_file_size,json_fp); + if(read_size!=json_file_size) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "fread file %s error.",json_file); + goto error_out; + } + + + json=cJSON_Parse(json_buff); + if (!json) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,"Error before: %-200.200s",cJSON_GetErrorPtr()); + goto error_out; + } + ret=set_iris_descriptor(json_file,json, &iris_cfg,logger); + if(ret<0) + { + goto error_out; + } + ret=create_tmp_dir(&iris_cfg); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "create tmp folder %s error",iris_cfg.tmp_iris_dir); + goto error_out; + } + ret=write_iris(json,&iris_cfg,logger); + if(ret<0) + { + goto error_out; + } + memcpy(iris_dir_buf,iris_cfg.tmp_iris_index_dir,MIN(strlen(iris_cfg.tmp_iris_index_dir)+1,(unsigned int)buf_len)); + + cJSON_Delete(json); + fclose(json_fp); + free(json_buff); + clear_iris_descriptor(&iris_cfg); + return 0; + + +error_out: + cJSON_Delete(json); + if(json_fp!=NULL) + { + fclose(json_fp); + } + free(json_buff); + clear_iris_descriptor(&iris_cfg); + return -1; +} + diff --git a/src/entry/json2iris.h b/src/entry/json2iris.h new file mode 100644 index 0000000..933dd53 --- /dev/null +++ b/src/entry/json2iris.h @@ -0,0 +1,2 @@ + +int json2iris(const char* json_file,char* iris_dir_buf,int buf_len,void* logger); diff --git a/src/entry/map_str2int.cpp b/src/entry/map_str2int.cpp new file mode 100644 index 0000000..07f4b38 --- /dev/null +++ b/src/entry/map_str2int.cpp @@ -0,0 +1,55 @@ +#include +void map_tmp_free(void* ptr) +{ + free(ptr); +} +MESA_htable_handle map_create(void) +{ + MESA_htable_handle string2int_map; + MESA_htable_create_args_t hargs; + memset(&hargs,0,sizeof(hargs)); + hargs.thread_safe=1; + hargs.hash_slot_size = 1024*1024; + hargs.max_elem_num = 0; + hargs.eliminate_type = HASH_ELIMINATE_ALGO_LRU; + hargs.expire_time = 0; + hargs.key_comp = NULL; + hargs.key2index = NULL; + hargs.recursive = 0; + hargs.data_free = map_tmp_free; + hargs.data_expire_with_condition = NULL; + string2int_map=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(string2int_map, 0); + return string2int_map; +} +void map_destroy(MESA_htable_handle p) +{ + MESA_htable_destroy(p,NULL); + return; +} +int map_register(MESA_htable_handle handle,const char* string,int value) +{ + unsigned int size=strlen(string); + unsigned char *key=(unsigned char *)string; + int *data=(int*)malloc(sizeof(int)); + int ret=0; + *data=value; + ret=MESA_htable_add(handle,key,size,data); + return ret; +} +int map_str2int(MESA_htable_handle handle,const char* string,int* value) +{ + int *data=NULL; + unsigned int size=strlen(string); + data=(int*)MESA_htable_search(handle,(unsigned char*)string,size); + if(data!=NULL) + { + *value=*data; + return 1; + } + else + { + return -1; + } +} + diff --git a/src/entry/map_str2int.h b/src/entry/map_str2int.h new file mode 100644 index 0000000..4652ee9 --- /dev/null +++ b/src/entry/map_str2int.h @@ -0,0 +1,8 @@ +#ifndef __MAP_STR2INT_H_INCLUDE_ +#define __MAP_STR2INT_H_INCLUDE_ +MESA_htable_handle map_create(void); +void map_destroy(MESA_htable_handle p); +int map_register(MESA_htable_handle handle,const char* string,int value); +int map_str2int(MESA_htable_handle handle,const char* string,int* value); +#endif + diff --git a/src/inc_internal/MESA_handle_logger.h b/src/inc_internal/MESA_handle_logger.h new file mode 100644 index 0000000..c7e031d --- /dev/null +++ b/src/inc_internal/MESA_handle_logger.h @@ -0,0 +1,75 @@ +#ifndef MESA_HANDLE__LOGGER_H +#define MESA_HANDLE__LOGGER_H + +/* + * runtime_log with handle, + * based on runtime_log. + * yang wei + * create time:2014-03-24 + * version:20140324 + */ + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include +#include +#include +#include +#include +#include + +#define RLOG_LV_DEBUG 10 +#define RLOG_LV_INFO 20 +#define RLOG_LV_FATAL 30 + + +#define MESA_HANDLE_RUNTIME_LOG(handle, lv, mod, fmt, args...) \ + MESA_handle_runtime_log((handle), (lv), (mod), "file %s, line %d, " fmt, \ + __FILE__, __LINE__, ##args) + +/* + * name: MESA_create_runtime_log_handle + * functionality: get runtime_log handle; + * params: + * file_path: path of log file; + * level: level of log; + * returns: + * not NULL, if succeeded; + * NULL, if file is not absolute path, or failed to create log file; + */ +void *MESA_create_runtime_log_handle(const char *file_path, int level); + +/* + * name: MESA_handle_runtime_log + * functionality: appends log message to runtime log file; + * params: + * handle:handle of runtime log, which is created by MESA_create_runtime_log_handle; + * level: log level, messages with level value smaller the global var + * "runtime_log_level" are ignored; + * module: name of loggin module; + * fmt: format string; + * returns: + * none; + */ +void MESA_handle_runtime_log(void *handle, int level, const char *module, const char *fmt, ...); + +/* + * name: MESA_destroy_runtime_log_handle + * functionality: release runtime log handle memory. + * params: + * handle: runtime log handle which is going to be released; + * returns: + * none; + */ +void MESA_destroy_runtime_log_handle(void *handle); + +#ifdef __cplusplus +} +#endif + +#endif + + diff --git a/src/inc_internal/MESA_htable.h b/src/inc_internal/MESA_htable.h new file mode 100644 index 0000000..ed5a074 --- /dev/null +++ b/src/inc_internal/MESA_htable.h @@ -0,0 +1,275 @@ +#ifndef _MESA_HASH_V3_H_ +#define _MESA_HASH_V3_H_ +#ifdef __cplusplus +extern "C" +{ +#endif + +#include +#include +#include +#include + +/* + * general purpose hash table implementation. + * + * xiang hong + * 2002-07-28 + *History: + * 2012-03-23 zhengchao add thread safe option and link expire feature; + * 2014-01-27 lijia add reentrant feature. + */ + +#define COMPLEX_KEY_SWITCH (1) + +#define MESA_HASH_DEBUG (0) + +#define ELIMINATE_TYPE_NUM (1) +#define ELIMINATE_TYPE_TIME (2) +#define ELIMINATE_TYPE_MANUAL (3) /* delete oldest item by manual */ + +typedef void * MESA_htable_handle; + + +#define HASH_MALLOC(_n_) malloc(_n_) +#define HASH_FREE(_p_) free(_p_) + + +#if 1 +#define HASH_TIME_NOW() time(NULL) +#else +extern volatile time_t g_CurrentTime; /* 此变量在另一个线程中每隔1秒自增一次 */ +#define HASH_TIME_NOW() (time_t)g_CurrentTime +#endif + +#ifndef uchar +#define uchar unsigned char +#endif +#ifndef uint +#define uint unsigned int +#endif + +/* eliminate algorithm */ +#define HASH_ELIMINATE_ALGO_FIFO (0) /* by default */ +#define HASH_ELIMINATE_ALGO_LRU (1) + +/* + * hash key compare function prototype, see hash_key_comp(). + * return value: + * 0:key1 and key2 are equal; + * other:key1 and key2 not equal. + */ +typedef int key_comp_fun_t(const uchar * key1, uint size1, const uchar * key2, uint size2); + +/* + * hash key->index computing function prototype, see hash_key2index(). + */ +typedef uint key2index_fun_t(const MESA_htable_handle table, const uchar * key, uint size); + +typedef long hash_cb_fun_t(void *data, const uchar *key, uint size, void *user_arg); + +/* + * thread_safe: 0:create hash table without thread safe features; + * positive:the bigger number has more performance, less collide, but less timeout accuracy. + * max number is 1024. + * recursive: 0:can't recursive call MESA_htable_xxx series function + * 1:can recursive call MESA_htable_xxx series function. + * hash_slot_size: how big do you want the table to be, must be 2^N; + * max_elem_num: the maximum elements of the HASH-table,0 means infinite; + * key_comp: hash key compare function, use default function if NULL; + * suggest implement by yourself. + * key2index: hash key->index computing function, use default function if NULL; + * suggest use MESA_htable built-in function. + * data_free: release resources function, only free attached data pointer if NULL; + * data_expire_with_condition: if expire_time > 0, call this function when a element expire, eliminate always if NULL; + * args: + * data: pointer to attached data; + * type: eliminate reason, ELIMINATE_TYPE_NUM or ELIMINATE_TYPE_TIME; + * return value of 'data_expire_with_condition': + * 1: can be eliminated; + * 0: can't be eliminated, renew the item. + * eliminate_type: the algorithm of elimanate a expired element, 0:FIFO; 1:LRU. + * expire_time: the element expire time in second, 0 means infinite. + */ +typedef struct{ + unsigned int thread_safe; + int recursive; + unsigned int hash_slot_size; + unsigned int max_elem_num; + int eliminate_type; + int expire_time; + key_comp_fun_t * key_comp; + key2index_fun_t * key2index; + void (* data_free)(void *data); + int (*data_expire_with_condition)(void *data, int type); +#if COMPLEX_KEY_SWITCH + uchar* (*complex_key_dup)(const uchar *key, uint key_size); + void (* complex_key_free)(uchar *key, uint key_size); +#endif +}MESA_htable_create_args_t; + +/* + * name: MESA_htable_create + * functionality: allocats memory for hash slots, and initialize hash structure; + * param: + * args: argments set; + * args_len: length of argment set; + * returns: + * NULL : error; + * Non-NULL : success; + */ +MESA_htable_handle MESA_htable_create(const MESA_htable_create_args_t *args, int args_struct_len); + +/* + * get total number of HASH element. +*/ +unsigned int MESA_htable_get_elem_num(const MESA_htable_handle table); + +/* + * name: MESA_htable_destroy + * functionality: cleans up hash structure, frees memory occupied; + * param: + * table: who is the victim; + * func: callback function to clean up data attached to hash items; + * returns: + * always returns 0; + */ +int MESA_htable_destroy(MESA_htable_handle table, void (* func)(void *)); + +/* + * name: MESA_htable_add + * functionality: adds item to table, call hash_expire() if elem_count gets + * bigger than threshold_hi, and adjust threshold; + * param: + * table: to which table do you want to add; + * key: what is the label; + * size: how long is the label; + * data: what data do you want to attach; + * returns: + * >0 success,return hash elems' linklist size + * -1, duplicates found and can't add this one; + * -2, memory failure; + * -3, other errors. + */ +int MESA_htable_add(MESA_htable_handle table, const uchar * key, uint size, const void *data); +#if 0 +/* + * name: hash_add_with_expire + * functionality: adds item to table, than call hash_expire() on its list + * param: + * table: to which table do you want to add; + * key: what is the label; + * size: how long is the label; + * data: what data do you want to attach; + * returns: + * >0 success,return hash elems' linklist size + * -1, duplicates found and can't add this one; + * -2, memory failure; + */ +int MESA_hash_add_with_expire_v3(MESA_htable_inner_t * table, uchar * key, uint size, void * data); + +#endif + + +/* + * name: MESA_htable_del + * functionality: deletes item from table. + * param: + * table: from which table do you want to delete; + * key : what is the label; + * size : how long is the label; + * func : callback function to clean up data attached to hash items, + if this pointer is NULL will call "data_free" in MESA_hash_create(), + * returns: + * 0, success; + * -1, no such thing; + */ +int MESA_htable_del(MESA_htable_handle table, const uchar * key, uint size, + void (* func)(void *)); + +/* + * name: MESA_htable_del_oldest_manual + * functionality: deletes oldest item from table. + * param: + * table: from which table do you want to delete; + * func : callback function to clean up data attached to hash items, + if this pointer is NULL will call "data_free" in MESA_hash_create(), + * batch_num: delete oldest items. + * returns: + * 0, success; + * -1, no such thing; + */ +int MESA_htable_del_oldest_manual(MESA_htable_handle table, void (* func)(void *), int batch_num); + +/* + * name: MESA_htable_search + * functionality: selects item from table; + * param: + * table: from which table do you want to select; + * key : what is the label; + * size : how long is the label; + * + * return: + * not NULL :pointer to attached data; + * NULL :not found(thus be careful if you are attaching NULL data on purpose). + */ +void *MESA_htable_search(const MESA_htable_handle table, const uchar * key, uint size); + +/* + * name: MESA_htable_search_cb + * functionality: selects item from table, and then call 'cb', reentrant; + * in param: + * table: from which table do you want to select; + * key : what is the label; + * size : how long is the label; + * cb : call this function when found the attached data; + * arg : the argument of "cb" function. + * out param: + * cb_ret: the return value of the function "cb". + * return: + * not NULL :pointer to attached data; + * NULL :not found(thus be careful if you are attaching NULL data on purpose). + */ +void *MESA_htable_search_cb(const MESA_htable_handle table, const uchar * key, uint size, + hash_cb_fun_t *cb, void *arg, long *cb_ret); + +/* + * name: hash_iterate + * functionality: iterates each hash item; + * params: + * table: what table is to be iterated; + * func: what do you want to do to each attached data item; + * returns: + * 0: iterates all items; + * -1: error; + */ +int MESA_htable_iterate(MESA_htable_handle table, void (* func)(const uchar * key, uint size, void * data, void *user), void * user); + +/* + args: + print_switch: + 0: disable print message; + 1: enable print message; +*/ +void MESA_htable_print_crtl(MESA_htable_handle table, int print_switch); + +#if 0 +/* + * name: hash_expire + * functionality: iterates each item and deletes those that are expired; + * params: + * table: what table do you want to check; + * returns: + * always 0; + */ +int MESA_hash_expire(MESA_htable_inner_t * table); +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /* _LIB_HASH_H_INCLUDED_ */ + + diff --git a/src/inc_internal/MESA_list.h b/src/inc_internal/MESA_list.h new file mode 100644 index 0000000..315e695 --- /dev/null +++ b/src/inc_internal/MESA_list.h @@ -0,0 +1,31 @@ +#ifndef _LIST_COMMON_H_ +#define _LIST_COMMON_H_ + +typedef struct MESA_list{ + struct MESA_list *nextele; + struct MESA_list *preele; + void *quiddity; +}MESA_list_t; + +#ifdef __cplusplus +extern "C" +{ +#endif + +void MESA_list_init_head(struct MESA_list *head); +long MESA_list_get_count(const struct MESA_list *head); +int MESA_list_is_empty(const struct MESA_list *head); +void MESA_list_add(struct MESA_list *head, struct MESA_list *new_list); +void MESA_list_add_tail(struct MESA_list *head, struct MESA_list *new_list); +void MESA_list_del(struct MESA_list *head, struct MESA_list *del_list); +void MESA_list_move(struct MESA_list *head, struct MESA_list *list); +void MESA_list_move_tail(struct MESA_list *head, struct MESA_list *list); +struct MESA_list *MESA_list_join_n(struct MESA_list *head, struct MESA_list *op_place, struct MESA_list *new_obj); +struct MESA_list *MESA_list_join_p(struct MESA_list *head, struct MESA_list *new_obj, struct MESA_list *op_place); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/inc_internal/MESA_list_queue.h b/src/inc_internal/MESA_list_queue.h new file mode 100644 index 0000000..88b830d --- /dev/null +++ b/src/inc_internal/MESA_list_queue.h @@ -0,0 +1,114 @@ +#ifndef _MESA_LIST_V3_H_ +#define _MESA_LIST_V3_H_ + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define MESA_LQUEUE_VERSION "MESA_lqueue_2014.03.24_12:00:00" + +/* + MESA_list 第三版， + 1-增加线程安全特性; + 2-隐藏内部结构, 更安全、接口更简洁; + 3-调用者无需自行管理节点结构，使用更方便; +*/ + + +#define MESA_LIST_OP_PLACE_HEAD (0x1) +#define MESA_LIST_OP_PLACE_TAIL (0x2) + +#define MESA_list_GET (0x1) +#define MESA_list_JOIN (0x2) + +#define MESA_list_BOLCK (0x4) +#define MESA_list_NONBOLCK (0x8) + +#define MESA_list_JOIN_BLOCK (MESA_list_JOIN|MESA_list_BOLCK) +#define MESA_list_JOIN_NONBLOCK (MESA_list_JOIN|MESA_list_NONBOLCK) +#define MESA_list_GET_BLOCK (MESA_list_GET|MESA_list_BOLCK) +#define MESA_list_GET_NONBLOCK (MESA_list_GET|MESA_list_NONBOLCK) + +typedef void * MESA_lqueue_head; +typedef int (* MESA_lqueue_cb_t)(void *data, long data_len, void *arg); + +/* All of the following functions return value */ +typedef enum{ + MESA_QUEUE_RET_OK = 0, /* success */ + MESA_QUEUE_RET_COMMON_ERR = -1, /* general、undefined errors */ + MESA_QUEUE_RET_ARG_ERR = -2, /* invalid args */ + MESA_QUEUE_RET_NUM_FULL = -3, /* queue number full */ + MESA_QUEUE_RET_MEM_FULL = -4, /* queue memory full */ + MESA_QUEUE_RET_QEMPTY = -5, /* queue empty */ + MESA_QUEUE_RET_LEN_ERR = -6, /* length error */ + MESA_QUEUE_RET_CANT_GET_LOCK = -7, /* can't get lock in non-block mode */ + MESA_QUEUE_RET_GET_LOCK_TMOUT = -8, /* get lock timeout */ +}MESA_queue_errno_t; + +/* + args description: + [IN] + thread_safe : 1:create thread safe queue; 0:without thread safe insurance. + max_item_num: maximum queue items of the queue, 0 means infinity. +*/ +MESA_lqueue_head MESA_lqueue_create(int thread_safe, long max_item_num); + +/* + attention: + The follow two functions is get some value of queue in a moment, + however, the value you got is not exactly, + because it's maybe changed immediately by other thread when this functions is return. +*/ +long MESA_lqueue_get_mem_used(MESA_lqueue_head head); +long MESA_lqueue_get_count(MESA_lqueue_head head); + + +/* + args description: + [IN]: + lq_head : the handler of MESA_lqueue. + + [OUT]: + data : receive buffer. + + [IN && OUT]: + data_len: + is value-result argument, like "addrlen of recvfrom(2)", + the caller should initialize the size of the 'data', + will modified on return to indicate the actual size of the queue item. + +*/ +int MESA_lqueue_read_head(MESA_lqueue_head lq_head, void *data, long *data_len); +int MESA_lqueue_get_head(MESA_lqueue_head lqhead, void *data, long *data_len); + +/* + if return value of "cb" is 0, the behaviour is like MESA_lqueue_read_head(), + else if return value of "cb" is not 0, the behaviour is like MESA_lqueue_get_head(). +*/ +int MESA_lqueue_detect_get_head(MESA_lqueue_head lq_head, MESA_lqueue_cb_t cb, void *data, long *data_len, void *cb_arg); +int MESA_lqueue_get_tail(MESA_lqueue_head lq_head, void *data, long *data_len); + +int MESA_lqueue_join_head(MESA_lqueue_head lq_head, const void *data, long data_len); +int MESA_lqueue_join_tail(MESA_lqueue_head lq_head, const void *data, long data_len); + + +/* these functions features same with above no "try", + except shall return immediately, in other word is "Non-block mode"! + */ +int MESA_lqueue_try_read_head(MESA_lqueue_head lq_head, void *data, long *data_len); +int MESA_lqueue_try_get_head(MESA_lqueue_head lq_head, void *data, long *data_len); +int MESA_lqueue_try_get_tail(MESA_lqueue_head lq_head, void *data, long *data_len); +int MESA_lqueue_try_join_head(MESA_lqueue_head lq_head, const void *data, long data_len); +int MESA_lqueue_try_join_tail(MESA_lqueue_head lq_head, const void *data, long data_len); + + +void MESA_lqueue_destroy(MESA_lqueue_head head, MESA_lqueue_cb_t cb, void *cb_arg); + + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/inc_internal/rulescan.h b/src/inc_internal/rulescan.h new file mode 100644 index 0000000..9230e64 --- /dev/null +++ b/src/inc_internal/rulescan.h @@ -0,0 +1,285 @@ +/* + * + * Copyright (c) 2014 + * String Algorithms Research Group + * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) + * National Engineering Laboratory for Information Security Technologies (NELIST) + * All rights reserved + * + * Written by: LIU YANBING (liuyanbing@iie.ac.cn) + * Last modification: 2015-02-03 + * + * This code is the exclusive and proprietary property of IIE-CAS and NELIST. + * Usage for direct or indirect commercial advantage is not allowed without + * written permission from the authors. + * + */ + +#ifndef H_RULE_SCAN_H +#define H_RULE_SCAN_H + +#ifdef __cplusplus +extern "C" +{ +#endif + + #define MAX_EXPR_ITEM_NUM (1U<<3) /* 每条与表达式最多由MAX_EXPR_ITEM_NUM个规则组成 */ + #define MAX_MATCH_POS_NUM 1024 /* 每条规则最多允许返回的命中位置的个数 */ + + /* 定义不同的规则类型 */ + const unsigned int RULETYPE_STR = 0; /* 字符串或二进制规则 */ + const unsigned int RULETYPE_REG = 1; /* 正则表达式规则 */ + const unsigned int RULETYPE_INT = 2; /* 数值区间规则 */ + const unsigned int RULETYPE_IPv4 = 3; /* IPv4规则 */ + const unsigned int RULETYPE_IPv6 = 4; /* IPv6规则 */ + + const unsigned int MAX_RULETYPE = 5; /* 规则类型数量 */ + const unsigned int MAX_SUB_RULETYPE = 4096; /* 规则子类型数量 */ + + /* 字符串类型规则（可表示文本字符串、二进制字符串、正则表达式） */ + typedef struct _string_rule_t + { + char * str; /* 字符串内容；如果是正则表达式，须以'\0'结束，可不指定长度 */ + unsigned int len; /* 字符串长度 */ + unsigned char case_sensitive; /* 是否大小写敏感匹配（1：敏感；0：不敏感） */ + unsigned char match_mode; /* 匹配模式：子串匹配（0），完整匹配（1）；仅对精确串匹配有效 */ + int l_offset; /* 表示模式串只能在文本范围[l_offset, r_offset]中出现，-1表示无限制,-2表示左匹配；仅对精确串匹配有效 */ + int r_offset; /* 表示模式串只能在文本范围[l_offset, r_offset]中出现，-1表示无限制,-2表示右匹配；仅对精确串匹配有效 */ + }string_rule_t; + + /* 整数数值区间规则，表示整数区间[lb, ub] */ + typedef struct _interval_rule_t + { + unsigned int lb; /* 数据区间的下界（包含lb），无限制默认为0 */ + unsigned int ub; /* 数据区间的下界（包含ub），无限制默认为0 */ + }interval_rule_t; + + /* 带掩码的IPv4规则 */ + typedef struct _ipv4_rule_t + { + unsigned int saddr; /* 源IP地址；0表示忽略本字段 */ + unsigned int smask; /* 源IP地址掩码；0表示固定IP=saddr */ + unsigned int daddr; /* 目的IP地址；0表示忽略本字段 */ + unsigned int dmask; /* 目的IP地址掩码；0表示固定IP=daddr */ + unsigned short int min_sport; /* 源端口范围下界；0表示忽略本字段 */ + unsigned short int max_sport; /* 源端口范围上界；0表示固定端口=min_sport */ + unsigned short int min_dport; /* 目的端口范围下界；0表示忽略本字段 */ + unsigned short int max_dport; /* 目的端口范围上界；0表示固定端口=min_dport */ + unsigned short int proto; /* 传输层协议，6表示TCP，17表示UDP；0表示忽略本字段 */ + unsigned short int direction; /* 方向，0表示双向，1表示单向 */ + }ipv4_rule_t; + + /* 带掩码的IPv6规则 */ + typedef struct _ipv6_rule_t + { + unsigned int saddr[4]; /* 源IP地址；0表示忽略本字段 */ + unsigned char smask_bits;/* 源IP地址掩码位数；0表示固定IP=saddr */ + unsigned int daddr[4]; /* 目的IP地址；0表示忽略本字段 */ + unsigned char dmask_bits;/* 目的IP地址掩码位数；0表示固定IP=daddr */ + unsigned short int min_sport; /* 源端口范围下界；0表示忽略本字段 */ + unsigned short int max_sport; /* 源端口范围上界；0表示固定端口=min_sport */ + unsigned short int min_dport; /* 目的端口范围下界；0表示忽略本字段 */ + unsigned short int max_dport; /* 目的端口范围上界；0表示固定端口=min_dport */ + unsigned short int proto; /* 传输层协议，6表示TCP，17表示UDP，无限制默认为0 */ + unsigned short int direction; /* 方向，0表示双向，1表示单向 */ + }ipv6_rule_t; + + /* 通用的规则类型 */ + typedef struct _scan_rule_t + { + unsigned int rule_type; /* 规则类型，必须为上述枚举规则类型之一 */ + unsigned int sub_type; /* 子类类型，用户自定义，但子类型个数不允许超过MAX_SUB_RULETYPE（见前文定义） */ + union /* 根据rule_type决定规则是字符串、数值区间、还是IP规则 */ + { + string_rule_t string_rule; /* 字符串规则（字符串、二进制、正则表达式） */ + interval_rule_t interval_rule; /* 整数数值区间规则 */ + ipv4_rule_t ipv4_rule; /* 带掩码的IPv4规则 */ + ipv6_rule_t ipv6_rule; /* 带掩码的IPv6规则 */ + }; + }scan_rule_t; + + /* 一条与表达式规则 */ + typedef struct _boolean_expr_t + { + unsigned int expr_id; /* 与表达式的ID */ + unsigned int operation; /* 对与表达式执行的操作：0表示增加，1表示删除 */ + unsigned int rnum; /* 该与表达式包含多少个项；如果operation=1，置rnum=0即可 */ + scan_rule_t * rules; /* 组成与表达式的项；如果operation=1，置rules=NULL即可 */ + void * tag; /* 用户自定义数据，命中时随匹配结果返回 */ + }boolean_expr_t; + + + /* 待扫描的文本数据类型 */ + typedef struct _text_data_t + { + const char * text; /* 文本数据内容 */ + unsigned int tlen; /* 文本数据长度 */ + int toffset;/* 本段文本数据在整个流数据中的偏移量，流式扫描情况下有效，由用户传入，其它情况置为0(这个必须置为0) */ + }text_data_t; + + /* 待扫描的IPv4元组 */ + typedef struct _ipv4_data_t + { + unsigned int saddr; /* 源IP地址 */ + unsigned int daddr; /* 目的IP地址 */ + unsigned short int sport; /* 源端口 */ + unsigned short int dport; /* 目的端口 */ + unsigned short int proto; /* 传输层协议，6表示TCP，17表示UDP */ + }ipv4_data_t; + + /* 待扫描的IPv6元组 */ + typedef struct _ipv6_data_t + { + unsigned int saddr[4]; /* 源IP地址 */ + unsigned int daddr[4]; /* 目的IP地址 */ + unsigned short int sport; /* 源端口 */ + unsigned short int dport; /* 目的端口 */ + unsigned short int proto; /* 传输层协议，6表示TCP，17表示UDP */ + }ipv6_data_t; + + /* 通用的待扫描数据类型 */ + typedef struct _scan_data_t + { + unsigned int rule_type; /* 规则类型，必须为上述枚举规则类型之一 */ + unsigned int sub_type; /* 子类类型，用户自定义，但子类型个数不允许超过MAX_SUB_RULETYPE（见前文定义） */ + union /* 根据rule_type决定数据负载是字符串、数值、还是IP元组 */ + { + text_data_t text_data; /* 待扫描文本数据（可匹配字符串、二进制、正则表达式） */ + unsigned int int_data; /* 整数数值（可匹配数值区间） */ + ipv4_data_t ipv4_data; /* 待扫描的IPv4元组 */ + ipv6_data_t ipv6_data; /* 待扫描的IPv6元组 */ + }; + }scan_data_t; + + /* + 扫描结果类型scan_result_t和rule_result_t说明： + 1、对于命中的每个布尔表达式，一条表达式对应一条scan_result_t结果，该布尔表达式包含rnum个规则，每个规则均对应于一个结果scan_result_t::result[k]（0<=k +#include +#include +#include +#include +#include +#include +#include + +#define USE_MESA_STREAM_HASH (1) /* 使用基于隧道模式新的HASH表 */ + +extern int MESA_PLATFORM_VERSION_20140917; /* 平台版本号 */ + +#define ADDR_UNSYMMETRY (0) /* 2014-04-25, 针对从XinJing捕获的包, 临时解决VLAN地址不对称问题, 此宏不应长期存在 */ + +#if 0 +#define FOR_863 (0) /* for 863项目用户标签模块 */ +#define FOR_108 (0) +#define RAW_IP_FRAG_PKT (0) /* 2013-08-21 LiJia add, 保存全部原始IP分片包 */ +#endif + +#ifndef UINT8 +typedef unsigned char UINT8; +#endif +#ifndef UCHAR +typedef unsigned char UCHAR; +#endif +#ifndef UINT16 +typedef unsigned short UINT16; +#endif +#ifndef UINT32 +typedef unsigned int UINT32; +#endif +#ifndef UINT64 +typedef unsigned long long UINT64; +#endif + +//流的方向定义 +#define DIR_C2S 0x01 +#define DIR_S2C 0x02 +#define DIR_DOUBLE 0x03 + +//单包的方向定义 +#define DIR_ROUTE_UP 0x00 +#define DIR_ROUTE_DOWN 0x01 + +//地址类型定义 +enum addr_type_t{ + __ADDR_TYPE_INIT = 0, + ADDR_TYPE_IPV4, /* 1, 基于IPv4地址的四元组信息 */ + ADDR_TYPE_IPV6, /* 2, 基于IPv6地址的四元组信息 */ + ADDR_TYPE_VLAN, /* 3 */ + ADDR_TYPE_MAC, /* 4 */ + ADDR_TYPE_ARP = 5, /* 5 */ + ADDR_TYPE_GRE, /* 6 */ + ADDR_TYPE_MPLS, /* 7 */ + ADDR_TYPE_PPPOE_SES, /* 8 */ + ADDR_TYPE_TCP, /* 9 */ + ADDR_TYPE_UDP = 10, /* 10 */ + ADDR_TYPE_L2TP, /* 11 */ + //ADDR_TYPE_STREAM_TUPLE4_V4, /* 12, 混合地址类型, 基于IPv4地址的四元组信息 */ + //ADDR_TYPE_STREAM_TUPLE4_V6, /* 13, 混合地址类型, 基于IPv6地址的四元组信息 */ + __ADDR_TYPE_IP_PAIR_V4, /* 14, 纯IPv4地址对 */ + __ADDR_TYPE_IP_PAIR_V6, /* 15, 纯IPv6地址对 */ + __ADDR_TYPE_MAX, /* 16 */ +}; + + +#define TCP_TAKEOVER_STATE_FLAG_OFF 0 +#define TCP_TAKEOVER_STATE_FLAG_ON 1 + +//应用层看到的链接状态定义 +#define OP_STATE_PENDING 0 +#define OP_STATE_REMOVE_ME 1 +#define OP_STATE_CLOSE 2 +#define OP_STATE_DATA 3 + +//应用层返回结果定义 +#define APP_STATE_GIVEME 0x00 +#define APP_STATE_DROPME 0x01 +#define APP_STATE_FAWPKT 0x00 +#define APP_STATE_DROPPKT 0x10 + +//流的类型定义 +enum stream_type_t{ + STREAM_TYPE_NON = 0, /* 无流的概念, 如VLAN, IP层等 */ + STREAM_TYPE_TCP, + STREAM_TYPE_UDP, + STREAM_TYPE_SOCKS4, + STREAM_TYPE_SOCKS5, + STREAM_TYPE_HTTP_PROXY, + STREAM_TYPE_PPPOE, +}; + +#define PROXY_STATE_SEL 0 +#define PROXY_STATE_LINK_IN 1 + + +/* 原始包结构 */ +typedef struct { + enum addr_type_t low_layer_type; /* 原始包最底层协议的类型, 可能是MAC(pcap捕包), 也可能是IPv4(pag捕包) */ + int raw_pkt_len; /* 原始包总长度 */ + const void *raw_pkt_data; /* 原始包头指针, 根据low_layer_type判断协议类型 */ + struct timeval raw_pkt_ts; /* 原始包捕获时间戳, 如果全为0则不支持此功能(如pag模式) */ +}raw_pkt_t; + +// 代理信息 +struct proxy_node +{ + UINT16 iType; // 代理类型, 0 表示无效 + UINT16 uiPort; // 代理服务器端口 + UINT16 uiUserLen; + UINT16 uiPwdLen; + UINT16 uiApendLen; + + UCHAR opstate; //当前链接所处状态 + UCHAR dealstate; //代理处理状态 + UINT32 uiIP; // 代理服务器IP地址v4, 按网络字节序 + UCHAR *pIpv6; // 代理服务器IP地址, v6地址 + UCHAR *pUser; // 代理用户名 + UCHAR *pPwd; // 代理密码 + UCHAR *append; // 其它附属信息，比如url + + void *apme; // 其它附属信息，比如url + struct proxy_node *pnext; +} ; + + +typedef struct raw_ipfrag_list{ + void *frag_packet; + int pkt_len; + int type; /* IPv4 or IPv6 */ + struct raw_ipfrag_list *next; +}raw_ipfrag_list_t; + + +struct buf_unorder +{ + struct buf_unorder *next; + struct buf_unorder *prev; + void *data; + UINT32 len; + UINT32 tcpdatalen; + UINT32 urg_ptr; + char fin; + char urg; + char rst; + unsigned char pad; + UINT32 seq; + UINT32 ack; + raw_ipfrag_list_t *ipfrag_list; + raw_pkt_t raw_pkt; /* 乱序包存储原始包 */ +}; + +/*半流结构体定义：*/ +struct half_tcpstream +{ + UCHAR *data; + UINT32 offset; /*data中第一个字节在TCP数据流中的偏移量*/ + UINT32 count; /*从连接建立起到现在为止，到达的数据总长度字节数*/ + UINT32 count_new; /*本次新到的数据字节数*/ + UINT32 count_ideal; /*从连接建立起到现在为止，理论上应该到达的数据总长度*/ + UINT32 pktcout; /*本侧累计到达的包个数*/ + UINT32 totallost; /*本侧累计丢包长度*/ + UINT32 seq; /*本侧数据期待的seq序号*/ + UINT32 first_data_seq; /*本侧数据起始的的seq序号*/ + UINT32 ack_seq; /*本侧数据最后使用的应答号*/ + UINT16 window; /*本侧数据滑动窗口大小*/ + UCHAR maxunorder; /*最大乱序数目*/ + UCHAR finstate; /*fin状态*/ + UINT32 pktcout_new; /*本次新到达的包个数*/ + UINT32 unorder_cnt; + struct buf_unorder *unorderlist; /*乱序包的链表*/ + struct buf_unorder *unorderlisttail; /*乱序包的链表尾部指针*/ + +}; + +/* 兼容papp */ +struct tuple4 { + u_int saddr; + u_int daddr; + u_short source; + u_short dest; +}; + +struct tuple6 +{ + UCHAR saddr[16] ; + UCHAR daddr[16] ; + UINT16 source; + UINT16 dest; +}; + +/* network-order */ +struct stream_tuple4_v4{ + UINT32 saddr; /* network order */ + UINT32 daddr; /* network order */ + UINT16 source; /* network order */ + UINT16 dest; /* network order */ +}; + + +#ifndef IPV6_ADDR_LEN +#define IPV6_ADDR_LEN (sizeof(struct in6_addr)) +#endif + +struct stream_tuple4_v6 +{ + UCHAR saddr[IPV6_ADDR_LEN] ; + UCHAR daddr[IPV6_ADDR_LEN] ; + UINT16 source; /* network order */ + UINT16 dest; /* network order */ +}; + + + +#define GRE_TAG_LEN (4) +struct layer_addr_gre +{ + UINT16 gre_id; +}; + + +#define VLAN_ID_MASK (0x0FFF) +#define VLAN_TAG_LEN (4) +struct layer_addr_vlan +{ + UINT16 vlan_id; /* network order */ +}; + +struct layer_addr_pppoe_session +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + unsigned int ver:4; + unsigned int type:4; +#endif +#if __BYTE_ORDER == __BIG_ENDIAN + unsigned int type:4; + unsigned int ver:4; +#endif + unsigned char code; + unsigned short session_id; +}; + +#ifndef MAC_ADDR_LEN +#define MAC_ADDR_LEN (6) +#endif + +struct layer_addr_mac +{ + UCHAR src_mac[MAC_ADDR_LEN]; /* network order */ + UCHAR dst_mac[MAC_ADDR_LEN]; /* network order */ +}; + +struct layer_addr_ipv4 +{ + UINT32 saddr; /* network order */ + UINT32 daddr; /* network order */ + /* 2014-04-21 lijia add, + 为了空间、易用性、和效率, 不强制按协议层次处理, + IP层存储完整的四元组信息, TCP层只需指针指向此块内存, + 插件获取四元组时, 不再需要get_tuple4()函数. + 对于隧道外层IP, 端口信息为0; + */ + UINT16 source; /* network order */ + UINT16 dest; /* network order */ +}; + +struct layer_addr_ipv6 +{ + UCHAR saddr[IPV6_ADDR_LEN] ; /* network order */ + UCHAR daddr[IPV6_ADDR_LEN] ; /* network order */ + /* 2014-04-21 lijia add, + 为了空间、易用性、和效率, 不强制按协议层次处理, + IP层存储完整的四元组信息, TCP层只需指针指向此块内存, + 插件获取四元组时, 不再需要get_tuple4()函数. + 对于隧道外层IP, 端口信息为0; + */ + UINT16 source;/* network order */ + UINT16 dest;/* network order */ +}; + +struct layer_addr_tcp +{ + UINT16 source; /* network order */ + UINT16 dest; /* network order */ +}; + +struct layer_addr_udp +{ + UINT16 source; /* network order */ + UINT16 dest; /* network order */ +}; + +struct layer_addr_l2tp +{ + UINT32 tunnelid; /* network order */ + UINT32 sessionid; /* network order */ +}; + +struct layer_addr +{ + UCHAR addrtype; /* 地址类型, 详见 enum addr_type_t */ + /* 为了方便应用插件取地址, 此处使用联合体, 省去指针类型强制转换步骤 */ + union + { + struct stream_tuple4_v4 *tuple4_v4; + struct stream_tuple4_v6 *tuple4_v6; + struct layer_addr_ipv4 *ipv4; + struct layer_addr_ipv6 *ipv6; + struct layer_addr_vlan *vlan; + struct layer_addr_mac *mac; + struct layer_addr_gre *gre; + struct layer_addr_tcp *tcp; + struct layer_addr_udp *udp; + struct layer_addr_pppoe_session *pppoe_ses; + struct layer_addr_l2tp *l2tp; + void *paddr; + }; + UCHAR addrlen; /* 地址结构长度 */ +}; + +/* 保留此结构用于和papp兼容, 用作指针时, 可与struct layer_addr强转. */ +struct ipaddr +{ + UCHAR addrtype; + union + { + struct stream_tuple4_v4 *v4; + struct stream_tuple4_v6 *v6; + }paddr; + // struct tuple4 *paddr; +}; + + +/* to do: + 变量地址对齐问题; + 变量隐藏, 内部变量对外不可见, 将stream.h拆分, 外部插件看到的是stream.h的部分只读视图; + */ +struct streaminfo +{ + struct layer_addr addr; //本层协议地址信息 + struct streaminfo *pfather;//上层流结构体 + UCHAR type; // 链接类型 + UCHAR threadnum; // 所属线程 + UCHAR dir:2; /* 流的生存期内有效, 流的单、双方向情况0x01:c-->s; 0x02:s-->c; 0x03 c<-->s; */ + UCHAR curdir:2; /* 单包有效, 当前来包上层流的逻辑方向, 0x01:c-->s; 0x02:s-->c */ + UCHAR layer_dir:2; /* 单包有效, 当前层的地址是否和默认规则"大端口是客户端"相同 */ + UCHAR stream_dir:2; /* 流的生存期内有效, 流的存储的地址是否和默认规则"大端口是客户端"相同 */ + UCHAR dirreverse; /* 建立连接时是否进行了ip地址反转, 即与"大端口是客户端"规则相反 */ + UCHAR opstate; //当前链接所处状态 + UCHAR pktstate; //链接的包序列 + UCHAR routedir; /* 物理包方向, 单包有效, 纯人工指定, 仅用于发包时标记是否与来包方向是否相同, 别无他意, 0:上行; 1:下行 */ + UCHAR addr_use_as_hash; /* 本层的addr是否做为HASH计算和比较的参数, 如:MAC地址不参与计算 */ + + UCHAR stream_killed_flag; /* 2014-08-22 lijia add, 串联模式下, 已经被插件Kill, 之后此流可直接Drop或Kill, 无需再给上层插件 */ + UCHAR __pad__[7]; //整体8字节对齐 + void *pproject; //每个工程可以自定义使用； + void *pdetail; //流的详细信息 + const void *p_layer_header; /* 指向本层包头的指针, 用于构造包时获取相关参数 */ + +#if 0 //FOR_108, ->pproject + unsigned long stream_id; +#endif + +#if 0 //RAW_IP_FRAG_PKT, ->pproject + raw_ipfrag_list_t *ipfrag_list; /* 2013-08-21 LiJia add, 原始分片包链表 */ +#endif + +#if 0 //FOR_863, ->pproject + char terminal_tag[40]; /* 用户标签 */ +#endif + }; + + +struct tcpdetail +{ + + void *pdata; //数据 + UINT32 datalen; //数据长度 + UINT32 lostlen; + UCHAR multisynflag:4; // multi syn + UCHAR needackflag:2; //需要上传ack报文 + UCHAR takeoverflag:2; + UCHAR tcpstateflag; // 用于记录tcp的会话SYN相关状态 + UCHAR link_state; // 链接的状态 + UCHAR creat_mod; + UINT32 regionid; //地域信息 + + struct half_tcpstream *pclient; //到client的TCP连接信息 + struct half_tcpstream *pserver; //到 server的TCP连接信息 + UINT32 serverpkt; + UINT32 clientpkt; + UINT32 servercount; + UINT32 clientcount; + UINT64 creattime; + UINT64 lastmtime; + + UINT32 iserverseq; //链接建立时临时存储seq + UINT32 iclientseq; //链接建立时临时存储seq + void *apme; //应用层上下文 + void *pAllpktpme; //无状态的tcp管理上下文 + struct proxy_node *pproxylist; //代理的链表,具体代理解析模块负责申请和释放 + +}; +struct udpdetail +{ + void *pdata; //数据 + UINT32 datalen; //数据长度 + UINT32 regionid; //地域信息 + UINT32 serverpkt; + UINT32 clientpkt; + UINT32 servercount; + UINT32 clientcount; + UINT64 creattime; + UINT64 lastmtime; + void *apme; //应用层上下文 +}; + + +//业务层调用解析层时session_state状态 +#define SESSION_STATE_PENDING 0x01 +#define SESSION_STATE_DATA 0x02 +#define SESSION_STATE_CLOSE 0x04 + +//解析层调用业务层时的返回值； +#define PROT_STATE_GIVEME 0x01 +#define PROT_STATE_DROPME 0x02 +#define PROT_STATE_KILLME 0x04 + +//解析层插件调用业务层插件时传入参数 +typedef struct _plugin_session_info +{ + unsigned short plugid; //plugid，平台分配 + char session_state; //会话状态，PENDING,DATA,CLOSE + char _pad_; //补齐 + int buflen; //当前字段长度 + long long prot_flag; //当前字段的flag值 + void *buf; //当前字段 + void* app_info; //解析层上下文信息 +}stSessionInfo; +#ifdef __cplusplus +extern "C" { +#endif + + +/* + args: + [IN]: + stream:流结构体指针; + [OUT] + addr_type:四元组类型, ADDR_TYPE_IPV4 or ADDR_TYPE_IPV6; + + return value: + NULL : error; + NON-NULL : 指向四元组结构体的指针; +*/ +struct layer_addr *get_stream_tuple4(struct streaminfo *this_stream); +int get_thread_count(void); + +typedef char (*STREAM_CB_FUN_T)(struct streaminfo *pstream,void **pme, int thread_seq,const void *raw_pkt); +typedef char (*IPv4_CB_FUN_T)(struct streaminfo *pstream,unsigned char routedir,int thread_seq, const void *raw_pkt); +typedef char (*IPv6_CB_FUN_T)(struct streaminfo *pstream,unsigned char routedir,int thread_seq, const void *raw_pkt); + +/*参数描述： + a_*： 本流上下文信息; + f_*: 本包所对应的父流信息; + raw_pkt: 原始包指针, 实际类型为'raw_pkt_t'; + pme: 私有数据指针，将来扩展用，暂时为NULL; + thread_seq：线程序号; + +函数返回值描述：为下面四个值的运算 + + APP_STATE_GIVEME：继续向本函数送包。 + APP_STATE_DROPME：不再向本函数送包。 + APP_STATE_FAWPKT：回注该数据包 + APP_STATE_DROPPKT：不回注该数据包 +*/ +char IPv4_ENTRY_EXAMPLE( struct streaminfo *f_stream,unsigned char routedir,int thread_seq, const void *raw_pkt); +char IPv6_ENTRY_EXAMPLE( struct streaminfo *f_stream,unsigned char routedir,int thread_seq,const void *raw_pkt); +char TCP_ENTRY_EXAMPLE(struct streaminfo *a_tcp, void **pme, int thread_seq,const void *raw_pkt); +char UDP_ENTRY_EXAMPLE(struct streaminfo *a_udp, void **pme, int thread_seq,const void *raw_pkt); + +int stream_register_tcp_allpkt (STREAM_CB_FUN_T fun); +int stream_register_tcp_takeover (STREAM_CB_FUN_T fun); + +int stream_register_tcp (STREAM_CB_FUN_T fun); +int stream_register_udp (STREAM_CB_FUN_T fun); +int stream_register_ip (IPv4_CB_FUN_T fun); +int stream_register_ipv6 (IPv6_CB_FUN_T fun); + + +void *dictator_malloc(int thread_seq,size_t size); +void dictator_free(int thread_seq,void *pbuf); +void *dictator_realloc(int thread_seq, void* pbuf, size_t size); + +char PROT_PROCESS(stSessionInfo* session_info, void **pme, int thread_seq,struct streaminfo *a_stream,void *a_packet); + +/* 用于应用层插件获取本流中对应的用户标签 */ +//int terminal_tag_probe(struct streaminfo *stream, struct mesa_tcp_hdr *this_tcphdr,void *rawippkt); +const unsigned char *get_terminal_tag(struct streaminfo *stream); +//int MESA_kill_tcp(struct streaminfo *stream, struct ip *a_packet); +int MESA_kill_tcp(struct streaminfo *stream, const void *raw_pkt); +int MESA_kill_tcp_synack(struct streaminfo *stream, const void *raw_pkt); + +/* + ARG: + stream: 流结构体指针; + payload: 要发送的数据指针; + payload_len: 要发送的数据负载长度; + raw_pkt: 原始包指针; + snd_routedir: 要发送数据的方向, 原始包方向为:stream->routedir , + 如果与原始包同向, snd_dir = stream->routedir, + 如果与原始包反向, snd_dir = stream->routedir ^ 3(即1<--->2, 2<--->1的数学转换); + return value: + -1: error. + >0: 发送的数据包实际总长度(payload_len + 底层包头长度); +*/ +int MESA_inject_pkt(struct streaminfo *stream, const char *payload, int payload_len, const void *raw_pkt, UCHAR snd_routedir); + +/* 2014-07-31 LiJia add, for set one stream unorder number */ +int tcp_set_single_stream_max_unorder(struct streaminfo *stream, UCHAR dir, UCHAR unorder_num); + +raw_ipfrag_list_t *get_raw_frag_list(struct streaminfo *stream); +//void frags_list_free_one(raw_ipfrag_list_t *frags_list, int); +//raw_ipfrag_list_t *frags_list_merge(raw_ipfrag_list_t *new_merge_list, raw_ipfrag_list_t *old_list); +//raw_ipfrag_list_t *raw_ip_frag_list_attach(int thread_num); +//void raw_ip_frag_list_detach(int thread_num); + + +unsigned long get_stream_id(struct streaminfo *stream); + +void MESA_stream_list_free(struct streaminfo *heap_stream); +struct streaminfo *MESA_stream_list_dup(struct streaminfo *stack_stream, int reverse); +int MESA_stream_list_cmp(struct streaminfo *stream1, struct streaminfo *stream2); + +#ifdef __cplusplus +} +#endif + + + +#endif + diff --git a/src/inc_internal/stream_old.h b/src/inc_internal/stream_old.h new file mode 100644 index 0000000..682462d --- /dev/null +++ b/src/inc_internal/stream_old.h @@ -0,0 +1,336 @@ +#ifndef _APP_STREAM_H_ +#define _APP_STREAM_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define FOR_863 (0) /* for 863项目用户标签模块 */ +#define FOR_108 (0) +#define RAW_IP_FRAG_PKT (0) /* 2013-08-21 LiJia add, 保存全部原始IP分片包 */ + +typedef unsigned char UINT8; +typedef unsigned long long UINT64; +typedef unsigned int UINT32; +typedef unsigned short UINT16; +typedef unsigned char UCHAR; + +//流的方向定义 +#define DIR_C2S 0x01 +#define DIR_S2C 0x02 +#define DIR_DOUBLE 0x03 + +//单包的方向定义 +#define DIR_ROUTE_UP 0x00 +#define DIR_ROUTE_DOWN 0x01 + +//地址类型定义 +#define ADDR_TYPE_IPV4 1 +#define ADDR_TYPE_IPV6 2 +#define ADDR_TYPE_VLAN 3 + +#define TCP_TAKEOVER_STATE_FLAG_OFF 0 +#define TCP_TAKEOVER_STATE_FLAG_ON 1 + +//应用层看到的链接状态定义 +#define OP_STATE_PENDING 0 +#define OP_STATE_REMOVE_ME 1 +#define OP_STATE_CLOSE 2 +#define OP_STATE_DATA 3 + +//应用层返回结果定义 +#define APP_STATE_GIVEME 0x00 +#define APP_STATE_DROPME 0x01 +#define APP_STATE_FAWPKT 0x00 +#define APP_STATE_DROPPKT 0x10 + +//流的类型定义 +#define STREAM_TYPE_TCP 1 +#define STREAM_TYPE_UDP 2 +#define STREAM_TYPE_VLAN 3 +#define STREAM_TYPE_SOCKS4 4 +#define STREAM_TYPE_SOCKS5 5 +#define STREAM_TYPE_HTTP_PROXY 6 + +#define STREAM_TYPE_MPLS 7 + +#define PROXY_STATE_SEL 0 +#define PROXY_STATE_LINK_IN 1 + + +// 代理信息 +struct proxy_node +{ + UINT16 iType; // 代理类型, 0 表示无效 + UINT16 uiPort; // 代理的真实服务器端口 + UINT16 uiUserLen; + UINT16 uiPwdLen; + UINT16 uiApendLen; + + UCHAR opstate; //当前链接所处状态 + UCHAR dealstate; //代理处理状态 + UINT32 uiIP; // 代理的真实服务器IP地址v4, 按网络字节序 + UCHAR *pIpv6; // 代理的真实服务器IP地址, v6地址 + UCHAR *pUser; // 代理用户名 + UCHAR *pPwd; // 代理密码 + UCHAR *append; // 其它附属信息，比如url + + void *apme; // 其它附属信息，比如url + struct proxy_node *pnext; +} ; + + +#if RAW_IP_FRAG_PKT +typedef struct raw_ipfrag_list{ + void *frag_packet; + int pkt_len; + int type; /* IPv4 or IPv6 */ + struct raw_ipfrag_list *next; +}raw_ipfrag_list_t; +#endif + +struct buf_unorder +{ + struct buf_unorder *next; + struct buf_unorder *prev; + void *data; + UINT32 len; + UINT32 tcpdatalen; + UINT32 urg_ptr; + char fin; + char urg; + char rst; + unsigned char pad; + UINT32 seq; + UINT32 ack; +#if RAW_IP_FRAG_PKT + raw_ipfrag_list_t *ipfrag_list; +#endif +}; +/*半流结构体定义：*/ +struct half_tcpstream +{ + UCHAR *data; + UINT32 offset; /*data中第一个字节在TCP数据流中的偏移量*/ + UINT32 count; /*从连接建立起到现在为止，到达的数据总长度字节数*/ + UINT32 count_new; /*本次新到的数据字节数*/ + UINT32 count_ideal; /*从连接建立起到现在为止，理论上应该到达的数据总长度*/ + UINT32 pktcout; /*本侧累计到达的包个数*/ + UINT32 totallost; /*本侧累计丢包长度*/ + UINT32 seq; /*本侧数据期待的seq序号*/ + UINT32 first_data_seq; /*本侧数据起始的的seq序号*/ + UINT32 ack_seq; /*本侧数据最后使用的应答号*/ + UINT16 window; /*本侧数据滑动窗口大小*/ + UCHAR maxunorder; /*最大乱序数目*/ + UCHAR finstate; /*fin状态*/ + UINT32 pktcout_new; /*本次新到达的包个数*/ + UINT32 unorder_cnt; + struct buf_unorder *unorderlist; /*乱序包的链表*/ + struct buf_unorder *unorderlisttail; /*乱序包的链表尾部指针*/ + +}; + +#ifndef STRUCT_TUPLE4_DEFINED +#define STRUCT_TUPLE4_DEFINED (1) +struct tuple4 { + UINT32 saddr; + UINT32 daddr; + UINT16 source; + UINT16 dest; +}; +#endif + +struct tuple6 +{ + UCHAR saddr[16] ; + UCHAR daddr[16] ; + UINT16 source; + UINT16 dest; +}; + +#define VLAN_ID_LEN 4 +struct tuplevlan +{ + UCHAR vlan_id[VLAN_ID_LEN]; +}; + +struct tupell2tp +{ + UINT32 tunnelid; + UINT32 sessionid; +}; +struct ipaddr +{ + UCHAR addrtype; //地址类型 ipv6 or ipv4 + union + { + struct tuple4 *v4; + struct tuple6 *v6; + }paddr; +// struct tuple4 *paddr; +}; + +struct streaminfo +{ + struct ipaddr addr; //ip端口信息 + struct streaminfo *pfather;//上层流结构体 + UCHAR type; // 链接类型 + UCHAR threadnum; // 所属线程 + UCHAR dir:2; // 流的方向情况0x01 c-->s 0x02 s-->c 0x03 c<-->s + UCHAR curdir:2; // 当前来包0x01 c-->s 0x02 s-->c + UCHAR dirreverse:4; //建立连接时是否进行了ip地址反转 + UCHAR opstate; //当前链接所处状态 + UCHAR pktstate; //链接的包序列 + UCHAR routedir; //最高位为1表示上行，为0表示下行; 在串联模式下,低7位表示特殊设备ID + UCHAR __pad__[2]; //整体8字节对齐 + void *pproject; //每个工程可以自定义使用； + void *pdetail; //流的详细信息 +#if FOR_108 + unsigned long stream_id; +#endif +#if RAW_IP_FRAG_PKT + raw_ipfrag_list_t *ipfrag_list; /* 2013-08-21 LiJia add, 原始分片包链表 */ +#endif + +#if FOR_863 + char terminal_tag[40]; /* 用户标签 */ +#endif + }; + + +struct tcpdetail +{ + + void *pdata; //数据 + UINT32 datalen; //数据长度 + UINT32 lostlen; //丢包数据长度 + + UCHAR multisynflag:4; // multi syn + UCHAR needackflag:2; //需要上传ack报文 + UCHAR takeoverflag:2; + UCHAR tcpstateflag; // 用于记录tcp的会话SYN相关状态 + UCHAR link_state; // 链接的状态 + UCHAR pad[5]; + + struct half_tcpstream *pclient; //到client的TCP连接信息 + struct half_tcpstream *pserver; //到 server的TCP连接信息 + UINT32 serverpkt; + UINT32 clientpkt; + UINT32 servercount; + UINT32 clientcount; + UINT64 creattime; + UINT64 lastmtime; + + UINT32 iserverseq; //链接建立时临时存储seq + UINT32 iclientseq; //链接建立时临时存储seq + void *apme; //应用层上下文 + void *pAllpktpme; //无状态的tcp管理上下文 + struct proxy_node *pproxylist; //代理的链表,具体代理解析模块负责申请和释放 + +}; +struct udpdetail +{ + void *pdata; //数据 + UINT32 datalen; //数据长度 + UINT32 pad; + UINT32 serverpkt; + UINT32 clientpkt; + UINT32 servercount; + UINT32 clientcount; + UINT64 creattime; + UINT64 lastmtime; + void *apme; //应用层上下文 +}; + + +//业务层调用解析层时session_state状态 +#define SESSION_STATE_PENDING 0x01 +#define SESSION_STATE_DATA 0x02 +#define SESSION_STATE_CLOSE 0x04 + +//解析层调用业务层时的返回值； +#define PROT_STATE_GIVEME 0x01 +#define PROT_STATE_DROPME 0x02 +#define PROT_STATE_KILLME 0x04 + +//解析层插件调用业务层插件时传入参数 +typedef struct _plugin_session_info +{ + unsigned short plugid; //plugid，平台分配 + char session_state; //会话状态，PENDING,DATA,CLOSE + char _pad_; //补齐 + int buflen; //当前字段长度 + long long prot_flag; //当前字段的flag值 + void *buf; //当前字段 + void* app_info; //解析层上下文信息 +}stSessionInfo; + +#ifdef __cplusplus +extern "C" { +#endif + +/*参数描述： +a_*： 本流上下文信息； +f_*: 本包所对应的父流信息； +a_packet: 原始ip包； +pme: 私有数据指针，将来扩展用，暂时为NULL +thread_seq：并行分析的时候分析当前连接的线程序号 + +函数返回值描述：为下面四个值的运算 + + APP_STATE_GIVEME：继续向本函数送包。 + APP_STATE_DROPME：不再向本函数送包。 + APP_STATE_FAWPKT：回注该数据包 + APP_STATE_DROPPKT：不回注该数据包 +*/ +char IP_ENTRY( struct streaminfo *f_stream,unsigned char routedir,int thread_seq,struct ip * a_packet); +char IPv6_ENTRY( struct streaminfo *f_stream,unsigned char routedir,int thread_seq,struct ip6_hdr *a_packet); +char TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet); +char UDP_ENTRY(struct streaminfo *a_udp, void **pme, int thread_seq,void *a_packet); + +int stream_register_tcp_allpkt (char (*x)(struct streaminfo *pstream,void **pme, int thread_seq,void *a_packet)); +int stream_register_tcp_takeover (char (*x)(struct streaminfo *pstream,void **pme, int thread_seq,void *a_packet)); + +int stream_register_tcp (char (*x)(struct streaminfo *pstream,void **pme, int thread_seq,void *a_packet)); +int stream_register_udp (char (*x)(struct streaminfo *pstream,void **pme, int thread_seq,void *a_packet)); +//int stream_register_ip (char (*x)(struct streaminfo *pstream,void **pme, int thread_seq,void *a_packet)); +int stream_register_ip (char (*x)(struct streaminfo *f_stream,unsigned char routedir,int thread_seq,struct ip * a_packet)); +int stream_register_ipv6 (char (*x)(struct streaminfo *f_stream,unsigned char routedir,int thread_seq,struct ip6_hdr *a_packet)); + +char * printaddr (struct ipaddr *paddrinfo,int threadindex); + + +void *dictator_malloc(int thread_seq,int size); +void dictator_free(int thread_seq,void *pbuf); + +char PROT_PROCESS(stSessionInfo* session_info, void **pme, int thread_seq,struct streaminfo *a_stream,void *a_packet); + + +/* 用于应用层插件获取本流中对应的用户标签 */ +int terminal_tag_probe(struct streaminfo *stream, struct tcphdr *this_tcphdr,void *rawippkt); +const unsigned char *get_terminal_tag(struct streaminfo *stream); +int MESA_kill_tcp(struct streaminfo *stream, struct ip *a_packet); + +#if RAW_IP_FRAG_PKT +extern raw_ipfrag_list_t **G_IP_FRAG_LIST; +void frags_list_free(raw_ipfrag_list_t *frags_list); +raw_ipfrag_list_t *frags_list_merge(raw_ipfrag_list_t *new_merge_list, raw_ipfrag_list_t *old_list); +#endif + +#if FOR_108 +unsigned long papp_get_streamid(); +#endif + + +#ifdef __cplusplus +} +#endif + + + +#endif diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 0000000..dfe9e2d --- /dev/null +++ b/test/Makefile @@ -0,0 +1,6 @@ +LIBS=../lib/libmaatframe.so -lMESA_handle_logger -lMESA_htable -lrulescan -lpcre -lpthread +INC=-I../inc/ -I/usr/include/MESA/ +all: + g++ -o maat_test -g -Wall maat_test.cpp $(INC) $(LIBS) +clean: + rm maat_test test.log* -f diff --git a/test/maat_json.json b/test/maat_json.json new file mode 100644 index 0000000..cde0b93 --- /dev/null +++ b/test/maat_json.json @@ -0,0 +1,185 @@ +{ + "compile_table": "COMPILE", + "group_table": "GROUP", + "rules": [ + { + "compile_id": 123, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_rage": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "group_1", + "regions": [ + { + "table_name": "IP_CONFIG", + "table_type": "ip", + "table_content": { + "addr_type": "ipv4", + "src_ip": "10.0.6.201", + "mask_src_ip": "255.255.0.0", + "src_port": "0", + "mask_src_port": "65535", + "dst_ip": "0.0.0.0", + "mask_dst_ip": "255.255.255.255", + "dst_port": "0", + "mask_dst_port": "65535", + "protocol": 6, + "direction": "double" + } + }, + { + "table_name": "IP_CONFIG", + "table_type": "ip", + "table_content": { + "addr_type": "ipv6", + "src_ip": "2001:da8:205:1::101", + "mask_src_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:0000", + "src_port": "0", + "mask_src_port": "65535", + "dst_ip": "0::0", + "mask_dst_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "dst_port": "0", + "mask_dst_port": "65535", + "protocol": 6, + "direction": "double" + } + } + ] + }, + { + "group_name": "group_2", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "abckkk&123", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 124, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_rage": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "group_1" + }, + { + "group_name": "group_3", + "regions": [ + { + "table_name": "CONTENT_SIZE", + "table_type": "intval", + "table_content": { + "low_boundary": 100, + "up_boundary": 500 + } + } + ] + } + ] + }, + { + "compile_id": 125, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_rage": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "group_4", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "action=search\\&query=(.*)", + "expr_type": "regex", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 126, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_rage": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "group_5", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "should_not_hit_any_rule", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "group_6", + "regions": [ + { + "table_name": "CONTENT_SIZE", + "table_type": "intval", + "table_content": { + "low_boundary": 2014, + "up_boundary": 2016 + } + } + ] + } + ] + } + ], + "plugin_table": [ + { + "table_name": "QD_ENTRY_INFO", + "table_content": [ + "1\t192.168.0.1\t101", + "2\t192.168.0.2\t101", + "3\t192.168.1.1\t102" + ] + }, + { + "table_name": "TEST_PLUGIN_TABLE", + "table_content": [ + "1\t3388\t99\t1", + "2\t3355\t66\t1", + "3\tcccc\t11\t1" + ] + } + ] +} \ No newline at end of file diff --git a/test/maat_json.json.bak b/test/maat_json.json.bak new file mode 100644 index 0000000..5416cc3 --- /dev/null +++ b/test/maat_json.json.bak @@ -0,0 +1,185 @@ +{ + "compile_table": "COMPILE", + "group_table": "GROUP", + "rules": [ + { + "compile_id": 123, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_rage": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "group_1", + "regions": [ + { + "table_name": "IP_CONFIG", + "table_type": "ip", + "table_content": { + "addr_type": "ipv4", + "src_ip": "10.0.6.201", + "mask_src_ip": "255.255.0.0", + "src_port": "0", + "mask_src_port": "65535", + "dst_ip": "0.0.0.0", + "mask_dst_ip": "255.255.255.255", + "dst_port": "0", + "mask_dst_port": "65535", + "protocol": 6, + "direction": "double" + } + }, + { + "table_name": "IP_CONFIG", + "table_type": "ip", + "table_content": { + "addr_type": "ipv6", + "src_ip": "2001:da8:205:1::101", + "mask_src_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "src_port": "0", + "mask_src_port": "65535", + "dst_ip": "0::0", + "mask_dst_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "dst_port": "0", + "mask_dst_port": "65535", + "protocol": 6, + "direction": "double" + } + } + ] + }, + { + "group_name": "group_2", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "abckkk&123", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 124, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_rage": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "group_1" + }, + { + "group_name": "group_3", + "regions": [ + { + "table_name": "CONTENT_SIZE", + "table_type": "intval", + "table_content": { + "low_boundary": 100, + "up_boundary": 500 + } + } + ] + } + ] + }, + { + "compile_id": 125, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_rage": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "group_4", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "action=search\\&query=(.*)", + "expr_type": "regex", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 126, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_rage": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "group_5", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "should_not_hit_any_rule", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "group_6", + "regions": [ + { + "table_name": "CONTENT_SIZE", + "table_type": "intval", + "table_content": { + "low_boundary": 2014, + "up_boundary": 2016 + } + } + ] + } + ] + } + ], + "plugin_table": [ + { + "table_name": "QD_ENTRY_INFO", + "table_content": [ + "1\t192.168.0.1\t101", + "2\t192.168.0.2\t101", + "3\t192.168.1.1\t102" + ] + }, + { + "table_name": "TEST_PLUGIN_TABLE", + "table_content": [ + "1\t3388\t99\t1", + "2\t3355\t66\t1", + "3\tcccc\t11\t1" + ] + } + ] +} \ No newline at end of file diff --git a/test/maat_json.json_iris_tmp/COMPILE.local b/test/maat_json.json_iris_tmp/COMPILE.local new file mode 100644 index 0000000..9704fb1 --- /dev/null +++ b/test/maat_json.json_iris_tmp/COMPILE.local @@ -0,0 +1,5 @@ +0000000004 +123 1 1 1 1 0 anything 1 +124 1 1 1 1 0 anything 1 +125 1 1 1 1 0 anything 1 +126 1 1 1 1 0 anything 1 diff --git a/test/maat_json.json_iris_tmp/CONTENT_SIZE.local b/test/maat_json.json_iris_tmp/CONTENT_SIZE.local new file mode 100644 index 0000000..e1bc0a6 --- /dev/null +++ b/test/maat_json.json_iris_tmp/CONTENT_SIZE.local @@ -0,0 +1,3 @@ +0000000002 +3 2 100 500 1 +6 5 2014 2016 1 diff --git a/test/maat_json.json_iris_tmp/GROUP.local b/test/maat_json.json_iris_tmp/GROUP.local new file mode 100644 index 0000000..b65def4 --- /dev/null +++ b/test/maat_json.json_iris_tmp/GROUP.local @@ -0,0 +1,8 @@ +0000000007 +0 123 1 +1 123 1 +0 124 1 +2 124 1 +3 125 1 +4 126 1 +5 126 1 diff --git a/test/maat_json.json_iris_tmp/HTTP_URL.local b/test/maat_json.json_iris_tmp/HTTP_URL.local new file mode 100644 index 0000000..b473eae --- /dev/null +++ b/test/maat_json.json_iris_tmp/HTTP_URL.local @@ -0,0 +1,4 @@ +0000000003 +2 1 abckkk&123 1 0 0 1 +4 3 action=search\&query=(.*) 2 0 0 1 +5 4 should_not_hit_any_rule 0 0 0 1 diff --git a/test/maat_json.json_iris_tmp/IP_CONFIG.local b/test/maat_json.json_iris_tmp/IP_CONFIG.local new file mode 100644 index 0000000..7936e99 --- /dev/null +++ b/test/maat_json.json_iris_tmp/IP_CONFIG.local @@ -0,0 +1,3 @@ +0000000002 +0 0 4 10.0.6.201 255.255.0.0 0 65535 0.0.0.0 255.255.255.255 0 65535 6 0 1 +1 0 4 192.168.0.1 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 6 0 1 diff --git a/test/maat_json.json_iris_tmp/QD_ENTRY_INFO.local b/test/maat_json.json_iris_tmp/QD_ENTRY_INFO.local new file mode 100644 index 0000000..cb1bd15 --- /dev/null +++ b/test/maat_json.json_iris_tmp/QD_ENTRY_INFO.local @@ -0,0 +1,4 @@ +0000000003 +1 192.168.0.1 101 +2 192.168.0.2 101 +3 192.168.1.1 102 diff --git a/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE.local b/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE.local new file mode 100644 index 0000000..763d635 --- /dev/null +++ b/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE.local @@ -0,0 +1,4 @@ +0000000003 +1 3388 99 1 +2 3355 66 1 +3 cccc 11 1 diff --git a/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE_NO1.local b/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE_NO1.local new file mode 100644 index 0000000..b278b17 --- /dev/null +++ b/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE_NO1.local @@ -0,0 +1,4 @@ +0000000003 +1 plugname1 1 +2 plugname2 1 +3 plugname3 1 diff --git a/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE_NO2.local b/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE_NO2.local new file mode 100644 index 0000000..62580c9 --- /dev/null +++ b/test/maat_json.json_iris_tmp/TEST_PLUGIN_TABLE_NO2.local @@ -0,0 +1 @@ +0000000000 diff --git a/test/maat_json.json_iris_tmp/index/full_config_index.0000000001 b/test/maat_json.json_iris_tmp/index/full_config_index.0000000001 new file mode 100644 index 0000000..74047de --- /dev/null +++ b/test/maat_json.json_iris_tmp/index/full_config_index.0000000001 @@ -0,0 +1,7 @@ +COMPILE 4 ./maat_json.json_iris_tmp/COMPILE.local +GROUP 7 ./maat_json.json_iris_tmp/GROUP.local +TEST_PLUGIN_TABLE 3 ./maat_json.json_iris_tmp/TEST_PLUGIN_TABLE.local +IP_CONFIG 2 ./maat_json.json_iris_tmp/IP_CONFIG.local +CONTENT_SIZE 2 ./maat_json.json_iris_tmp/CONTENT_SIZE.local +HTTP_URL 3 ./maat_json.json_iris_tmp/HTTP_URL.local +QD_ENTRY_INFO 3 ./maat_json.json_iris_tmp/QD_ENTRY_INFO.local diff --git a/test/maat_test.cpp b/test/maat_test.cpp new file mode 100644 index 0000000..546bf39 --- /dev/null +++ b/test/maat_test.cpp @@ -0,0 +1,201 @@ +#include "Maat_rule.h" +#include +#include +#include +#include //inet_addr +#include //inet_addr +#include //inet_addr +#include +#include +#include +#include +#include +#include + +void Maat_read_entry_start_cb(int update_type,void* u_para) +{ + return; +} +void Maat_read_entry_cb(int table_id,const char* table_line,void* u_para) +{ + char ip_str[16]={0}; + int entry_id=-1,seq=-1; + unsigned int ip_uint=0; + unsigned int local_ip_nr=16820416;//192.168.0.1 + sscanf(table_line,"%d\t%s\t%d",&seq,ip_str,&entry_id); + inet_pton(AF_INET,ip_str,&ip_uint); + if(local_ip_nr==ip_uint) + { + printf("Load entry id %d SUCCESS.\n",entry_id); + } + return; +} +void Maat_read_entry_finish_cb(void* u_para) +{ + return; +} +int main() +{ + Maat_feather_t feather=NULL; + int cb_table_id=-1,url_scan_table_id=-1,size_scan_table_id=-1,ip_scan_table_id=-1; + int ret=-1; + int g_iThreadNum=4; + const char* table_info_path="./table_info.conf"; + const char* json_path="./maat_json.json"; +// const char* ful_cfg_dir="./maat_json.json_iris_tmp/index"; +// const char* inc_cfg_dir="./rules/inc/index"; + const char* log_file="./test.log"; + const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; + int scan_val=2015; + struct Maat_rule_t result[4]; + int found_pos[4]; + scan_status_t mid=NULL; + + void *logger=MESA_create_runtime_log_handle(log_file,0); + feather=Maat_summon_feather_json(g_iThreadNum, + table_info_path, + json_path, + logger); +// feather=Maat_summon_feather(g_iThreadNum,table_info_path,ful_cfg_dir,inc_cfg_dir,logger); + if(feather==NULL) + { + printf("Maat initial error, see %s\n",log_file); + return -1; + } + + cb_table_id=Maat_table_register(feather,"QD_ENTRY_INFO"); + if(cb_table_id==-1) + { + printf("Database table QD_ENTRY_INFO register failed."); + } + else + { + ret=Maat_table_callback_register(feather, cb_table_id, + Maat_read_entry_start_cb, + Maat_read_entry_cb, + Maat_read_entry_finish_cb, + logger); + if(ret<0) + { + printf("Maat callback register table QD_ENTRY_INFO error."); + } + } + + url_scan_table_id=Maat_table_register(feather,"HTTP_URL"); + if(url_scan_table_id==-1) + { + printf("Database table HTTP_URL register failed."); + + } + else + { + ret=Maat_full_scan_string(feather, url_scan_table_id,CHARSET_GBK, scan_data, strlen(scan_data), + result,found_pos, 4, + &mid, 0); + switch(ret) + { + case -1: + printf("scan error.\n"); + break; + case -2: + printf("hit current region,but not hit compile rule.\n"); + break; + case 0: + printf("nothing hit\n"); + break; + default://>0 + printf("hit %d rules\n",ret); + break; + } + } + Maat_clean_status(&mid); + size_scan_table_id=Maat_table_register(feather,"CONTENT_SIZE"); + if(size_scan_table_id==-1) + { + printf("Database table CONTENT_SIZE register failed."); + + } + else + { + ret=Maat_scan_intval(feather, size_scan_table_id, scan_val, result,4,&mid, 0); + switch(ret) + { + case -1: + printf("scan error.\n"); + break; + case -2: + printf("hit current region,but not hit compile rule.\n"); + break; + case 0: + printf("nothing hit\n"); + break; + default://>0 + printf("hit %d rules\n",ret); + break; + } + } + Maat_clean_status(&mid); + struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10); + stream_para_t sp=Maat_stream_scan_string_start(feather,url_scan_table_id,0); + int detail_ret=0; + if(sp==NULL) + { + printf("stream scan start failed.\n"); + return -1; + } + ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,"www.cyberessays.com", strlen("www.cyberessays.com") + ,result,4,hit_detail,10 + ,&detail_ret,&mid); + ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data, strlen(scan_data) + ,result,4,hit_detail,10 + ,&detail_ret,&mid); + Maat_stream_scan_string_end(&sp); + + struct ipaddr ipv4_addr,ipv6_addr; + struct stream_tuple4_v4 v4_addr; + struct stream_tuple4_v6 v6_addr; + ipv4_addr.addrtype=ADDR_TYPE_IPV4; + inet_pton(AF_INET,"10.0.6.205",&(v4_addr.saddr)); + v4_addr.source=htons(50001); + inet_pton(AF_INET,"10.0.6.201",&(v4_addr.daddr)); + v4_addr.dest=htons(80); + ipv4_addr.v4=&v4_addr; + + ip_scan_table_id=Maat_table_register(feather,"IP_CONFIG"); + if(ip_scan_table_id==-1) + { + printf("Database table IP_CONFIG register failed."); + + } + else + { + ret=Maat_scan_proto_addr(feather,ip_scan_table_id,&ipv4_addr,6,result,4, &mid,0); + if(ret>0) + { + printf("ipv4 scan hit compile rule id %d.\n",result[0].config_id); + } + } + Maat_clean_status(&mid); + ipv6_addr.addrtype=ADDR_TYPE_IPV6; + inet_pton(AF_INET6,"2001:da8:205:1::101",&(v6_addr.saddr)); + v6_addr.source=htons(50001); + inet_pton(AF_INET6,"2001:da8:205:1::102",&(v6_addr.daddr)); + v6_addr.dest=htons(80); + ipv6_addr.v6=&v6_addr; + if(ip_scan_table_id>=0) + { + ret=Maat_scan_proto_addr(feather,ip_scan_table_id,&ipv6_addr,6,result,4, &mid,0); + if(ret==-2) + { + printf("ipv6 scan hit region.\n"); + } + else + { + printf("ipv6 scan result:%d ,shoulde be -2.\n",ret); + + } + } + Maat_burn_feather(feather); + free(hit_detail); + return 0; +} diff --git a/test/table_info.conf b/test/table_info.conf new file mode 100644 index 0000000..1846fc3 --- /dev/null +++ b/test/table_info.conf @@ -0,0 +1,15 @@ +#each collumn seperate with '\t' +#id (0~65535) +#name string +#type one of ip,expr,compile or plugin +#src_charset one of GBK,BIG5,UNICODE,UTF8 +#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/' +#do_merege yes or no +#id name type src_charset dst_charset do_merge +0 COMPILE compile GBK GBK no 0 +1 GROUP group GBK GBK no 0 +2 HTTP_URL expr GBK GBK/BIG5/UNICODE/UTF8 yes 128 +3 KEYWORDS_TABLE expr GBK GBK/BIG5/UNICODE/UTF8 yes 0 +4 IP_CONFIG ip GBK GBK no 0 +5 CONTENT_SIZE intval GBK GBK no 0 +6 QD_ENTRY_INFO plugin GBK GBK no 0