diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 090c194..03bbcb6 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -105,7 +105,7 @@ int region_compile(struct _scan_status_t *_mid,void* region_hit,int region_type_ _mid->cur_hit_cnt=0; for(i=0;igroup_id<0) { continue; @@ -359,7 +359,7 @@ Maat_feather_t Maat_summon_feather(int max_thread_num, { _Maat_feather_t* feather=(_Maat_feather_t*)calloc(sizeof(struct _Maat_feather_t),1); - feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path); + feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path,logger); feather->map_tablename2id=map_create(); int i=0; for(i=0;itable_id>=num) { fprintf(stderr,"Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,num); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,num); + goto error_jump; } if(p_table_info[p->table_id]!=NULL) { fprintf(stderr,"Maat read table info %s line %d error:duplicated table id %d.\n",table_info_path,i,p->table_id); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:duplicated table id %d.\n",table_info_path,i,p->table_id); + goto error_jump; } if(p->table_type==TABLE_TYPE_PLUGIN) diff --git a/src/entry/Maat_rule_internal.h b/src/entry/Maat_rule_internal.h index 0496c87..37008a3 100644 --- a/src/entry/Maat_rule_internal.h +++ b/src/entry/Maat_rule_internal.h @@ -321,7 +321,7 @@ struct _maat_garbage_t void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bool_expr_t* a_set); -int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path); +int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,void*logger); void maat_start_cb(unsigned int new_version,int update_type,void*u_para); void maat_update_cb(const char* table_name,const char* line,void *u_para); void maat_finish_cb(void* u_para); diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 30cc77d..cdf5a98 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -448,7 +448,7 @@ int write_digest_rule(cJSON *region_json,struct iris_description_t *p_iris,const cmd_cnt++; json_cmd[cmd_cnt].json_string="raw_len"; - json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; json_cmd[cmd_cnt].json_string="digest"; @@ -456,7 +456,7 @@ int write_digest_rule(cJSON *region_json,struct iris_description_t *p_iris,const cmd_cnt++; json_cmd[cmd_cnt].json_string="cfds_level"; - json_cmd[cmd_cnt].json_type=cJSON_String; + json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; json_cmd[cmd_cnt].json_string="is_valid"; diff --git a/src/entry/mesa_fuzzy.c b/src/entry/mesa_fuzzy.c index b9666f7..22adcba 100644 --- a/src/entry/mesa_fuzzy.c +++ b/src/entry/mesa_fuzzy.c @@ -186,6 +186,10 @@ void fuzzy_destroy_handle(fuzzy_handle_t * handle) */ unsigned int fuzzy_feed(fuzzy_handle_t * handle, const char * data, unsigned int size, unsigned long long offset) { + if(data==NULL||size==0) + { + return 0; + } fuzzy_node * node = (fuzzy_node *)calloc(sizeof(fuzzy_node), 1); node->right_status_r = (struct roll_state *)calloc(sizeof (struct roll_state), 1); roll_init(node->right_status_r); diff --git a/test/Makefile b/test/Makefile index 6d74f36..44d1c16 100644 --- a/test/Makefile +++ b/test/Makefile @@ -2,5 +2,6 @@ LIBS=../lib/libmaatframe.so INC=-I../inc/ -I/usr/include/MESA/ all: g++ -o maat_test -g -Wall maat_test.cpp $(INC) $(LIBS) + g++ -o digest_gen -g digest_gen.c -I../src/inc_internal/ ../lib/libmaatframe.so clean: rm maat_test test.log* -f diff --git a/test/digest_gen.c b/test/digest_gen.c new file mode 100644 index 0000000..bd32201 --- /dev/null +++ b/test/digest_gen.c @@ -0,0 +1,135 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "mesa_fuzzy.h" + +void* entropy_start(void) +{ + unsigned long long * char_num=(unsigned long long*)calloc(sizeof(unsigned long long),256+1); + return (void*)char_num; +} +void entropy_feed(void* handle,const unsigned char*buff, int size) +{ + int i=0; + unsigned long long * char_num=(unsigned long long *)handle; + for(i=0;id_name, ".") ||!strcmp(file->d_name, "..")||file->d_type!=DT_REG) + { + continue; + } + ret=stat(file->d_name,&digest_fstat); + if(ret!=0) + { + printf("fstat %s error.\n",file->d_name); + continue; + } + + off_t file_size = digest_fstat.st_size; + fp = fopen(file->d_name, "r"); + if(NULL == fp) + { + printf("Can't open file %s\n", file->d_name); + continue; + } + read_size=0; + feed_offset=0; + fuzzy_handle_t * fhandle = fuzzy_create_handle((unsigned long long)file_size); + entropy_handle=entropy_start(); + while(0==feof(fp)) + { + read_size=fread(read_buff,1,sizeof(read_buff),fp); + fuzzy_feed(fhandle,read_buff,read_size,feed_offset); + feed_offset+=read_size; + entropy_feed(entropy_handle,(const unsigned char*) read_buff, read_size); + } + file_entropy=entropy_stop(entropy_handle); + hash_length = fuzzy_status(fhandle, HASH_LENGTH); + file_effective_length = fuzzy_status(fhandle, EFFECTIVE_LENGTH); + digest_result_buff= (char *)malloc(sizeof(char) * (hash_length)); + if(fuzzy_digest(fhandle, digest_result_buff, hash_length) != 0) + { + printf("error\n"); + continue; + } + fprintf(result_fp, "%u\t%s\t%llu\t%lu\t%lf\n", file_id, file->d_name,file_size, hash_length,file_entropy); + fprintf(result_fp, "%s\n", digest_result_buff); + printf("%u %s\n", file_id,file->d_name); + file_id++; + fuzzy_destroy_handle(fhandle); + fclose(fp); + free(digest_result_buff); + } + fclose(result_fp); + closedir(dir); + printf("write result to %s\n", result_file); + +} + + +int main(int argc, char * argv[]) +{ + dir_digest(argc, argv); + //overlap_test(argc, argv); + return 0; +} diff --git a/test/maat_json.json b/test/maat_json.json index cde0b93..b69da02 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -162,6 +162,32 @@ ] } ] + }, + { + "compile_id": 127, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_rage": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "group_7", + "regions": [ + { + "table_name": "FILE_DIGEST", + "table_type": "digest", + "table_content": { + "raw_len": 46292790, + "digest": "5nN+XW1badOlW7Odi6K[7203:46299992]", + "cfds_level": 3 + } + } + ] + } + ] } ], "plugin_table": [ @@ -182,4 +208,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/test/maat_json.json.bak b/test/maat_json.json.bak deleted file mode 100644 index 5416cc3..0000000 --- a/test/maat_json.json.bak +++ /dev/null @@ -1,185 +0,0 @@ -{ - "compile_table": "COMPILE", - "group_table": "GROUP", - "rules": [ - { - "compile_id": 123, - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, - "effective_rage": 0, - "user_region": "anything", - "is_valid": "yes", - "groups": [ - { - "group_name": "group_1", - "regions": [ - { - "table_name": "IP_CONFIG", - "table_type": "ip", - "table_content": { - "addr_type": "ipv4", - "src_ip": "10.0.6.201", - "mask_src_ip": "255.255.0.0", - "src_port": "0", - "mask_src_port": "65535", - "dst_ip": "0.0.0.0", - "mask_dst_ip": "255.255.255.255", - "dst_port": "0", - "mask_dst_port": "65535", - "protocol": 6, - "direction": "double" - } - }, - { - "table_name": "IP_CONFIG", - "table_type": "ip", - "table_content": { - "addr_type": "ipv6", - "src_ip": "2001:da8:205:1::101", - "mask_src_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", - "src_port": "0", - "mask_src_port": "65535", - "dst_ip": "0::0", - "mask_dst_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", - "dst_port": "0", - "mask_dst_port": "65535", - "protocol": 6, - "direction": "double" - } - } - ] - }, - { - "group_name": "group_2", - "regions": [ - { - "table_name": "HTTP_URL", - "table_type": "string", - "table_content": { - "keywords": "abckkk&123", - "expr_type": "and", - "match_method": "sub", - "format": "uncase plain" - } - } - ] - } - ] - }, - { - "compile_id": 124, - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, - "effective_rage": 0, - "user_region": "anything", - "is_valid": "yes", - "groups": [ - { - "group_name": "group_1" - }, - { - "group_name": "group_3", - "regions": [ - { - "table_name": "CONTENT_SIZE", - "table_type": "intval", - "table_content": { - "low_boundary": 100, - "up_boundary": 500 - } - } - ] - } - ] - }, - { - "compile_id": 125, - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, - "effective_rage": 0, - "user_region": "anything", - "is_valid": "yes", - "groups": [ - { - "group_name": "group_4", - "regions": [ - { - "table_name": "HTTP_URL", - "table_type": "string", - "table_content": { - "keywords": "action=search\\&query=(.*)", - "expr_type": "regex", - "match_method": "sub", - "format": "uncase plain" - } - } - ] - } - ] - }, - { - "compile_id": 126, - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, - "effective_rage": 0, - "user_region": "anything", - "is_valid": "yes", - "groups": [ - { - "group_name": "group_5", - "regions": [ - { - "table_name": "HTTP_URL", - "table_type": "string", - "table_content": { - "keywords": "should_not_hit_any_rule", - "expr_type": "none", - "match_method": "sub", - "format": "uncase plain" - } - } - ] - }, - { - "group_name": "group_6", - "regions": [ - { - "table_name": "CONTENT_SIZE", - "table_type": "intval", - "table_content": { - "low_boundary": 2014, - "up_boundary": 2016 - } - } - ] - } - ] - } - ], - "plugin_table": [ - { - "table_name": "QD_ENTRY_INFO", - "table_content": [ - "1\t192.168.0.1\t101", - "2\t192.168.0.2\t101", - "3\t192.168.1.1\t102" - ] - }, - { - "table_name": "TEST_PLUGIN_TABLE", - "table_content": [ - "1\t3388\t99\t1", - "2\t3355\t66\t1", - "3\tcccc\t11\t1" - ] - } - ] -} \ No newline at end of file diff --git a/test/maat_test.cpp b/test/maat_test.cpp index 4b92c1d..ff43fc4 100644 --- a/test/maat_test.cpp +++ b/test/maat_test.cpp @@ -219,7 +219,7 @@ int main() if(fp!=NULL) { sp=Maat_stream_scan_digest_start(feather, digest_scan_table_id, digest_fstat.st_size, 0); - while(0!=feof(fp)) + while(0==feof(fp)) { read_size=fread(digest_test_buff,1,sizeof(digest_test_buff),fp); ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,&mid);