From 6bac9a6fc84d1635f30f77e7d2f1568240ecd61c Mon Sep 17 00:00:00 2001 From: zhengchao Date: Thu, 12 Nov 2015 17:49:57 +0800 Subject: [PATCH] =?UTF-8?q?1=E3=80=81=E6=96=B0=E5=A2=9E=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=94=A8=E4=BE=8B=EF=BC=9B2=E3=80=81json=E6=94=AF=E6=8C=81dige?= =?UTF-8?q?st=E7=B1=BB=E5=9E=8B=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 9 ++++---- src/entry/json2iris.cpp | 39 ++++++++++++++++++++++++++++++- src/inc_internal/mesa_fuzzy.h | 2 +- test/maat_test.cpp | 43 ++++++++++++++++++++++++++++++++++- 4 files changed, 86 insertions(+), 7 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index c0f3860..fe5732e 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -1039,13 +1039,14 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, sp->table_id=table_id; sp->thread_num=thread_num; sp->total_len=total_len; - sp->fuzzy_hash_handle=fuzzy_create_handle(); + sp->fuzzy_hash_handle=fuzzy_create_handle(total_len); return sp; } -#define QUERY_MIN_RATE (3) //30% -#define QUERY_MIN_LEN (1024*1024*4) + inline int REACH_QUERY_THRESH(unsigned long long total_len,unsigned long long acc_len,unsigned char* query_point,int point_size) { + const int QUERY_MIN_RATE=(3); //30% + const int QUERY_MIN_LEN=(1024*1024*4); //do query every 10 percent since 30%, e.g. 0.3/0.4/0.5/.../1.0 unsigned long long rate=(acc_len*10)/total_len; // if(acc_len>QUERY_MIN_LEN) @@ -1134,7 +1135,7 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para) } fuzzy_destroy_handle(sp->fuzzy_hash_handle); assert(sp->last_cache==NULL); - assert(sp->scan_buff==NULL) + assert(sp->scan_buff==NULL); free(sp); *stream_para=NULL; return; diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index f33e7d8..30cc77d 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -111,7 +111,8 @@ int set_iris_descriptor(const char* json_file,cJSON *json,struct iris_descriptio map_register(iris_cfg->str2int_map, "ip",TABLE_TYPE_IP); map_register(iris_cfg->str2int_map, "string",TABLE_TYPE_EXPR); map_register(iris_cfg->str2int_map, "intval",TABLE_TYPE_INTVAL); - + map_register(iris_cfg->str2int_map, "digest",TABLE_TYPE_DIGEST); + map_register(iris_cfg->str2int_map, "ipv4",4); map_register(iris_cfg->str2int_map, "ipv6",6); @@ -431,6 +432,39 @@ int write_intval_rule(cJSON *region_json,struct iris_description_t *p_iris,const return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); +} +int write_digest_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +{ + struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; + int cmd_cnt=0; + memset(json_cmd,0,sizeof(json_cmd)); + + json_cmd[cmd_cnt].json_string="region_id"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="group_id"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="raw_len"; + json_cmd[cmd_cnt].json_type=cJSON_String; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="digest"; + json_cmd[cmd_cnt].json_type=cJSON_String; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="cfds_level"; + json_cmd[cmd_cnt].json_type=cJSON_String; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string="is_valid"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + } struct iris_table_t* query_table_info(iris_description_t* p_iris,const char* table_name) { @@ -572,6 +606,9 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri case TABLE_TYPE_INTVAL: ret=write_intval_rule(table_content, p_iris, table_info->table_path, logger); break; + case TABLE_TYPE_DIGEST: + ret=write_digest_rule(table_content, p_iris, table_info->table_path, logger); + break; default: assert(0); break; diff --git a/src/inc_internal/mesa_fuzzy.h b/src/inc_internal/mesa_fuzzy.h index c5d5adb..6497d89 100644 --- a/src/inc_internal/mesa_fuzzy.h +++ b/src/inc_internal/mesa_fuzzy.h @@ -34,7 +34,7 @@ typedef struct * create a fuzzy hash handle and return it. * @return [handle] */ -fuzzy_handle_t * fuzzy_create_handle(void); +fuzzy_handle_t * fuzzy_create_handle(unsigned long long total_len); /** * destroy context by a fuzzy hash handle. diff --git a/test/maat_test.cpp b/test/maat_test.cpp index 546bf39..a7ac6d9 100644 --- a/test/maat_test.cpp +++ b/test/maat_test.cpp @@ -37,7 +37,7 @@ void Maat_read_entry_finish_cb(void* u_para) int main() { Maat_feather_t feather=NULL; - int cb_table_id=-1,url_scan_table_id=-1,size_scan_table_id=-1,ip_scan_table_id=-1; + int cb_table_id=-1,url_scan_table_id=-1,size_scan_table_id=-1,ip_scan_table_id=-1,digest_scan_table_id=-1; int ret=-1; int g_iThreadNum=4; const char* table_info_path="./table_info.conf"; @@ -46,6 +46,10 @@ int main() // const char* inc_cfg_dir="./rules/inc/index"; const char* log_file="./test.log"; const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; + const char* digest_test_file="./1.data"; + struct stat digest_fstat; + unsigned long long read_size=0,scan_offset=0; + char digest_test_buff[4096]={0}; int scan_val=2015; struct Maat_rule_t result[4]; int found_pos[4]; @@ -195,6 +199,43 @@ int main() } } + Maat_clean_status(&mid); + + digest_scan_table_id=Maat_table_register(feather, "FILE_DIGEST"); + if(digest_scan_table_id<0) + { + printf("registe table FILE_DIGEST error.\n"); + return 0; + } + ret=fstat(digest_test_file,&digest_fstat); + if(ret!=0) + { + printf("fstat %s error.\n",digest_test_file); + return 0; + } + FILE* fp=fopen(digest_test_file,"r"); + if(fp!=NULL) + { + sp=Maat_stream_scan_digest_start(feather, digest_scan_table_id, digest_fstat->off_t, 0); + while(0!=feof(fp)) + { + read_size=fread(fp,digest_test_buff,sizeof(digest_test_buff)); + ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,&mid); + scan_offset+=read_size; + if(ret>0) + { + printf("digest scan hit %d.\n",result[0].config_id); + + } + } + fclose(fp); + } + else + { + printf("fopen %s error.\n",digest_test_file); + } + Maat_stream_scan_string_end(&sp); + Maat_clean_status(&mid); Maat_burn_feather(feather); free(hit_detail); return 0;