gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

在test目录增加digest_gen工具，计算摘要和信息熵。

Browse Source
This commit is contained in:
zhengchao
2015-11-13 18:08:55 +08:00
parent 76cdf5eaab
commit d1c1bf32aa
10 changed files with 188 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/entry/Maat_api.cpp
View File

@@ -105,7 +105,7 @@ int region_compile(struct _scan_status_t *_mid,void* region_hit,int region_type_
_mid->cur_hit_cnt=0;
for(i=0;i<region_hit_num;i++)
{
group_rule=(struct _Maat_group_rule_t*)((char*)region_hit+region_type_size*i+group_offset);
group_rule=*(struct _Maat_group_rule_t**)((char*)region_hit+region_type_size*i+group_offset);
if(group_rule->group_id<0)
{
continue;
@@ -359,7 +359,7 @@ Maat_feather_t Maat_summon_feather(int max_thread_num,
{
_Maat_feather_t* feather=(_Maat_feather_t*)calloc(sizeof(struct _Maat_feather_t),1);
feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path);
feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path,logger);
feather->map_tablename2id=map_create();
int i=0;
for(i=0;i<MAX_TABLE_NUM;i++)
@@ -1028,7 +1028,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
struct _Maat_scanner_t* scanner=NULL;
struct _Maat_table_info_t *p_table=NULL;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR);
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST);
if(p_table==NULL)
{
return NULL;

15
src/entry/Maat_rule.cpp
View File

@@ -253,7 +253,7 @@ int HASH_delete_by_id(MESA_htable_handle hash,int id)
ret=MESA_htable_del(hash,(unsigned char*)&id, sizeof(id), NULL);
return ret;
}
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path)
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,void* logger)
{
FILE*fp=NULL;
char line[MAX_TABLE_LINE_SIZE];
@@ -268,6 +268,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
map_register(string2int_map,"compile", TABLE_TYPE_COMPILE);
map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN);
map_register(string2int_map,"intval", TABLE_TYPE_INTVAL);
map_register(string2int_map,"digest", TABLE_TYPE_DIGEST);
map_register(string2int_map,"group", TABLE_TYPE_GROUP);
map_register(string2int_map,"bin", CHARSET_NONE);
map_register(string2int_map,"gbk", CHARSET_GBK);
@@ -281,6 +282,8 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
if(fp==NULL)
{
fprintf(stderr,"Maat read table info %s error.\n",table_info_path);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s error.\n",table_info_path);
}
while(NULL!=fgets(line,sizeof(line),fp))
{
@@ -307,6 +310,8 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
if(ret[j]<0)
{
fprintf(stderr,"Maat read table info %s line %d error.\n",table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error.\n",table_info_path,i);
goto error_jump;
}
}
@@ -328,6 +333,8 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
else
{
fprintf(stderr,"Maat read table info %s line %d error.\n",table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error.\n",table_info_path,i);
goto error_jump;
}
@@ -335,11 +342,17 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
if(p->table_id>=num)
{
fprintf(stderr,"Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,num);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,num);
goto error_jump;
}
if(p_table_info[p->table_id]!=NULL)
{
fprintf(stderr,"Maat read table info %s line %d error:duplicated table id %d.\n",table_info_path,i,p->table_id);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:duplicated table id %d.\n",table_info_path,i,p->table_id);
goto error_jump;
}
if(p->table_type==TABLE_TYPE_PLUGIN)

2
src/entry/Maat_rule_internal.h
View File

@@ -321,7 +321,7 @@ struct _maat_garbage_t
void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bool_expr_t* a_set);
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path);
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,void*logger);
void maat_start_cb(unsigned int new_version,int update_type,void*u_para);
void maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para);

4
src/entry/json2iris.cpp
View File

@@ -448,7 +448,7 @@ int write_digest_rule(cJSON *region_json,struct iris_description_t *p_iris,const
cmd_cnt++;
json_cmd[cmd_cnt].json_string="raw_len";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].json_type=cJSON_Number;
cmd_cnt++;
json_cmd[cmd_cnt].json_string="digest";
@@ -456,7 +456,7 @@ int write_digest_rule(cJSON *region_json,struct iris_description_t *p_iris,const
cmd_cnt++;
json_cmd[cmd_cnt].json_string="cfds_level";
json_cmd[cmd_cnt].json_type=cJSON_String;
json_cmd[cmd_cnt].json_type=cJSON_Number;
cmd_cnt++;
json_cmd[cmd_cnt].json_string="is_valid";

4
src/entry/mesa_fuzzy.c
View File

@@ -186,6 +186,10 @@ void fuzzy_destroy_handle(fuzzy_handle_t * handle)
*/
unsigned int fuzzy_feed(fuzzy_handle_t * handle, const char * data, unsigned int size, unsigned long long offset)
{
if(data==NULL||size==0)
{
return 0;
}
fuzzy_node * node = (fuzzy_node *)calloc(sizeof(fuzzy_node), 1);
node->right_status_r = (struct roll_state *)calloc(sizeof (struct roll_state), 1);
roll_init(node->right_status_r);

1
test/Makefile
View File

@@ -2,5 +2,6 @@ LIBS=../lib/libmaatframe.so
INC=-I../inc/ -I/usr/include/MESA/
all:
g++ -o maat_test -g -Wall maat_test.cpp $(INC) $(LIBS)
g++ -o digest_gen -g digest_gen.c -I../src/inc_internal/ ../lib/libmaatframe.so
clean:
rm maat_test test.log* -f

135
test/digest_gen.c Normal file
View File

@@ -0,0 +1,135 @@
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<unistd.h>
#include<dirent.h>
#include<sys/stat.h>
#include<time.h>
#include<math.h>
#include "mesa_fuzzy.h"
void* entropy_start(void)
{
unsigned long long * char_num=(unsigned long long*)calloc(sizeof(unsigned long long),256+1);
return (void*)char_num;
}
void entropy_feed(void* handle,const unsigned char*buff, int size)
{
int i=0;
unsigned long long * char_num=(unsigned long long *)handle;
for(i=0;i<size;i++)
{
char_num[buff[i]+1]++;;
}
char_num[0]+=size;
return;
}
double entropy_stop(void* handle)
{
unsigned long long * char_num=(unsigned long long *)handle;
int i;
double sum = 0,p=0;
for(i = 0; i < 256; i++)
{
p = (double)char_num[i+1]/char_num[0];
if(p != 0)
{
sum += (p*(log(p)/log(2)));
}
}
free(handle);
return (-sum);
}
void dir_digest(int argc, char * argv[])
{
if(argc != 2)
{
printf("uasge: ./digest_gen [Dir]\n");
exit(-1);
}
DIR * dir;
struct dirent * file;
char * dir_path = argv[1];
char read_buff[1024*4];
unsigned long long read_size=0,feed_offset=0;
dir = opendir(dir_path);
chdir(dir_path);
int ret =0;
unsigned int file_id = 1;
unsigned long hash_length=0,file_effective_length=0;
FILE * result_fp = NULL,*fp=NULL;
struct stat digest_fstat;
char * digest_result_buff=NULL;
const char* result_file="./digest_result.txt";
result_fp = fopen(result_file,"a");
void * entropy_handle=NULL;
double file_entropy=0.0;
if(NULL == result_fp)
{
printf("open file failed!");
exit(-1);
}
while((file = readdir(dir)) != NULL)
{
if(!strcmp(file->d_name, ".") ||!strcmp(file->d_name, "..")||file->d_type!=DT_REG)
{
continue;
}
ret=stat(file->d_name,&digest_fstat);
if(ret!=0)
{
printf("fstat %s error.\n",file->d_name);
continue;
}
off_t file_size = digest_fstat.st_size;
fp = fopen(file->d_name, "r");
if(NULL == fp)
{
printf("Can't open file %s\n", file->d_name);
continue;
}
read_size=0;
feed_offset=0;
fuzzy_handle_t * fhandle = fuzzy_create_handle((unsigned long long)file_size);
entropy_handle=entropy_start();
while(0==feof(fp))
{
read_size=fread(read_buff,1,sizeof(read_buff),fp);
fuzzy_feed(fhandle,read_buff,read_size,feed_offset);
feed_offset+=read_size;
entropy_feed(entropy_handle,(const unsigned char*) read_buff, read_size);
}
file_entropy=entropy_stop(entropy_handle);
hash_length = fuzzy_status(fhandle, HASH_LENGTH);
file_effective_length = fuzzy_status(fhandle, EFFECTIVE_LENGTH);
digest_result_buff= (char *)malloc(sizeof(char) * (hash_length));
if(fuzzy_digest(fhandle, digest_result_buff, hash_length) != 0)
{
printf("error\n");
continue;
}
fprintf(result_fp, "%u\t%s\t%llu\t%lu\t%lf\n", file_id, file->d_name,file_size, hash_length,file_entropy);
fprintf(result_fp, "%s\n", digest_result_buff);
printf("%u %s\n", file_id,file->d_name);
file_id++;
fuzzy_destroy_handle(fhandle);
fclose(fp);
free(digest_result_buff);
}
fclose(result_fp);
closedir(dir);
printf("write result to %s\n", result_file);
}
int main(int argc, char * argv[])
{
dir_digest(argc, argv);
//overlap_test(argc, argv);
return 0;
}

26
test/maat_json.json
View File

@@ -162,6 +162,32 @@
]
}
]
},
{
"compile_id": 127,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"effective_rage": 0,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "group_7",
"regions": [
{
"table_name": "FILE_DIGEST",
"table_type": "digest",
"table_content": {
"raw_len": 46292790,
"digest": "5nN+XW1badOlW7Odi6K[7203:46299992]",
"cfds_level": 3
}
}
]
}
]
}
],
"plugin_table": [

185
test/maat_json.json.bak
View File

@@ -1,185 +0,0 @@
{
"compile_table": "COMPILE",
"group_table": "GROUP",
"rules": [
{
"compile_id": 123,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"effective_rage": 0,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "group_1",
"regions": [
{
"table_name": "IP_CONFIG",
"table_type": "ip",
"table_content": {
"addr_type": "ipv4",
"src_ip": "10.0.6.201",
"mask_src_ip": "255.255.0.0",
"src_port": "0",
"mask_src_port": "65535",
"dst_ip": "0.0.0.0",
"mask_dst_ip": "255.255.255.255",
"dst_port": "0",
"mask_dst_port": "65535",
"protocol": 6,
"direction": "double"
}
},
{
"table_name": "IP_CONFIG",
"table_type": "ip",
"table_content": {
"addr_type": "ipv6",
"src_ip": "2001:da8:205:1::101",
"mask_src_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff",
"src_port": "0",
"mask_src_port": "65535",
"dst_ip": "0::0",
"mask_dst_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff",
"dst_port": "0",
"mask_dst_port": "65535",
"protocol": 6,
"direction": "double"
}
}
]
},
{
"group_name": "group_2",
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "abckkk&123",
"expr_type": "and",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 124,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"effective_rage": 0,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "group_1"
},
{
"group_name": "group_3",
"regions": [
{
"table_name": "CONTENT_SIZE",
"table_type": "intval",
"table_content": {
"low_boundary": 100,
"up_boundary": 500
}
}
]
}
]
},
{
"compile_id": 125,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"effective_rage": 0,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "group_4",
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "action=search\\&query=(.*)",
"expr_type": "regex",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 126,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"effective_rage": 0,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "group_5",
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "should_not_hit_any_rule",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
},
{
"group_name": "group_6",
"regions": [
{
"table_name": "CONTENT_SIZE",
"table_type": "intval",
"table_content": {
"low_boundary": 2014,
"up_boundary": 2016
}
}
]
}
]
}
],
"plugin_table": [
{
"table_name": "QD_ENTRY_INFO",
"table_content": [
"1\t192.168.0.1\t101",
"2\t192.168.0.2\t101",
"3\t192.168.1.1\t102"
]
},
{
"table_name": "TEST_PLUGIN_TABLE",
"table_content": [
"1\t3388\t99\t1",
"2\t3355\t66\t1",
"3\tcccc\t11\t1"
]
}
]
}

2
test/maat_test.cpp
View File

@@ -219,7 +219,7 @@ int main()
if(fp!=NULL)
{
sp=Maat_stream_scan_digest_start(feather, digest_scan_table_id, digest_fstat.st_size, 0);
while(0!=feof(fp))
while(0==feof(fp))
{
read_size=fread(digest_test_buff,1,sizeof(digest_test_buff),fp);
ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,&mid);