gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Makefile中增加Maat_api.o，修复若干编译错误。

Browse Source
This commit is contained in:
zhengchao
2015-11-12 18:28:58 +08:00
parent 6bac9a6fc8
commit 0c9449fd3f
5 changed files with 94 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/entry/Maat_api.cpp
View File

@@ -1,9 +1,17 @@
#include <stdlib.h> #include <stdlib.h>
#include <stdio.h> #include <stdio.h>
#include <assert.h>
#include <MESA/MESA_handle_logger.h>
#include "rulescan.h" #include "rulescan.h"
#include "UniversalBoolMatch.h" #include "UniversalBoolMatch.h"
#include "Maat_rule.h" #include "Maat_rule.h"
#include "Maat_rule_internal.h" #include "Maat_rule_internal.h"
#include "dynamic_array.h"
#include "config_monitor.h"
#include "map_str2int.h"
#include "rulescan.h"
#include "json2iris.h"
struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type) struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type)
{ {
@@ -35,6 +43,7 @@ inline void DEC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num)
scanner->ref_cnt[offset]--; scanner->ref_cnt[offset]--;
return; return;
} }
//return 1 if insert a unique id //return 1 if insert a unique id
//return 0 if id is duplicated //return 0 if id is duplicated
//return -1 if set is full //return -1 if set is full
@@ -648,11 +657,11 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
INC_SCANNER_REF(my_scanner, int thread_num); INC_SCANNER_REF(my_scanner,thread_num);
region_ret=rulescan_search(my_scanner->region, thread_num, &intval_scan_data, region_result, MAX_SCANNER_HIT_NUM); region_ret=rulescan_search(my_scanner->region, thread_num, &intval_scan_data, region_result, MAX_SCANNER_HIT_NUM);
if(region_ret<0) if(region_ret<0)
{ {
DEC_SCANNER_REF(my_scanner, int thread_num); DEC_SCANNER_REF(my_scanner, thread_num);
return -1; return -1;
} }
else if(region_ret>0) else if(region_ret>0)
@@ -672,7 +681,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
result,compile_result,rule_num); result,compile_result,rule_num);
} }
DEC_SCANNER_REF(my_scanner, int thread_num); DEC_SCANNER_REF(my_scanner,thread_num);
if(compile_ret==0&&region_ret>0) if(compile_ret==0&&region_ret>0)
{ {
return -2; return -2;
@@ -691,7 +700,6 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
scan_data_t ip_scan_data; scan_data_t ip_scan_data;
scan_result_t *region_result=NULL; scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num]; _compile_result_t compile_result[rule_num];
int table_cfg_num=0;
_Maat_table_info_t* p_table=NULL; _Maat_table_info_t* p_table=NULL;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
@@ -946,8 +954,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
if(sp->scan_buff!=NULL) if(sp->scan_buff!=NULL)
{ {
*detail_ret=fill_region_hit_detail(sp->scan_buff,_mid, *detail_ret=fill_region_hit_detail(sp->scan_buff,_mid,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), region_result,hit_region_cnt,
hit_region_cnt,
compile_result,compile_ret, compile_result,compile_ret,
hit_detail,detail_num); hit_detail,detail_num);
} }
@@ -1021,10 +1028,10 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
struct _Maat_scanner_t* scanner=NULL; struct _Maat_scanner_t* scanner=NULL;
struct _Maat_table_info_t *p_table=NULL; struct _Maat_table_info_t *p_table=NULL;
p_table=acqurie_table(_feather, table_id, thread_num, TABLE_TYPE_EXPR); p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR);
if(p_table==NULL) if(p_table==NULL)
{ {
return -1; return NULL;
} }
struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1); struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1);
scanner=_feather->scanner; scanner=_feather->scanner;
@@ -1045,15 +1052,15 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
inline int REACH_QUERY_THRESH(unsigned long long total_len,unsigned long long acc_len,unsigned char* query_point,int point_size) inline int REACH_QUERY_THRESH(unsigned long long total_len,unsigned long long acc_len,unsigned char* query_point,int point_size)
{ {
const int QUERY_MIN_RATE=(3); //30% const unsigned long long QUERY_MIN_RATE=(3); //30%
const int QUERY_MIN_LEN=(1024*1024*4); // const unsigned long long QUERY_MIN_LEN=(1024*1024*4);
//do query every 10 percent since 30%, e.g. 0.3/0.4/0.5/.../1.0 //do query every 10 percent since 30%, e.g. 0.3/0.4/0.5/.../1.0
unsigned long long rate=(acc_len*10)/total_len; unsigned long long rate=(acc_len*10)/total_len;
// if(acc_len>QUERY_MIN_LEN) // if(acc_len>QUERY_MIN_LEN)
// { // {
// return 1; // return 1;
// } // }
assert(rate<point_size+QUERY_MIN_RATE); assert(rate<(unsigned long long)(point_size+QUERY_MIN_RATE));
if(rate>=QUERY_MIN_RATE&&query_point[rate-QUERY_MIN_RATE]==0) if(rate>=QUERY_MIN_RATE&&query_point[rate-QUERY_MIN_RATE]==0)
{ {
query_point[rate-QUERY_MIN_RATE]=1; query_point[rate-QUERY_MIN_RATE]=1;
@@ -1072,9 +1079,9 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
unsigned long long digest_len=0; unsigned long long digest_len=0;
char* digest_buff=NULL; char* digest_buff=NULL;
struct _scan_status_t* _mid=(struct _scan_status_t*)(*mid); struct _scan_status_t* _mid=(struct _scan_status_t*)(*mid);
pthread_rwlock_t GIE_rwlock=&(sp->feather->scanner->digest_rwlock[sp->table_id]); pthread_rwlock_t *GIE_rwlock=&(sp->feather->scanner->digest_rwlock[sp->table_id]);
sp->acc_scan_len+=fuzzy_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset); sp->acc_scan_len+=fuzzy_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset);
do_query=REACH_QUERY_THRESH(sp->total_len, sp->acc_len, sp->query_point,8); do_query=REACH_QUERY_THRESH(sp->total_len, sp->acc_scan_len, sp->query_point,8);
if(do_query==0) if(do_query==0)
{ {
return 0; return 0;
@@ -1088,7 +1095,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
fuzzy_digest(sp->fuzzy_hash_handle,digest_buff, digest_len); fuzzy_digest(sp->fuzzy_hash_handle,digest_buff, digest_len);
if(0==pthread_rwlock_tryrdlock(GIE_rwlock)) if(0==pthread_rwlock_tryrdlock(GIE_rwlock))
{ {
hit_region_cnt=GIE_query(GIE_handle, sp->origin_len, digest_buff, query_result, MAX_SCANNER_HIT_NUM); hit_region_cnt=GIE_query(GIE_handle, sp->total_len, digest_buff, query_result, MAX_SCANNER_HIT_NUM);
pthread_rwlock_unlock(GIE_rwlock); pthread_rwlock_unlock(GIE_rwlock);
} }
free(digest_buff); free(digest_buff);

85
src/entry/Maat_rule.cpp
View File

@@ -17,12 +17,16 @@
#include "json2iris.h" #include "json2iris.h"
#include "dynamic_array.h" #include "dynamic_array.h"
#include "config_monitor.h" #include "config_monitor.h"
#include "map_str2int.h" #include "map_str2int.h"
#include "rulescan.h" #include "rulescan.h"
#include "UniversalBoolMatch.h" #include "UniversalBoolMatch.h"
#include "mesa_fuzzy.h"
#include "great_index_engine.h"
int MAAT_FRAME_VERSION_1_2_20151103=1; int MAAT_FRAME_VERSION_1_2_20151103=1;
const char *maat_module="MAAT Frame"; const char *maat_module="MAAT Frame";
const char* CHARSET_STRING[]={"CHARSET_NONE","GBK","BIG5","UNICODE","UTF-8"}; const char* CHARSET_STRING[]={"CHARSET_NONE","GBK","BIG5","UNICODE","UTF-8"};
int converHextoint(char srctmp) int converHextoint(char srctmp)
@@ -214,15 +218,6 @@ int cnt_maskbits(struct in6_addr mask)
} }
return bits_cnt; return bits_cnt;
} }
inline void ipv6_ntoh(unsigned int *v6_addr)
{
unsigned int i=0;
for(i=0;i<4;i++)
{
v6_addr[i]=ntohl(v6_addr[i]);
}
return;
}
int aligment_int_array_sum(int * array,int size) int aligment_int_array_sum(int * array,int size)
{ {
int sum=0,i=0; int sum=0,i=0;
@@ -635,7 +630,35 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule)
op_expr->p_expr->rnum++; op_expr->p_expr->rnum++;
return; return;
} }
GIE_digest_t* create_digest_rule(int id,short op,unsigned long long origin_len,const char* digest,
short cfds_lvl,struct _Maat_group_rule_t* tag)
{
GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1);
int digest_len=strlen(digest);
rule->id=id;
rule->operation=op;
rule->origin_len=origin_len;
if(digest!=NULL)
{
rule->fuzzy_hash=(char*)calloc(sizeof(char),digest_len+1);
memcpy(rule->fuzzy_hash,digest,digest_len);
}
rule->cfds_lvl=cfds_lvl;
rule->tag=(void*)tag;
return rule;
}
void destroy_digest_rule(GIE_digest_t*rule)
{
if(rule->fuzzy_hash!=NULL)
{
free(rule->fuzzy_hash);
rule->fuzzy_hash=NULL;
}
free(rule);
rule=NULL;
return;
}
struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread_num,MESA_lqueue_head tomb) struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread_num,MESA_lqueue_head tomb)
{ {
int i=0; int i=0;
@@ -683,7 +706,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread
for(i=0;i<MAX_TABLE_NUM;i++) for(i=0;i<MAX_TABLE_NUM;i++)
{ {
scanner->digest_update_q[i]=MESA_lqueue_create(0,0); scanner->digest_update_q[i]=MESA_lqueue_create(0,0);
pthread_rwlock_init(&(scanner->digest_rwlock),NULL); pthread_rwlock_init(&(scanner->digest_rwlock[i]),NULL);
} }
return scanner; return scanner;
} }
@@ -740,14 +763,14 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner)
destroy_digest_rule(digest_rule); destroy_digest_rule(digest_rule);
} }
MESA_lqueue_destroy(scanner->digest_update_q[i], lqueue_destroy_cb, NULL); MESA_lqueue_destroy(scanner->digest_update_q[i], lqueue_destroy_cb, NULL);
pthread_rwlock_destroy(&(scanner->digest_rwlock)); pthread_rwlock_destroy(&(scanner->digest_rwlock[i]));
} }
free(scanner); free(scanner);
return; return;
} }
inline unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge) unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge)
{ {
unsigned int sub_type=0; unsigned int sub_type=0;
if(do_charset_merge==TRUE) if(do_charset_merge==TRUE)
@@ -1938,35 +1961,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line
return; return;
} }
GIE_digest_t* create_digest_rule(int id,short op,unsigned long long origin_len,const char* digest,
short cfds_lvl,struct _Maat_group_rule_t* tag)
{
GIE_digest_t* rule=calloc(sizeof(struct GIE_digest_t),1);
int digest_len=strlen(digest);
rule->id=id;
rule->operation=op;
rule->origin_len=origin_len;
if(digest!=NULL)
{
rule->fuzzy_hash=calloc(sizeof(char),digest_len+1);
memcpy(rule->fuzzy_hash,digest,digest_len);
}
rule->cfds_lvl=cfds_lvl;
rule->tag=(void*)tag;
return rule;
}
void destroy_digest_rule(GIE_digest_t*rule)
{
if(rule->fuzzy_hash!=NULL)
{
free(rule->fuzzy_hash);
rule->fuzzy_hash=NULL;
}
free(rule);
rule=NULL;
return;
}
void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
{ {
struct db_digest_rule_t* digest_rule=(struct db_digest_rule_t*)calloc(sizeof(struct db_digest_rule_t),1); struct db_digest_rule_t* digest_rule=(struct db_digest_rule_t*)calloc(sizeof(struct db_digest_rule_t),1);
@@ -2018,7 +2013,7 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,
if(ret<0) if(ret<0)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"duplicate config of intval table %s config_id=%d",table->table_name,intval_rule->region_id); "duplicate config of intval table %s config_id=%d",table->table_name,digest_rule->region_id);
} }
else else
{ {
@@ -2129,8 +2124,8 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_
int i=0; int i=0;
long q_cnt; long q_cnt;
GIE_create_para_t para; GIE_create_para_t para;
para->index_interval=100; para.index_interval=100;
para->query_accuracy=0.1; para.query_accuracy=0.1;
tmp1=create_bool_matcher(scanner->compile_hash, tmp1=create_bool_matcher(scanner->compile_hash,
scan_thread_num, scan_thread_num,
logger); logger);
@@ -2155,9 +2150,9 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_
pthread_rwlock_wrlock(&(scanner->digest_rwlock[i])); pthread_rwlock_wrlock(&(scanner->digest_rwlock[i]));
if(scanner->digest_handle[i]==NULL) if(scanner->digest_handle[i]==NULL)
{ {
scanner->digest_handle[i]=GIE_create(&para) scanner->digest_handle[i]=GIE_create(&para);
} }
digest_batch_update(scanner->digest_handle[i], scanner->digest_update_q[i],logger) digest_batch_update(scanner->digest_handle[i], scanner->digest_update_q[i],logger);
pthread_rwlock_unlock(&(scanner->digest_rwlock[i])); pthread_rwlock_unlock(&(scanner->digest_rwlock[i]));
} }
scanner->last_update_time=time(NULL); scanner->last_update_time=time(NULL);

26
src/entry/Maat_rule_internal.h
View File

@@ -1,3 +1,6 @@
#ifndef H_MAAT_RULE_INTERNAL_H_INCLUDE
#define H_MAAT_RULE_INTERNAL_H_INCLUDE
#include "Maat_rule.h" #include "Maat_rule.h"
#include <MESA/MESA_htable.h> #include <MESA/MESA_htable.h>
@@ -6,9 +9,15 @@
#include "UniversalBoolMatch.h" #include "UniversalBoolMatch.h"
#include "rulescan.h" #include "rulescan.h"
#include "mesa_fuzzy.h"
#include "great_index_engine.h"
#include <pthread.h> #include <pthread.h>
#include <iconv.h> #include <iconv.h>
extern const char *maat_module;
#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411) #if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411)
#define atomic_inc(x) __sync_add_and_fetch((x),1) #define atomic_inc(x) __sync_add_and_fetch((x),1)
#define atomic_dec(x) __sync_sub_and_fetch((x),1) #define atomic_dec(x) __sync_sub_and_fetch((x),1)
@@ -311,4 +320,21 @@ struct _maat_garbage_t
}; };
void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bool_expr_t* a_set);
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path);
void maat_start_cb(unsigned int new_version,int update_type,void*u_para);
void maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para);
void *thread_rule_monitor(void *arg);
unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge);
inline void ipv6_ntoh(unsigned int *v6_addr)
{
unsigned int i=0;
for(i=0;i<4;i++)
{
v6_addr[i]=ntohl(v6_addr[i]);
}
return;
}
#endif

2
src/entry/Makefile
View File

@@ -13,7 +13,7 @@ H_DIR =-I$(G_H_DIR) -I../../inc
LIBMAAT = libmaatframe.a LIBMAAT = libmaatframe.a
LIBMAAT_SO = libmaatframe.so LIBMAAT_SO = libmaatframe.so
OBJS=config_monitor.o Maat_rule.o UniversalBoolMatch.o dynamic_array.o cJSON.o json2iris.o map_str2int.o OBJS=config_monitor.o Maat_rule.o Maat_api.o UniversalBoolMatch.o dynamic_array.o cJSON.o json2iris.o map_str2int.o
.c.o: .c.o:
$(CC) -c $(CFLAGS) -I. $(H_DIR) $< $(CC) -c $(CFLAGS) -I. $(H_DIR) $<

10
test/maat_test.cpp
View File

@@ -6,11 +6,13 @@
#include <netinet/in.h>//inet_addr #include <netinet/in.h>//inet_addr
#include <arpa/inet.h>//inet_addr #include <arpa/inet.h>//inet_addr
#include <net/if.h> #include <net/if.h>
#include <sys/types.h> #include <sys/types.h>//fstat
#include <sys/ioctl.h> #include <sys/ioctl.h>
#include <string.h> #include <string.h>
#include <stdio.h> #include <stdio.h>
#include <MESA/stream.h> #include <MESA/stream.h>
#include <sys/types.h>//fstat
#include <sys/stat.h>//fstat
void Maat_read_entry_start_cb(int update_type,void* u_para) void Maat_read_entry_start_cb(int update_type,void* u_para)
{ {
@@ -207,7 +209,7 @@ int main()
printf("registe table FILE_DIGEST error.\n"); printf("registe table FILE_DIGEST error.\n");
return 0; return 0;
} }
ret=fstat(digest_test_file,&digest_fstat); ret=stat(digest_test_file,&digest_fstat);
if(ret!=0) if(ret!=0)
{ {
printf("fstat %s error.\n",digest_test_file); printf("fstat %s error.\n",digest_test_file);
@@ -216,10 +218,10 @@ int main()
FILE* fp=fopen(digest_test_file,"r"); FILE* fp=fopen(digest_test_file,"r");
if(fp!=NULL) if(fp!=NULL)
{ {
sp=Maat_stream_scan_digest_start(feather, digest_scan_table_id, digest_fstat->off_t, 0); sp=Maat_stream_scan_digest_start(feather, digest_scan_table_id, digest_fstat.st_size, 0);
while(0!=feof(fp)) while(0!=feof(fp))
{ {
read_size=fread(fp,digest_test_buff,sizeof(digest_test_buff)); read_size=fread(digest_test_buff,1,sizeof(digest_test_buff),fp);
ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,&mid); ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,&mid);
scan_offset+=read_size; scan_offset+=read_size;
if(ret>0) if(ret>0)