Makefile中增加Maat_api.o，修复若干编译错误。

2015-11-12 18:28:58 +08:00
parent 6bac9a6fc8
commit 0c9449fd3f
5 changed files with 94 additions and 64 deletions
--- a/src/entry/Maat_api.cpp
+++ b/src/entry/Maat_api.cpp
@@ -1,9 +1,17 @@
 #include <stdlib.h>
 #include <stdio.h>
+#include <assert.h>
+#include <MESA/MESA_handle_logger.h>
+
 #include "rulescan.h"
 #include "UniversalBoolMatch.h"
 #include "Maat_rule.h"
 #include "Maat_rule_internal.h"
+#include "dynamic_array.h"
+#include "config_monitor.h"
+#include "map_str2int.h"
+#include "rulescan.h"
+#include "json2iris.h"

 struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type)
 {
@@ -35,6 +43,7 @@ inline void DEC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num)
 	scanner->ref_cnt[offset]--;
 	return;
 }
+
 //return 1 if insert a unique id
 //return 0 if id is duplicated
 //return -1 if set is full
@@ -648,11 +657,11 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id

 	region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
 	
-	INC_SCANNER_REF(my_scanner, int thread_num);
+	INC_SCANNER_REF(my_scanner,thread_num);
 	region_ret=rulescan_search(my_scanner->region, thread_num, &intval_scan_data, region_result, MAX_SCANNER_HIT_NUM);
 	if(region_ret<0)
 	{
-		DEC_SCANNER_REF(my_scanner, int thread_num);
+		DEC_SCANNER_REF(my_scanner, thread_num);
 		return -1;
 	}
 	else if(region_ret>0)
@@ -672,7 +681,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
 									result,compile_result,rule_num);
 	}
 	
-	DEC_SCANNER_REF(my_scanner, int thread_num);
+	DEC_SCANNER_REF(my_scanner,thread_num);
 	if(compile_ret==0&&region_ret>0)
 	{
 		return -2;
@@ -691,7 +700,6 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
 	scan_data_t ip_scan_data;
 	scan_result_t *region_result=NULL;
 	_compile_result_t compile_result[rule_num];
-	int table_cfg_num=0;
 	_Maat_table_info_t* p_table=NULL;
 	
 	struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
@@ -946,8 +954,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
 			if(sp->scan_buff!=NULL)
 			{
 				*detail_ret=fill_region_hit_detail(sp->scan_buff,_mid,
-												region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
-												hit_region_cnt,
+												region_result,hit_region_cnt,
 												compile_result,compile_ret,
 												hit_detail,detail_num);
 			}
@@ -1021,10 +1028,10 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
 	struct _Maat_scanner_t* scanner=NULL;

 	struct _Maat_table_info_t *p_table=NULL;
-	p_table=acqurie_table(_feather, table_id, thread_num, TABLE_TYPE_EXPR);
+	p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR);
 	if(p_table==NULL)
 	{
-		return -1;
+		return NULL;
 	}
 	struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1);
 	scanner=_feather->scanner;
@@ -1045,15 +1052,15 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,

 inline int REACH_QUERY_THRESH(unsigned long long total_len,unsigned long long acc_len,unsigned char* query_point,int point_size)
 {
-	const int QUERY_MIN_RATE=(3);	//30%
-	const int QUERY_MIN_LEN=(1024*1024*4);
+	const unsigned long long QUERY_MIN_RATE=(3);	//30%
+//	const unsigned long long QUERY_MIN_LEN=(1024*1024*4);
 //do query every 10 percent since 30%, e.g. 0.3/0.4/0.5/.../1.0
 	unsigned long long rate=(acc_len*10)/total_len;
 //	if(acc_len>QUERY_MIN_LEN)
 //	{
 //		return 1;
 //	}
-	assert(rate<point_size+QUERY_MIN_RATE);
+	assert(rate<(unsigned long long)(point_size+QUERY_MIN_RATE));
 	if(rate>=QUERY_MIN_RATE&&query_point[rate-QUERY_MIN_RATE]==0)
 	{
 		query_point[rate-QUERY_MIN_RATE]=1;
@@ -1072,9 +1079,9 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
 	unsigned long long digest_len=0;
 	char* digest_buff=NULL;
 	struct _scan_status_t* _mid=(struct _scan_status_t*)(*mid);
-	pthread_rwlock_t GIE_rwlock=&(sp->feather->scanner->digest_rwlock[sp->table_id]);
+	pthread_rwlock_t *GIE_rwlock=&(sp->feather->scanner->digest_rwlock[sp->table_id]);
 	sp->acc_scan_len+=fuzzy_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset);
-	do_query=REACH_QUERY_THRESH(sp->total_len, sp->acc_len, sp->query_point,8);
+	do_query=REACH_QUERY_THRESH(sp->total_len, sp->acc_scan_len, sp->query_point,8);
 	if(do_query==0)
 	{
 		return 0;
@@ -1088,7 +1095,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
 	fuzzy_digest(sp->fuzzy_hash_handle,digest_buff, digest_len);
 	if(0==pthread_rwlock_tryrdlock(GIE_rwlock))
 	{
-		hit_region_cnt=GIE_query(GIE_handle, sp->origin_len, digest_buff, query_result, MAX_SCANNER_HIT_NUM);
+		hit_region_cnt=GIE_query(GIE_handle, sp->total_len, digest_buff, query_result, MAX_SCANNER_HIT_NUM);
 		pthread_rwlock_unlock(GIE_rwlock);
 	}
 	free(digest_buff);
--- a/src/entry/Maat_rule.cpp
+++ b/src/entry/Maat_rule.cpp
@@ -17,12 +17,16 @@
 #include "json2iris.h"
 #include "dynamic_array.h"
 #include "config_monitor.h"
+
 #include "map_str2int.h"
 #include "rulescan.h"
 #include "UniversalBoolMatch.h"
+#include "mesa_fuzzy.h"
+#include "great_index_engine.h"

 int MAAT_FRAME_VERSION_1_2_20151103=1;
 const char *maat_module="MAAT Frame";
+
 const char* CHARSET_STRING[]={"CHARSET_NONE","GBK","BIG5","UNICODE","UTF-8"};

 int converHextoint(char srctmp)
@@ -214,15 +218,6 @@ int cnt_maskbits(struct in6_addr mask)
 	}
 	return bits_cnt;
 }
-inline void ipv6_ntoh(unsigned int *v6_addr)
-{
-	unsigned int i=0;
-	for(i=0;i<4;i++)
-	{
-		v6_addr[i]=ntohl(v6_addr[i]);
-	}
-	return;
-}
 int aligment_int_array_sum(int * array,int size)
 {	
 	int sum=0,i=0;
@@ -635,7 +630,35 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule)
 	op_expr->p_expr->rnum++;
 	return;
 }
+GIE_digest_t* create_digest_rule(int id,short op,unsigned long long origin_len,const char* digest,
+						short cfds_lvl,struct _Maat_group_rule_t* tag)
+{
+	GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1);
+	int digest_len=strlen(digest);
+	rule->id=id;
+	rule->operation=op;
+	rule->origin_len=origin_len;
+	if(digest!=NULL)
+	{
+		rule->fuzzy_hash=(char*)calloc(sizeof(char),digest_len+1);
+		memcpy(rule->fuzzy_hash,digest,digest_len);

+	}
+	rule->cfds_lvl=cfds_lvl;
+	rule->tag=(void*)tag;
+	return rule;
+}
+void destroy_digest_rule(GIE_digest_t*rule)
+{
+	if(rule->fuzzy_hash!=NULL)
+	{
+		free(rule->fuzzy_hash);
+		rule->fuzzy_hash=NULL;
+	}
+	free(rule);
+	rule=NULL;
+	return;
+}
 struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread_num,MESA_lqueue_head tomb)
 {
 	int i=0;
@@ -683,7 +706,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread
 	for(i=0;i<MAX_TABLE_NUM;i++)
 	{
 		scanner->digest_update_q[i]=MESA_lqueue_create(0,0);
-		pthread_rwlock_init(&(scanner->digest_rwlock),NULL);
+		pthread_rwlock_init(&(scanner->digest_rwlock[i]),NULL);
 	}
 	return scanner;
 }
@@ -740,14 +763,14 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner)
 			destroy_digest_rule(digest_rule);
 		}
 		MESA_lqueue_destroy(scanner->digest_update_q[i], lqueue_destroy_cb, NULL);
-		pthread_rwlock_destroy(&(scanner->digest_rwlock));
+		pthread_rwlock_destroy(&(scanner->digest_rwlock[i]));
 	}
 	free(scanner);
 	return;
 }


-inline unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge)
+unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge)
 {
 	unsigned int sub_type=0;
 	if(do_charset_merge==TRUE)
@@ -1938,35 +1961,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line
 	
 	return;	
 }
-GIE_digest_t* create_digest_rule(int id,short op,unsigned long long origin_len,const char* digest,
-						short cfds_lvl,struct _Maat_group_rule_t* tag)
-{
-	GIE_digest_t* rule=calloc(sizeof(struct GIE_digest_t),1);
-	int digest_len=strlen(digest);
-	rule->id=id;
-	rule->operation=op;
-	rule->origin_len=origin_len;
-	if(digest!=NULL)
-	{
-		rule->fuzzy_hash=calloc(sizeof(char),digest_len+1);
-		memcpy(rule->fuzzy_hash,digest,digest_len);

-	}
-	rule->cfds_lvl=cfds_lvl;
-	rule->tag=(void*)tag;
-	return rule;
-}
-void destroy_digest_rule(GIE_digest_t*rule)
-{
-	if(rule->fuzzy_hash!=NULL)
-	{
-		free(rule->fuzzy_hash);
-		rule->fuzzy_hash=NULL;
-	}
-	free(rule);
-	rule=NULL;
-	return;
-}
 void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
 {
 	struct db_digest_rule_t* digest_rule=(struct db_digest_rule_t*)calloc(sizeof(struct db_digest_rule_t),1);
@@ -2018,7 +2013,7 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,
 		if(ret<0)
 		{
 			MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
-							"duplicate config of intval table %s config_id=%d",table->table_name,intval_rule->region_id);
+							"duplicate config of intval table %s config_id=%d",table->table_name,digest_rule->region_id);
 		}
 		else
 		{
@@ -2129,8 +2124,8 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_
 	int i=0;
 	long q_cnt;
 	GIE_create_para_t para;
-	para->index_interval=100;
-	para->query_accuracy=0.1;
+	para.index_interval=100;
+	para.query_accuracy=0.1;
 	tmp1=create_bool_matcher(scanner->compile_hash,
 											scan_thread_num,
 											logger);
@@ -2155,9 +2150,9 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_
 		pthread_rwlock_wrlock(&(scanner->digest_rwlock[i]));
 		if(scanner->digest_handle[i]==NULL)
 		{
-			scanner->digest_handle[i]=GIE_create(&para)
+			scanner->digest_handle[i]=GIE_create(&para);
 		}
-		digest_batch_update(scanner->digest_handle[i], scanner->digest_update_q[i],logger)
+		digest_batch_update(scanner->digest_handle[i], scanner->digest_update_q[i],logger);
 		pthread_rwlock_unlock(&(scanner->digest_rwlock[i]));
 	}
 	scanner->last_update_time=time(NULL);
--- a/src/entry/Maat_rule_internal.h
+++ b/src/entry/Maat_rule_internal.h
@@ -1,3 +1,6 @@
+#ifndef H_MAAT_RULE_INTERNAL_H_INCLUDE
+#define H_MAAT_RULE_INTERNAL_H_INCLUDE
+
 #include "Maat_rule.h"

 #include <MESA/MESA_htable.h>
@@ -6,9 +9,15 @@
 #include "UniversalBoolMatch.h"
 #include "rulescan.h"

+#include "mesa_fuzzy.h"
+#include "great_index_engine.h"
+
 #include <pthread.h>
 #include <iconv.h>

+extern const char *maat_module;
+
+
 #if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411)
 #define atomic_inc(x) __sync_add_and_fetch((x),1)
 #define atomic_dec(x) __sync_sub_and_fetch((x),1)
@@ -311,4 +320,21 @@ struct _maat_garbage_t
 };
 void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
 void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
+void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bool_expr_t* a_set);
+int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path);
+void maat_start_cb(unsigned int new_version,int update_type,void*u_para);
+void maat_update_cb(const char* table_name,const char* line,void *u_para);
+void maat_finish_cb(void* u_para);
+void *thread_rule_monitor(void *arg);
+unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge);
+inline void ipv6_ntoh(unsigned int *v6_addr)
+{
+	unsigned int i=0;
+	for(i=0;i<4;i++)
+	{
+		v6_addr[i]=ntohl(v6_addr[i]);
+	}
+	return;
+}
+#endif

--- a/src/entry/Makefile
+++ b/src/entry/Makefile
@@ -13,7 +13,7 @@ H_DIR			=-I$(G_H_DIR) -I../../inc
 LIBMAAT = libmaatframe.a
 LIBMAAT_SO = libmaatframe.so 

-OBJS=config_monitor.o Maat_rule.o UniversalBoolMatch.o dynamic_array.o cJSON.o json2iris.o map_str2int.o
+OBJS=config_monitor.o Maat_rule.o Maat_api.o UniversalBoolMatch.o dynamic_array.o cJSON.o json2iris.o map_str2int.o
 .c.o:
 	$(CC) -c $(CFLAGS) -I. $(H_DIR) $<

--- a/test/maat_test.cpp
+++ b/test/maat_test.cpp
@@ -6,11 +6,13 @@
 #include <netinet/in.h>//inet_addr
 #include <arpa/inet.h>//inet_addr
 #include <net/if.h>
-#include <sys/types.h>
+#include <sys/types.h>//fstat
 #include <sys/ioctl.h>
 #include <string.h>
 #include <stdio.h>
 #include <MESA/stream.h>
+#include <sys/types.h>//fstat
+#include <sys/stat.h>//fstat

 void Maat_read_entry_start_cb(int update_type,void* u_para)
 {
@@ -207,7 +209,7 @@ int main()
 		printf("registe table FILE_DIGEST error.\n");
 		return 0;
 	}
-	ret=fstat(digest_test_file,&digest_fstat);
+	ret=stat(digest_test_file,&digest_fstat);
 	if(ret!=0)
 	{
 		printf("fstat %s error.\n",digest_test_file);
@@ -216,10 +218,10 @@ int main()
 	FILE* fp=fopen(digest_test_file,"r");
 	if(fp!=NULL)
 	{
-		sp=Maat_stream_scan_digest_start(feather, digest_scan_table_id, digest_fstat->off_t, 0);
+		sp=Maat_stream_scan_digest_start(feather, digest_scan_table_id, digest_fstat.st_size, 0);
 		while(0!=feof(fp))
 		{
-			read_size=fread(fp,digest_test_buff,sizeof(digest_test_buff));
+			read_size=fread(digest_test_buff,1,sizeof(digest_test_buff),fp);
 			ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,&mid);
 			scan_offset+=read_size;
 			if(ret>0)