gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add runtime statistic feature.

Browse Source
This commit is contained in:
zhengchao
2016-02-10 10:01:18 +08:00
parent 57dd773c71
commit 2a9bbc3c81
9 changed files with 628 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
inc/Maat_rule.h
View File

@@ -108,17 +108,36 @@ struct Maat_hit_detail_t
struct Maat_region_pos_t region_pos[MAAT_MAX_HIT_RULE_NUM]; struct Maat_region_pos_t region_pos[MAAT_MAX_HIT_RULE_NUM];
}; };
//--------------------HITTING DETAIL DESCRIPTION END //--------------------HITTING DETAIL DESCRIPTION END
//Abondon interface ,left for compatible.
Maat_feather_t Maat_summon_feather(int max_thread_num, Maat_feather_t Maat_summon_feather(int max_thread_num,
const char* table_info_path, const char* table_info_path,
const char* ful_cfg_dir, const char* ful_cfg_dir,
const char* inc_cfg_dir, const char* inc_cfg_dir,
void*logger); void*logger);//MESA_handle_logger
//Abondon interface ,left for compatible.
Maat_feather_t Maat_summon_feather_json(int max_thread_num, Maat_feather_t Maat_summon_feather_json(int max_thread_num,
const char* table_info_path, const char* table_info_path,
const char* json_rule, const char* json_rule,
void* logger); void* logger);
Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger);
int Maat_initiate_feather(Maat_feather_t feather);
enum MAAT_INIT_OPT{
MAAT_OPT_SCANDIR_INTERVAL_MS=1, //VALUE is interger,SIZE=sizeof(int). DEFAULT:1,000 milliseconds.
MAAT_OPT_EFFECT_INVERVAL_MS, //VALUE is interger,SIZE=sizeof(int). DEFAULT:60,000 milliseconds.
MAAT_OPT_FULL_CFG_DIR, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default.
MAAT_OPT_INC_CFG_DIR, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default.
MAAT_OPT_JSON_FILE_PATH, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default.
MAAT_OPT_STAT_ON, //VALUE is indifferent,SIZE is indifferent.MAAT_OPT_STAT_FILE_PATH must be set.Default: stat OFF.
MAAT_OPT_PERF_ON, //VALUE is indifferent,SIZE is indifferent.MAAT_OPT_STAT_FILE_PATH must be set.Default: stat OFF.
MAAT_OPT_STAT_FILE_PATH, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default.
MAAT_OPT_SCAN_DETAIL //VALUE is interger,SIZE=sizeof(int). 0: not return any detail;1: return hit pos, not include regex grouping;
// 2 return hit pos and regex grouping pos;DEFAULT:2
};
//return -1 if failed, return 0 on success;
int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size);
void Maat_burn_feather(Maat_feather_t feather); void Maat_burn_feather(Maat_feather_t feather);
//return table_id(>=0) if success,otherwise return -1; //return table_id(>=0) if success,otherwise return -1;
@@ -179,10 +198,5 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para);
void Maat_clean_status(scan_status_t* mid); void Maat_clean_status(scan_status_t* mid);
#define MAAT_OPT_SCANDIR_INTERVAL_MS 1 //opt value is interger,size=sizeof(int),1,000 milliseconds as default
#define MAAT_OPT_EFFECT_INVERVAL_MS 2 //opt value is interger,size=sizeof(int),60,000 milliseconds as default
//return -1 if failed, return 0 on success;
int Maat_set_feather_opt(Maat_feather_t feather,int type,void* value,int size);
#endif // H_MAAT_RULE_H_INCLUDE #endif // H_MAAT_RULE_H_INCLUDE

324
src/entry/Maat_api.cpp
View File

@@ -1,6 +1,7 @@
#include <stdlib.h> #include <stdlib.h>
#include <stdio.h> #include <stdio.h>
#include <assert.h> #include <assert.h>
#include <sys/time.h>
#include <MESA/MESA_handle_logger.h> #include <MESA/MESA_handle_logger.h>
#include "rulescan.h" #include "rulescan.h"
@@ -101,7 +102,6 @@ int region_compile(struct _scan_status_t *_mid,void* region_hit,int region_type_
struct _Maat_compile_rule_t* array_mi_rule[MAX_SCANNER_HIT_NUM]; struct _Maat_compile_rule_t* array_mi_rule[MAX_SCANNER_HIT_NUM];
struct _Maat_compile_rule_t* _mi_rule=NULL; struct _Maat_compile_rule_t* _mi_rule=NULL;
int region_pos[MAX_SCANNER_HIT_NUM]; int region_pos[MAX_SCANNER_HIT_NUM];
_mid->cur_hit_cnt=0; _mid->cur_hit_cnt=0;
for(i=0;i<region_hit_num;i++) for(i=0;i<region_hit_num;i++)
{ {
@@ -157,10 +157,12 @@ int region_compile(struct _scan_status_t *_mid,void* region_hit,int region_type_
result_cnt++; result_cnt++;
} }
pthread_rwlock_unlock(&(_mi_rule->rwlock)); pthread_rwlock_unlock(&(_mi_rule->rwlock));
}
}
} }
if(result_cnt>0)
{
_mid->feather->hit_cnt++;
}
return result_cnt; return result_cnt;
} }
@@ -320,6 +322,7 @@ struct _scan_status_t* _Maat_make_status(struct _Maat_feather_t* feather,int thr
_mid->hit_group_cnt=0; _mid->hit_group_cnt=0;
_mid->hit_group_size=4; _mid->hit_group_size=4;
_mid->hitted_group_id=(unsigned int*)malloc(sizeof(unsigned int)*_mid->hit_group_size); _mid->hitted_group_id=(unsigned int*)malloc(sizeof(unsigned int)*_mid->hit_group_size);
feather->mid_cnt++;
return _mid; return _mid;
} }
@@ -350,14 +353,8 @@ int detain_last_data(char* buff,int buff_size,int detained_len,const char* data,
return ret_len; return ret_len;
} }
Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger)
Maat_feather_t Maat_summon_feather(int max_thread_num,
const char* table_info_path,
const char* ful_cfg_dir,
const char* inc_cfg_dir,
void* logger)
{ {
_Maat_feather_t* feather=(_Maat_feather_t*)calloc(sizeof(struct _Maat_feather_t),1); _Maat_feather_t* feather=(_Maat_feather_t*)calloc(sizeof(struct _Maat_feather_t),1);
feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path,logger); feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path,logger);
feather->map_tablename2id=map_create(); feather->map_tablename2id=map_create();
@@ -373,65 +370,25 @@ Maat_feather_t Maat_summon_feather(int max_thread_num,
map_register(feather->map_tablename2id,feather->p_table_info[i]->table_name,feather->p_table_info[i]->table_id); map_register(feather->map_tablename2id,feather->p_table_info[i]->table_name,feather->p_table_info[i]->table_id);
} }
} }
memcpy(feather->inc_dir,inc_cfg_dir,strlen(inc_cfg_dir));
memcpy(feather->full_dir,ful_cfg_dir,strlen(ful_cfg_dir));
feather->logger=logger; feather->logger=logger;
feather->scan_thread_num=max_thread_num; feather->scan_thread_num=max_thread_num;
feather->garbage_q=MESA_lqueue_create(0,0); feather->garbage_q=MESA_lqueue_create(0,0);
config_monitor_traverse(feather->maat_version,
ful_cfg_dir,
maat_start_cb,
maat_update_cb,
maat_finish_cb,
feather,
logger);
if(feather->update_tmp_scanner==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"At initiation: no valid index file in %s",ful_cfg_dir);
}
feather->scanner=feather->update_tmp_scanner;
feather->update_tmp_scanner=NULL;
feather->still_working=1;
if(feather->scanner!=NULL)
{
feather->maat_version=feather->scanner->version;
}
feather->effect_interval_ms=60*1000; feather->effect_interval_ms=60*1000;
feather->scan_interval_ms=1*1000; feather->scan_interval_ms=1*1000;
pthread_t cfg_mon_t;
pthread_create(&cfg_mon_t, NULL, thread_rule_monitor, (void*)feather);
return feather; return feather;
} }
Maat_feather_t Maat_summon_feather_json(int max_thread_num, int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size)
const char* table_info_path,
const char* json_rule,
void* logger)
{
Maat_feather_t feather;
char full_index_dir[256]={0};
int ret=-1;
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"Maat initial with JSON file %s.",json_rule);
ret=json2iris(json_rule, full_index_dir,sizeof(full_index_dir),logger);
if(ret<0)
{
return NULL;
}
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"generate index file %s OK.",full_index_dir);
feather=Maat_summon_feather(max_thread_num,table_info_path, full_index_dir, full_index_dir,logger);
return feather;
}
int Maat_set_feather_opt(Maat_feather_t feather,int type,void* value,int size)
{ {
_Maat_feather_t* _feather=(_Maat_feather_t*)feather; _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int intval=0; int intval=0,ret=-1;
if(_feather->still_working==1)// not allowed set after Maat_initiate_feather;
{
return -2;
}
switch(type) switch(type)
{ {
case MAAT_OPT_EFFECT_INVERVAL_MS: case MAAT_OPT_EFFECT_INVERVAL_MS:
intval=*(int*)value; intval=*(const int*)value;
if(size!=sizeof(int)||intval<=0) if(size!=sizeof(int)||intval<=0)
{ {
return -1; return -1;
@@ -439,18 +396,168 @@ int Maat_set_feather_opt(Maat_feather_t feather,int type,void* value,int size)
_feather->effect_interval_ms=intval; _feather->effect_interval_ms=intval;
break; break;
case MAAT_OPT_SCANDIR_INTERVAL_MS: case MAAT_OPT_SCANDIR_INTERVAL_MS:
intval=*(int*)value; intval=*(const int*)value;
if(size!=sizeof(int)||intval<0) if(size!=sizeof(int)||intval<0)
{ {
return -1; return -1;
} }
_feather->scan_interval_ms=intval; _feather->scan_interval_ms=intval;
break; break;
case MAAT_OPT_FULL_CFG_DIR:
if(size>(int)sizeof(_feather->full_dir))
{
return -1;
}
memcpy(_feather->full_dir,(const char*)value,size);
break;
case MAAT_OPT_INC_CFG_DIR:
if(size>(int)sizeof(_feather->inc_dir))
{
return -1;
}
memcpy(_feather->inc_dir,(const char*)value,size);
break;
case MAAT_OPT_JSON_FILE_PATH:
ret=json2iris((const char*)value, _feather->full_dir,sizeof(_feather->full_dir),_feather->logger);
if(ret<0)
{
return -1;
}
memcpy(_feather->inc_dir,_feather->full_dir,sizeof(_feather->inc_dir));
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"Maat initial with JSON file %s,generate index file %s OK."
,(const char*)value
,_feather->full_dir);
break;
case MAAT_OPT_STAT_ON:
_feather->stat_on=1;
_feather->stat_handle=FS_create_handle();
break;
case MAAT_OPT_PERF_ON:
_feather->perf_on=1;
break;
case MAAT_OPT_STAT_FILE_PATH:
if(size>(int)sizeof(_feather->stat_file))
{
return -1;
}
memcpy(_feather->stat_file,(const char*)value,size);
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"Maat performance statistic output to %s."
,(const char*)value);
_feather->stat_on=1;
break;
case MAAT_OPT_SCAN_DETAIL:
intval=*(const int*)value;
_feather->rule_scan_type=intval;
break;
default: default:
return -1; return -1;
} }
return 0; return 0;
} }
int Maat_initiate_feather(Maat_feather_t feather)
{
_Maat_feather_t* _feather=(_Maat_feather_t*)feather;
config_monitor_traverse(_feather->maat_version,
_feather->full_dir,
maat_start_cb,
maat_update_cb,
maat_finish_cb,
_feather,
_feather->logger);
if(_feather->update_tmp_scanner==NULL)
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module ,
"At initiation: no valid index file in %s",_feather->full_dir);
}
_feather->scanner=_feather->update_tmp_scanner;
_feather->update_tmp_scanner=NULL;
_feather->still_working=1;
_feather->rule_scan_type=2;
if(_feather->scanner!=NULL)
{
_feather->maat_version=_feather->scanner->version;
}
if(strlen(_feather->stat_file)==0)
{
_feather->perf_on=0;
_feather->stat_on=0;
}
maat_stat_init(_feather);
pthread_t cfg_mon_t;
pthread_create(&cfg_mon_t, NULL, thread_rule_monitor, (void*)_feather);
return 0;
}
Maat_feather_t Maat_summon_feather(int max_thread_num,
const char* table_info_path,
const char* ful_cfg_dir,
const char* inc_cfg_dir,
void* logger)
{
int ret=-1;
Maat_feather_t feather=NULL;
feather=Maat_feather(max_thread_num,table_info_path,logger);
if(feather==NULL)
{
return NULL;
}
ret=Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, ful_cfg_dir, strlen(ful_cfg_dir)+1);
if(ret<0)
{
goto error_out;
}
ret=Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, inc_cfg_dir, strlen(inc_cfg_dir)+1);
if(ret<0)
{
goto error_out;
}
ret=Maat_initiate_feather(feather);
if(ret<0)
{
goto error_out;
}
return feather;
error_out:
Maat_burn_feather(feather);
return NULL;
}
Maat_feather_t Maat_summon_feather_json(int max_thread_num,
const char* table_info_path,
const char* json_rule,
void* logger)
{
int ret=-1;
Maat_feather_t feather=NULL;
feather=Maat_feather(max_thread_num,table_info_path,logger);
if(feather==NULL)
{
return NULL;
}
ret=Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, json_rule, strlen(json_rule)+1);
if(ret<0)
{
goto error_out;
}
ret=Maat_initiate_feather(feather);
if(ret<0)
{
goto error_out;
} return feather;
error_out:
Maat_burn_feather(feather);
return NULL;
}
void Maat_burn_feather(Maat_feather_t feather) void Maat_burn_feather(Maat_feather_t feather)
{ {
_Maat_feather_t* _feather=(_Maat_feather_t*)feather; _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
@@ -530,8 +637,12 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
scan_result_t *region_result=NULL; scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];//dynamic array _compile_result_t compile_result[rule_num];//dynamic array
struct _Maat_table_info_t *p_table=NULL; struct _Maat_table_info_t *p_table=NULL;
struct timeval start,end;
if(_feather->perf_on==1)
{
gettimeofday(&start,NULL);
}
p_table=acqurie_table(_feather, table_id,TABLE_TYPE_EXPR); p_table=acqurie_table(_feather, table_id,TABLE_TYPE_EXPR);
if(p_table==NULL) if(p_table==NULL)
{ {
@@ -587,6 +698,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
} }
if(hit_region_cnt>0) if(hit_region_cnt>0)
{ {
p_table->hit_cnt++;
if(*mid==NULL) if(*mid==NULL)
{ {
_mid=_Maat_make_status(_feather,thread_num); _mid=_Maat_make_status(_feather,thread_num);
@@ -609,6 +721,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
} }
} }
DEC_SCANNER_REF(my_scanner, thread_num); DEC_SCANNER_REF(my_scanner, thread_num);
if(_feather->perf_on==1)
{
gettimeofday(&end,NULL);
maat_stat_table(p_table,data_len,&start, &end);
}
if(compile_ret==0&&hit_region_cnt>0) if(compile_ret==0&&hit_region_cnt>0)
{ {
return -2; return -2;
@@ -643,6 +760,11 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0);
intval_scan_data.int_data=intval; intval_scan_data.int_data=intval;
_Maat_table_info_t* p_table=NULL; _Maat_table_info_t* p_table=NULL;
struct timeval start,end;
if(_feather->perf_on==1)
{
gettimeofday(&start,NULL);
}
p_table=acqurie_table(_feather,table_id,TABLE_TYPE_INTVAL); p_table=acqurie_table(_feather,table_id,TABLE_TYPE_INTVAL);
if(p_table==NULL) if(p_table==NULL)
{ {
@@ -669,6 +791,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
} }
else if(region_ret>0) else if(region_ret>0)
{ {
p_table->hit_cnt++;
if(*mid==NULL) if(*mid==NULL)
{ {
_mid=_Maat_make_status(_feather,thread_num); _mid=_Maat_make_status(_feather,thread_num);
@@ -685,6 +808,11 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
} }
DEC_SCANNER_REF(my_scanner,thread_num); DEC_SCANNER_REF(my_scanner,thread_num);
if(_feather->perf_on==1)
{
gettimeofday(&end,NULL);
maat_stat_table(p_table,0,&start, &end);
}
if(compile_ret==0&&region_ret>0) if(compile_ret==0&&region_ret>0)
{ {
return -2; return -2;
@@ -707,6 +835,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* my_scanner=NULL; struct _Maat_scanner_t* my_scanner=NULL;
struct timeval start,end;
if(_feather->perf_on==1)
{
gettimeofday(&start,NULL);
}
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_IP); p_table=acqurie_table(_feather, table_id, TABLE_TYPE_IP);
if(p_table==NULL) if(p_table==NULL)
{ {
@@ -758,6 +891,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
} }
else if(region_ret>0) else if(region_ret>0)
{ {
p_table->hit_cnt++;
if(*mid==NULL) if(*mid==NULL)
{ {
_mid=_Maat_make_status(_feather,thread_num); _mid=_Maat_make_status(_feather,thread_num);
@@ -773,7 +907,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
result,compile_result,rule_num); result,compile_result,rule_num);
} }
DEC_SCANNER_REF(my_scanner,thread_num); DEC_SCANNER_REF(my_scanner,thread_num);
if(_feather->perf_on==1)
{
gettimeofday(&end,NULL);
maat_stat_table(p_table,0,&start, &end);
}
if(compile_ret==0&&region_ret>0) if(compile_ret==0&&region_ret>0)
{ {
return -2; return -2;
@@ -834,6 +972,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
{ {
sp->do_regex=1; sp->do_regex=1;
} }
p_table->stream_num++;
sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num); sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num);
return sp; return sp;
} }
@@ -852,6 +991,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
scan_result_t *region_result; scan_result_t *region_result;
_compile_result_t compile_result[rule_num];//dynamic array _compile_result_t compile_result[rule_num];//dynamic array
scan_data_t region_scan_data; scan_data_t region_scan_data;
struct timeval start,end;
if(sp->feather->perf_on==1)
{
gettimeofday(&start,NULL);
}
if(data==NULL||data_len==0) if(data==NULL||data_len==0)
{ {
return 0; return 0;
@@ -939,6 +1083,8 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
} }
if(hit_region_cnt>0) if(hit_region_cnt>0)
{ {
sp->feather->p_table_info[sp->table_id]->hit_cnt++;
if(*mid==NULL) if(*mid==NULL)
{ {
_mid=_Maat_make_status(sp->feather,sp->thread_num); _mid=_Maat_make_status(sp->feather,sp->thread_num);
@@ -975,6 +1121,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
free(sp->scan_buff); free(sp->scan_buff);
sp->scan_buff=0; sp->scan_buff=0;
} }
if(sp->feather->perf_on==1)
{
gettimeofday(&end,NULL);
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end);
}
if(compile_ret==0&&hit_region_cnt>0) if(compile_ret==0&&hit_region_cnt>0)
{ {
return -2; return -2;
@@ -998,7 +1149,8 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para)
{ {
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner; struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id];
p_table->stream_num--;
if(scanner!=NULL) if(scanner!=NULL)
{ {
if(sp->version==sp->feather->maat_version) if(sp->version==sp->feather->maat_version)
@@ -1056,6 +1208,8 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
sp->total_len=total_len; sp->total_len=total_len;
sp->fuzzy_hash_handle=tmp_fuzzy_handle; sp->fuzzy_hash_handle=tmp_fuzzy_handle;
pthread_mutex_init(&(sp->fuzzy_mutex),NULL); pthread_mutex_init(&(sp->fuzzy_mutex),NULL);
p_table->stream_num++;
return sp; return sp;
} }
@@ -1092,6 +1246,11 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
char* digest_buff=NULL; char* digest_buff=NULL;
struct _scan_status_t* _mid=(struct _scan_status_t*)(*mid); struct _scan_status_t* _mid=(struct _scan_status_t*)(*mid);
pthread_rwlock_t *GIE_rwlock=&(sp->feather->scanner->digest_rwlock[sp->table_id]); pthread_rwlock_t *GIE_rwlock=&(sp->feather->scanner->digest_rwlock[sp->table_id]);
struct timeval start,end;
if(sp->feather->perf_on==1)
{
gettimeofday(&start,NULL);
}
if(sp->acc_scan_len+(unsigned long long)data_len > sp->total_len) if(sp->acc_scan_len+(unsigned long long)data_len > sp->total_len)
{ {
return 0; return 0;
@@ -1129,24 +1288,30 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{ {
return -1; return -1;
} }
if(hit_region_cnt==0) if(hit_region_cnt>0)
{ {
return 0; sp->feather->p_table_info[sp->table_id]->hit_cnt++;
if(*mid==NULL)
{
_mid=_Maat_make_status(sp->feather,sp->thread_num);
*mid=_mid;
}
else
{
_mid=(struct _scan_status_t*)(*mid);
}
compile_ret=region_compile(_mid,
query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag),
hit_region_cnt,
result,compile_result,rule_num);
}
if(sp->feather->perf_on==1)
{
gettimeofday(&end,NULL);
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end);
} }
if(*mid==NULL)
{
_mid=_Maat_make_status(sp->feather,sp->thread_num);
*mid=_mid;
}
else
{
_mid=(struct _scan_status_t*)(*mid);
}
compile_ret=region_compile(_mid,
query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag),
hit_region_cnt,
result,compile_result,rule_num);
if(compile_ret==0&&hit_region_cnt>0) if(compile_ret==0&&hit_region_cnt>0)
{ {
return -2; return -2;
@@ -1157,7 +1322,8 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para)
{ {
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner; struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id];
p_table->stream_num--;
if(scanner!=NULL) if(scanner!=NULL)
{ {
if(sp->version==sp->feather->maat_version) if(sp->version==sp->feather->maat_version)
@@ -1171,6 +1337,7 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para)
assert(sp->scan_buff==NULL); assert(sp->scan_buff==NULL);
free(sp); free(sp);
*stream_para=NULL; *stream_para=NULL;
return; return;
} }
void Maat_clean_status(scan_status_t* mid) void Maat_clean_status(scan_status_t* mid)
@@ -1181,6 +1348,7 @@ void Maat_clean_status(scan_status_t* mid)
return; return;
} }
_mid=(struct _scan_status_t*)(*mid); _mid=(struct _scan_status_t*)(*mid);
_mid->feather->mid_cnt--;
free(_mid->hitted_group_id); free(_mid->hitted_group_id);
free(_mid); free(_mid);
*mid=NULL; *mid=NULL;

15
src/entry/Maat_rule.cpp
View File

@@ -11,6 +11,7 @@
#include <MESA/MESA_htable.h> #include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h> #include <MESA/MESA_list_queue.h>
#include <MESA/MESA_handle_logger.h> #include <MESA/MESA_handle_logger.h>
#include <MESA/field_stat.h>
#include "Maat_rule.h" #include "Maat_rule.h"
#include "Maat_rule_internal.h" #include "Maat_rule_internal.h"
@@ -24,11 +25,10 @@
#include "mesa_fuzzy.h" #include "mesa_fuzzy.h"
#include "great_index_engine.h" #include "great_index_engine.h"
int MAAT_FRAME_VERSION_1_3_20151224=1; int MAAT_FRAME_VERSION_1_4_20160210=1;
const char *maat_module="MAAT Frame"; const char *maat_module="MAAT Frame";
const char* CHARSET_STRING[]={"CHARSET_NONE","GBK","BIG5","UNICODE","UTF-8"}; const char* CHARSET_STRING[]={"CHARSET_NONE","GBK","BIG5","UNICODE","UTF-8"};
int converHextoint(char srctmp) int converHextoint(char srctmp)
{ {
if(isdigit(srctmp)) if(isdigit(srctmp))
@@ -673,7 +673,7 @@ void destroy_digest_rule(GIE_digest_t*rule)
rule=NULL; rule=NULL;
return; return;
} }
struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread_num,MESA_lqueue_head tomb) struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread_num,MESA_lqueue_head tomb,int rs_scan_type)
{ {
int i=0; int i=0;
@@ -714,6 +714,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread
scanner->ref_cnt=(int*)calloc(CPU_CACHE_ALIGMENT,scan_thread_num); scanner->ref_cnt=(int*)calloc(CPU_CACHE_ALIGMENT,scan_thread_num);
scanner->region_update_q=MESA_lqueue_create(0,0); scanner->region_update_q=MESA_lqueue_create(0,0);
scanner->region=rulescan_initialize(scan_thread_num); scanner->region=rulescan_initialize(scan_thread_num);
rulescan_set_param(scanner->region,rs_scan_type);
scanner->tomb_ref=tomb; scanner->tomb_ref=tomb;
scanner->region_rslt_buff=(scan_result_t*)malloc(sizeof(scan_result_t)*MAX_SCANNER_HIT_NUM*scan_thread_num); scanner->region_rslt_buff=(scan_result_t*)malloc(sizeof(scan_result_t)*MAX_SCANNER_HIT_NUM*scan_thread_num);
@@ -2191,7 +2192,7 @@ void maat_start_cb(unsigned int new_version,int update_type,void*u_para)
int i=0,j=0; int i=0,j=0;
if(update_type==CM_UPDATE_TYPE_FULL) if(update_type==CM_UPDATE_TYPE_FULL)
{ {
feather->update_tmp_scanner=create_maat_scanner(new_version,feather->scan_thread_num,feather->garbage_q); feather->update_tmp_scanner=create_maat_scanner(new_version,feather->scan_thread_num,feather->garbage_q,feather->rule_scan_type);
MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
"Full config version %u -> %u update start", "Full config version %u -> %u update start",
feather->maat_version,new_version); feather->maat_version,new_version);
@@ -2375,12 +2376,17 @@ void *thread_rule_monitor(void *arg)
} }
} }
garbage_bury(feather->garbage_q,feather->logger); garbage_bury(feather->garbage_q,feather->logger);
if(feather->stat_on==1)
{
maat_stat_output(feather);
}
} }
MESA_htable_destroy(feather->map_tablename2id,free); MESA_htable_destroy(feather->map_tablename2id,free);
destroy_maat_scanner(feather->scanner); destroy_maat_scanner(feather->scanner);
garbage_bury(feather->garbage_q,feather->logger); garbage_bury(feather->garbage_q,feather->logger);
MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL); MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL);
FS_stop(&(feather->stat_handle));
int i=0,j=0; int i=0,j=0;
struct dynamic_array_t* d_array=NULL; struct dynamic_array_t* d_array=NULL;
@@ -2406,3 +2412,4 @@ void *thread_rule_monitor(void *arg)
free(feather); free(feather);
return NULL; return NULL;
} }

35
src/entry/Maat_rule_internal.h
View File

@@ -5,6 +5,7 @@
#include <MESA/MESA_htable.h> #include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h> #include <MESA/MESA_list_queue.h>
#include <MESA/field_stat.h>
#include "dynamic_array.h" #include "dynamic_array.h"
#include "UniversalBoolMatch.h" #include "UniversalBoolMatch.h"
#include "rulescan.h" #include "rulescan.h"
@@ -47,6 +48,7 @@ typedef int atomic_t;
#define MAX_FAILED_NUM 128 #define MAX_FAILED_NUM 128
#define MAX_MAAT_STAT_NUM 64
#ifndef MAX #ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b)) #define MAX(a, b) (((a) > (b)) ? (a) : (b))
#endif #endif
@@ -91,6 +93,7 @@ enum MAAT_MATCH_METHOD
MATCH_METHOD_LEFT, MATCH_METHOD_LEFT,
MATCH_METHOD_FULL MATCH_METHOD_FULL
}; };
struct db_str_rule_t struct db_str_rule_t
{ {
int region_id; int region_id;
@@ -224,6 +227,13 @@ struct _Maat_table_info_t
int expr_rule_cnt; //expr_type=0,1,3 int expr_rule_cnt; //expr_type=0,1,3
int regex_rule_cnt; //expr_type=2 int regex_rule_cnt; //expr_type=2
struct _plugin_table_info *cb_info; struct _plugin_table_info *cb_info;
//for stat>>>>>>>>
int stat_line_id;
long long scan_cnt;
long long scan_cpu_time; //microseconds
long long input_bytes;
long long stream_num;
long long hit_cnt;
}; };
struct _scan_status_t struct _scan_status_t
{ {
@@ -294,17 +304,28 @@ struct _Maat_feather_t
struct _Maat_scanner_t *update_tmp_scanner; struct _Maat_scanner_t *update_tmp_scanner;
MESA_lqueue_head garbage_q; MESA_lqueue_head garbage_q;
int table_cnt; int table_cnt;
int GROUP_MODE_ON;
int still_working;
int scan_interval_ms;
int effect_interval_ms;
int stat_on;
int perf_on;
struct _Maat_table_info_t *p_table_info[MAX_TABLE_NUM]; struct _Maat_table_info_t *p_table_info[MAX_TABLE_NUM];
MESA_htable_handle map_tablename2id; MESA_htable_handle map_tablename2id;
void* logger; void* logger;
int maat_version; int maat_version;
int scan_thread_num; int scan_thread_num;
int rule_scan_type;
char inc_dir[MAX_TABLE_NAME_LEN]; char inc_dir[MAX_TABLE_NAME_LEN];
char full_dir[MAX_TABLE_NAME_LEN]; char full_dir[MAX_TABLE_NAME_LEN];
int GROUP_MODE_ON; char stat_file[MAX_TABLE_NAME_LEN];
int still_working; //for stat>>>>
int scan_interval_ms; screen_stat_handle_t stat_handle;
int effect_interval_ms; int total_stat_id;
int fs_status_id[MAX_MAAT_STAT_NUM];
int fs_column_id[MAX_MAAT_STAT_NUM];
long long mid_cnt;
long long hit_cnt;
}; };
struct _maat_garbage_t struct _maat_garbage_t
{ {
@@ -338,5 +359,11 @@ inline void ipv6_ntoh(unsigned int *v6_addr)
} }
return; return;
} }
void maat_stat_init(struct _Maat_feather_t* feather);
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timeval* start, struct timeval* end);
void maat_stat_output(struct _Maat_feather_t* feather);
#endif #endif

206
src/entry/Maat_stat.cpp Normal file
View File

@@ -0,0 +1,206 @@
#include "Maat_rule_internal.h"
#include "field_stat.h"
enum MAAT_FS_STATUS{
STATUS_VERSION=0,
STATUS_TABLE_NUM,
STATUS_MID_NUM,
STATUS_GARBAGE_QSIZE,
};
enum MAAT_FS_COLUMN
{
COLUMN_TABLE_RULE_NUM=0,
COLUMN_TABLE_REGEX_NUM,
COLUMN_TABLE_STREAM_NUM,
COLUMN_TABLE_SCAN_CNT,
COLUMN_TABLE_SCAN_BYTES,
COLUMN_TABLE_CPU_TIME,//microseconds
COLUMN_TABLE_HIT_CNT
};
void maat_stat_init(struct _Maat_feather_t* feather)
{
int value=0;
int i=0;
struct _Maat_table_info_t* p_table=NULL;
feather->stat_handle=FS_create_handle();
FS_set_para(feather->stat_handle, OUTPUT_DEVICE, feather->stat_file, strlen(feather->stat_file)+1);
value=1;
FS_set_para(feather->stat_handle, PRINT_MODE, &value, sizeof(value));
value=0;
FS_set_para(feather->stat_handle, CREATE_THREAD, &value, sizeof(value));
feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"version");
feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"table_num");
feather->fs_status_id[STATUS_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"mid_num");
feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"garbage_num");
feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule");
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"regex");
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"stream");
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"scan_cps");
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"IN_Bps");
if(feather->perf_on==1)
{
feather->fs_column_id[COLUMN_TABLE_CPU_TIME]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"cpu_us");
value=feather->fs_column_id[COLUMN_TABLE_CPU_TIME];
FS_set_para(feather->stat_handle, ID_INVISBLE, &value, sizeof(value));
FS_register_ratio(feather->stat_handle,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_STYLE_COLUMN,
FS_CALC_SPEED,
"PROC_Bps");
}
feather->fs_column_id[COLUMN_TABLE_HIT_CNT]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"hit_cnt");
value=feather->fs_column_id[COLUMN_TABLE_HIT_CNT];
FS_set_para(feather->stat_handle, ID_INVISBLE, &value, sizeof(value));
FS_register_ratio(feather->stat_handle,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_STYLE_COLUMN,
FS_CALC_SPEED,
"hit_rate");
feather->total_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT,"Sum");
for(i=0;i<MAX_TABLE_NUM;i++)
{
p_table=feather->p_table_info[i];
if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN
||p_table->table_type==TABLE_TYPE_GROUP
||p_table->table_type==TABLE_TYPE_COMPILE)
{
continue;
}
p_table->stat_line_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT,p_table->table_name);
}
FS_start(feather->stat_handle);
return;
}
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timeval* start, struct timeval* end)
{
p_table->scan_cnt++;
p_table->input_bytes+=scan_len;
p_table->scan_cpu_time+=(end->tv_sec-start->tv_sec)*1000000+end->tv_usec-start->tv_usec;
return;
}
void maat_stat_output(struct _Maat_feather_t* feather)
{
long value=0;
long long total_cfg_num=0, total_input_bytes=0, total_regex_num=0;
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0;
int i=0;
struct _Maat_table_info_t* p_table=NULL;
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_MID_NUM], 0,FS_OP_SET,feather->mid_cnt);
value=MESA_lqueue_get_count(feather->garbage_q);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value);
for(i=0;i<MAX_TABLE_NUM;i++)
{
p_table=feather->p_table_info[i];
if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN
||p_table->table_type==TABLE_TYPE_GROUP
||p_table->table_type==TABLE_TYPE_COMPILE)
{
continue;
}
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
FS_OP_SET,
p_table->cfg_num);
total_cfg_num+=p_table->cfg_num;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM],
FS_OP_SET,
p_table->regex_rule_cnt);
total_regex_num+= p_table->regex_rule_cnt;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
FS_OP_SET,
p_table->stream_num);
total_stream_cnt+= p_table->stream_num;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_ADD,
p_table->scan_cnt);
total_scan_cnt+=p_table->scan_cnt;
p_table->scan_cnt=0;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_ADD,
p_table->input_bytes);
total_input_bytes+=p_table->input_bytes;
p_table->input_bytes=0;
if(feather->perf_on==1)
{
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_ADD,
p_table->scan_cpu_time);
total_cpu_time+=p_table->scan_cpu_time;
p_table->scan_cpu_time=0;
}
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_ADD,
p_table->hit_cnt);
//total hit count stat in region_compile
p_table->hit_cnt=0;
}
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
FS_OP_SET,
total_cfg_num);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM],
FS_OP_SET,
total_regex_num);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
FS_OP_SET,
total_stream_cnt);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_ADD,
total_scan_cnt);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_ADD,
total_input_bytes);
if(feather->perf_on==1)
{
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_ADD,
total_cpu_time);
}
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_ADD,
feather->hit_cnt);
feather->hit_cnt=0;
FS_passive_output(feather->stat_handle);
return;
}

6
src/entry/Makefile
View File

@@ -6,7 +6,7 @@ CCC = g++
CFLAGS = -Wall -g -fPIC CFLAGS = -Wall -g -fPIC
CFLAGS += $(OPTFLAGS) CFLAGS += $(OPTFLAGS)
LDDICTATOR = -Wl,-wrap,malloc -Wl,-wrap,calloc -Wl,-wrap,free -Wl,-wrap,realloc LDDICTATOR = -Wl,-wrap,malloc -Wl,-wrap,calloc -Wl,-wrap,free -Wl,-wrap,realloc
LDFLAGS = -lMESA_handle_logger -lMESA_htable -lpthread -lm -lrulescan -lpcre LDFLAGS = -lMESA_handle_logger -lMESA_htable -lpthread -lm -lrulescan -lpcre -lMESA_field_stat
#LDFLAGS += $(LDDICTATOR) #LDFLAGS += $(LDDICTATOR)
MAILLIB = ../lib MAILLIB = ../lib
@@ -15,8 +15,8 @@ H_DIR =-I$(G_H_DIR) -I../../inc
LIBMAAT = libmaatframe.a LIBMAAT = libmaatframe.a
LIBMAAT_SO = libmaatframe.so LIBMAAT_SO = libmaatframe.so
OBJS=config_monitor.o Maat_rule.o Maat_api.o UniversalBoolMatch.o dynamic_array.o cJSON.o json2iris.o map_str2int.o\ OBJS=config_monitor.o Maat_rule.o Maat_api.o Maat_stat.o UniversalBoolMatch.o dynamic_array.o cJSON.o\
interval_index.o great_index_engine.o mesa_fuzzy.o json2iris.o map_str2int.o interval_index.o great_index_engine.o mesa_fuzzy.o
.c.o: .c.o:
$(CC) -c $(CFLAGS) -I. $(H_DIR) $< $(CC) -c $(CFLAGS) -I. $(H_DIR) $<

70
src/inc_internal/field_stat.h Normal file
View File

@@ -0,0 +1,70 @@
#ifndef H_SCREEN_STAT_H_INCLUDE
#define H_SCREEN_STAT_H_INCLUDE
#include <stdio.h>
#ifndef __cplusplus
#error("This file should be compiled with C++ compiler")
#endif
enum field_dsp_style_t
{
FS_STYLE_FIELD=0,
FS_STYLE_COLUMN,
FS_STYLE_LINE,
FS_STYLE_STATUS
};
enum field_calc_algo
{
FS_CALC_CURRENT=0,
FS_CALC_SPEED
};
enum field_op
{
FS_OP_ADD=1,
FS_OP_SET
};
typedef void* screen_stat_handle_t;
enum FS_option
{
OUTPUT_DEVICE, //VALUE is a const char*, indicate a file path string, SIZE = strlen(string+'\0')+1.DEFAULT:output to stdout.
PRINT_MODE, //VALUE is an interger,1:Rewrite ,2: Append. SIZE=4,DEFALUT:REWRITE.
STAT_CYCLE, //VALUE is an interger idicate interval seconds of every output, SIZE=4 ,DEFUALT:2 seconds.
PRINT_TRIGGER, //VALUE is an interger,1:Do print,0: Don't print.SIZE=4.DEFAULT:1.
CREATE_THREAD,//VALUE is an interger,1: Create a print thread,0:not create,output by call passive_output function,
//and the STAT_CYCLE is meaningless.SIZE=4,DEFAULT:0.
ID_INVISBLE,//value is field_id/status_id/column_id, not output this string, SIZE=4,DEFAULT: shutdown NO one.
};
//Always success.
screen_stat_handle_t FS_create_handle(void);
int FS_set_para(screen_stat_handle_t handle, enum FS_option type,const void* value,int size);
void FS_start(screen_stat_handle_t handle);
void FS_stop(screen_stat_handle_t* handle);
//return field_id/line_id/column_id greater than zero if success,return an interger less than zero if failed.
int FS_register(screen_stat_handle_t handle,enum field_dsp_style_t style,enum field_calc_algo calc_type,const char* name);
//numerator_id and denominator_id must be column/field/status style.
int FS_register_ratio(screen_stat_handle_t handle,int numerator_id,int denominator_id,enum field_dsp_style_t style,enum field_calc_algo calc_type,const char* name);
//id: when id's type is FIELD , column_id is ignore.
int FS_operate(screen_stat_handle_t handle,int id,int column_id,enum field_op op,long long value);
void FS_passive_output(screen_stat_handle_t handle);
screen_stat_handle_t init_screen_stat(FILE* output_fp,int stat_cycle,int screen_print_trigger);
//return field_id >=0 when success, return -1 when failed.
int stat_field_register(screen_stat_handle_t handle,const char* field_name);
//return >=0 when success, return -1 when failed.
#define FS_OP_TYPE_ADD (FS_OP_ADD)
#define FS_OP_TYPE_SET (FS_OP_SET)
int stat_field_operation(screen_stat_handle_t handle,int field_id,int operation,long long value);
#endif

24
src/inc_internal/rulescan.h
View File

@@ -23,10 +23,21 @@ extern "C"
{ {
#endif #endif
/* <20><><EFBFBD>岻ͬ<E5B2BB><CDAC>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
enum ScanType
{
SCANTYPE_DEFAULT = 0,
SCANTYPE_DETAIL_RESULT = 1,
SCANTYPE_REGEX_GROUP = 2
};
#define MAX_REGEX_GROUP_NUM 5 /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD><EFBFBD>֧<EFBFBD>ֵ<EFBFBD><D6B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ĸ<EFBFBD><C4B8><EFBFBD> */
#define MAX_EXPR_ITEM_NUM (1U<<3) /* ÿ<><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>MAX_EXPR_ITEM_NUM<55><4D><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */ #define MAX_EXPR_ITEM_NUM (1U<<3) /* ÿ<><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>MAX_EXPR_ITEM_NUM<55><4D><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
#define MAX_MATCH_POS_NUM 1024 /* ÿ<><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD>λ<EFBFBD>õĸ<C3B5><C4B8><EFBFBD> */ #define MAX_MATCH_POS_NUM 1024 /* ÿ<><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD>λ<EFBFBD>õĸ<C3B5><C4B8><EFBFBD> */
#define MATCH_POS_NUM_INC 64 /* ÿ<><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD>λ<EFBFBD>õĸ<C3B5><C4B8><EFBFBD><EFBFBD><EFBFBD>ʼֵ<CABC><D6B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֵ */
/* <20><><EFBFBD>岻ͬ<E5B2BB>Ĺ<EFBFBD><C4B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */ /* <20><><EFBFBD>岻ͬ<E5B2BB>Ĺ<EFBFBD><C4B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
const unsigned int RULETYPE_STR = 0; /* <20>ַ<EFBFBD><D6B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ƹ<EFBFBD><C6B9><EFBFBD> */ const unsigned int RULETYPE_STR = 0; /* <20>ַ<EFBFBD><D6B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ƹ<EFBFBD><C6B9><EFBFBD> */
const unsigned int RULETYPE_REG = 1; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD><EFBFBD> */ const unsigned int RULETYPE_REG = 1; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD><EFBFBD> */
const unsigned int RULETYPE_INT = 2; /* <20><>ֵ<EFBFBD><D6B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */ const unsigned int RULETYPE_INT = 2; /* <20><>ֵ<EFBFBD><D6B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
@@ -195,6 +206,17 @@ extern "C"
*/ */
void * rulescan_initialize(unsigned int max_thread_num); void * rulescan_initialize(unsigned int max_thread_num);
/*
<09><><EFBFBD>ܣ<EFBFBD>
<09><><EFBFBD><EFBFBD>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>rulescan_update֮ǰ<D6AE>ɶ<EFBFBD><C9B6>ε<EFBFBD><CEB5>ã<EFBFBD>ÿ<EFBFBD><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
<09><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
instance[in]: ɨ<><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ָ<EFBFBD>룻
scan_type_flag[in]: ɨ<><C9A8><EFBFBD><EFBFBD><EFBFBD>͵ı<CDB5>־<EFBFBD><D6BE>Ϣ<EFBFBD><CFA2>0<EFBFBD><30><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>λ<EFBFBD>õ<EFBFBD><C3B5><EFBFBD>Ϣ<EFBFBD><CFA2>1<EFBFBD><31><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>λ<EFBFBD>õ<EFBFBD><C3B5><EFBFBD>Ϣ<EFBFBD><CFA2><EFBFBD><EFBFBD><EFBFBD>Dz<EFBFBD><C7B2><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ϣ<EFBFBD><CFA2>2<EFBFBD><32><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>λ<EFBFBD><CEBB><EFBFBD><EFBFBD>Ϣ<EFBFBD><CFA2><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ϣ
<09><><EFBFBD><EFBFBD>ֵ<EFBFBD><D6B5>
1<><31><EFBFBD><EFBFBD>ȷ<EFBFBD><C8B7><EFBFBD>ã<EFBFBD>-1<><31><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʧ<EFBFBD>ܡ<EFBFBD>
*/
int rulescan_set_param(void * instance, int scan_type_flag);
/* /*
<09><><EFBFBD>ܣ<EFBFBD><DCA3><EFBFBD>̬ע<CCAC><D7A2>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>󡣶<EFBFBD><F3A1A3B6><EFBFBD>ͬһ<CDAC><D2BB>instance<63><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͬʱ<CDAC>ж<EFBFBD><D0B6><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>̡߳<DFB3> <09><><EFBFBD>ܣ<EFBFBD><DCA3><EFBFBD>̬ע<CCAC><D7A2>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>󡣶<EFBFBD><F3A1A3B6><EFBFBD>ͬһ<CDAC><D2BB>instance<63><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͬʱ<CDAC>ж<EFBFBD><D0B6><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>̡߳<DFB3>
<09><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <09><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

21
test/maat_test.cpp
View File

@@ -13,6 +13,7 @@
#include <MESA/stream.h> #include <MESA/stream.h>
#include <sys/types.h>//fstat #include <sys/types.h>//fstat
#include <sys/stat.h>//fstat #include <sys/stat.h>//fstat
#include <unistd.h>
void Maat_read_entry_start_cb(int update_type,void* u_para) void Maat_read_entry_start_cb(int update_type,void* u_para)
{ {
@@ -36,7 +37,7 @@ void Maat_read_entry_finish_cb(void* u_para)
{ {
return; return;
} }
int main() int main(int argc,char* argv[])
{ {
Maat_feather_t feather=NULL; Maat_feather_t feather=NULL;
int cb_table_id=-1,url_scan_table_id=-1,size_scan_table_id=-1,ip_scan_table_id=-1,digest_scan_table_id=-1; int cb_table_id=-1,url_scan_table_id=-1,size_scan_table_id=-1,ip_scan_table_id=-1,digest_scan_table_id=-1;
@@ -49,20 +50,31 @@ int main()
const char* log_file="./test.log"; const char* log_file="./test.log";
const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
const char* digest_test_file="./digest_test.data"; const char* digest_test_file="./digest_test.data";
const char* stat_file="./scan_staus.log";
struct stat digest_fstat; struct stat digest_fstat;
unsigned long long read_size=0,scan_offset=0; unsigned long long read_size=0,scan_offset=0;
char digest_test_buff[4096]={0}; char digest_test_buff[4096]={0};
int scan_val=2015; int scan_val=2015;
struct Maat_rule_t result[4]; struct Maat_rule_t result[4];
int found_pos[4]; int found_pos[4];
int scan_detail=0;
scan_status_t mid=NULL; scan_status_t mid=NULL;
void *logger=MESA_create_runtime_log_handle(log_file,0); void *logger=MESA_create_runtime_log_handle(log_file,0);
feather=Maat_summon_feather_json(g_iThreadNum,
/* feather=Maat_summon_feather_json(g_iThreadNum,
table_info_path, table_info_path,
json_path, json_path,
logger); logger);
*/
// feather=Maat_summon_feather(g_iThreadNum,table_info_path,ful_cfg_dir,inc_cfg_dir,logger); // feather=Maat_summon_feather(g_iThreadNum,table_info_path,ful_cfg_dir,inc_cfg_dir,logger);
feather=Maat_feather(g_iThreadNum, table_info_path, logger);
Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, json_path, sizeof(json_path)+1);
Maat_set_feather_opt(feather, MAAT_OPT_STAT_FILE_PATH, stat_file, strlen(stat_file));
Maat_set_feather_opt(feather, MAAT_OPT_STAT_ON, NULL, 0);
Maat_set_feather_opt(feather, MAAT_OPT_PERF_ON, NULL, 0);
Maat_set_feather_opt(feather, MAAT_OPT_SCAN_DETAIL, &scan_detail, sizeof(scan_detail));
Maat_initiate_feather(feather);
if(feather==NULL) if(feather==NULL)
{ {
printf("Maat initial error, see %s\n",log_file); printf("Maat initial error, see %s\n",log_file);
@@ -140,6 +152,7 @@ int main()
break; break;
} }
} }
sleep(2);
Maat_clean_status(&mid); Maat_clean_status(&mid);
struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10); struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10);
stream_para_t sp=Maat_stream_scan_string_start(feather,url_scan_table_id,0); stream_para_t sp=Maat_stream_scan_string_start(feather,url_scan_table_id,0);
@@ -202,7 +215,6 @@ int main()
} }
} }
Maat_clean_status(&mid); Maat_clean_status(&mid);
digest_scan_table_id=Maat_table_register(feather, "FILE_DIGEST"); digest_scan_table_id=Maat_table_register(feather, "FILE_DIGEST");
if(digest_scan_table_id<0) if(digest_scan_table_id<0)
{ {
@@ -238,6 +250,7 @@ int main()
} }
Maat_stream_scan_string_end(&sp); Maat_stream_scan_string_end(&sp);
Maat_clean_status(&mid); Maat_clean_status(&mid);
sleep(4);
Maat_burn_feather(feather); Maat_burn_feather(feather);
free(hit_detail); free(hit_detail);
return 0; return 0;