gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完成单元测试。

Browse Source
This commit is contained in:
zhengchao
2018-12-05 18:00:55 +08:00
parent 7b5baacf62
commit 899a61e371
11 changed files with 466 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
src/entry/Maat_api.cpp
View File

@@ -443,7 +443,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
return NULL;
}
_Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1);
feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger);
feather->table_cnt=read_table_description(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger);
feather->map_tablename2id=map_create();
int i=0,j=0,ret=0;
for(i=0;i<MAX_TABLE_NUM;i++)
@@ -930,7 +930,6 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
{
return -1;
}
plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin);
//plugin table register blocks background update.
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
idx=plugin_desc->cb_plug_cnt;
@@ -944,6 +943,12 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
plugin_desc->cb_plug[idx].update=update;
plugin_desc->cb_plug[idx].finish=finish;
plugin_desc->cb_plug[idx].u_para=u_para;
if(_feather->scanner==NULL)
{
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return 1;
}
plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin);
if(plugin_aux->cache_line_num>0)
{
if(start!=NULL)
@@ -1051,46 +1056,128 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa
pthread_rwlock_unlock(&(compile_inner->rwlock));
return ad;
}
MESA_htable_handle plugin_EX_htable_new(const struct plugin_table_desc* plugin_desc,
struct dynamic_array_t* lines, size_t line_cnt)
struct wrap_plugin_EX_data
{
MAAT_RULE_EX_DATA exdata;
const struct Maat_table_desc* ref_plugin_table;
};
void wrap_plugin_EX_data_free(void *data)
{
struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data;
const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc);
ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp);
wrap_data->ref_plugin_table=NULL;
free(wrap_data);
return;
}
MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index)
{
MESA_htable_handle key2ex_hash=NULL;
unsigned int slot_size=1;
const char* line=NULL;
size_t key_offset=0, key_len=0;
long long estimate_size=plugin_desc->estimate_size, i=0;
int ret=0;
MAAT_RULE_EX_DATA exdata=NULL;
while(estimate_size>0)
while(estimate_size!=0)
{
estimate_size=estimate_size<<1;
estimate_size=estimate_size>>1;
slot_size*=2;
}
if(slot_size==1)
{
slot_size=4096;
}
MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs));
hargs.thread_safe=1;
hargs.thread_safe=0;
hargs.hash_slot_size = slot_size;
hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
hargs.expire_time = 0;
hargs.key_comp = NULL;
hargs.key2index = plugin_desc->ex_desc.key2index_func;
hargs.key2index = NULL; //Not supported yet.
hargs.recursive = 1;
hargs.data_free = NULL ;
hargs.data_free = wrap_plugin_EX_data_free;
hargs.data_expire_with_condition = NULL;
key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(key2ex_hash, 0);
return key2ex_hash;
}
int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line,
MESA_htable_handle key2ex_hash, void *logger)
{
size_t key_offset=0, key_len=0;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
int ret=0;
ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data del error: cannot find column %d of %s",
plugin_desc->key_column, line);
return -1;
}
ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data del error: no such key %.*s of %s",
key_len, line+key_offset, line);
return -1;
}
return 0;
}
int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line,
MESA_htable_handle key2ex_hash, void *logger)
{
char* key=NULL;
size_t key_offset=0, key_len=0;
MAAT_RULE_EX_DATA exdata=NULL;
struct wrap_plugin_EX_data* wrap_data=NULL;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
int ret=0;
ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data add error: cannot find column %d of %s",
plugin_desc->key_column, line);
return -1;
}
key=ALLOC(char, key_len+1);
memcpy(key, line+key_offset, key_len);
plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata,
plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
wrap_data=ALLOC(struct wrap_plugin_EX_data, 1);
wrap_data->exdata=exdata;
wrap_data->ref_plugin_table=plugin_table;
ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data);
free(key);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data add error: duplicated key %.*s of %s",
key_len, line+key_offset, line);
wrap_plugin_EX_data_free(wrap_data);
return -1;
}
return 0;
}
MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table,
struct dynamic_array_t* lines, size_t line_cnt, void* logger)
{
MESA_htable_handle key2ex_hash=NULL;
size_t i=0;
const char* line=NULL;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func);
for(i=0; i< line_cnt; i++)
{
line=(const char*)dynamic_array_read(lines, i);
ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
exdata=plugin_desc->ex_desc.new_func(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);;
MESA_htable_add(key2ex_hash, line+key_offset, key_len, const void * data)
plugin_EX_data_new(plugin_table, line, key2ex_hash, logger);
}
return key2ex_hash;
}
int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
@@ -1103,26 +1190,71 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_table_desc *table_desc=_feather->p_table_info[table_id];
struct plugin_table_desc* plugin_desc=&(table_desc->plugin);
struct Maat_table_runtime* table_rt=_feather->scanner->table_rt[table_id];
MESA_htable_handle key2ex_hash=NULL;
if(table_desc->table_type!=TABLE_TYPE_PLUGIN || new_func==NULL || free_func==NULL || dup_func==NULL)
struct Maat_table_runtime* table_rt=NULL;
if(new_func==NULL || free_func==NULL || dup_func==NULL )
{
assert(0);
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
return -1;
}
if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata
|| plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0)
{
assert(0);
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module,
"%s failed: key or valid flag column are not specified", __FUNCTION__);
return -1;
}
plugin_desc->ex_desc.new_func=new_func;
plugin_desc->ex_desc.free_func=free_func;
plugin_desc->ex_desc.dup_func=dup_func;
plugin_desc->ex_desc.key2index_func=key2index_func;
plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used.
plugin_desc->ex_desc.argl=argl;
plugin_desc->ex_desc.argp=argp;
plugin_desc->have_exdata=1;
if(_feather->scanner==NULL)
{
return 0;
}
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
table_rt->plugin.key2ex_hash=plugin_EX_htable_new(plugin_desc,table_rt->plugin.cache_lines, table_rt->plugin.cache_line_num);
table_rt=_feather->scanner->table_rt[table_id];
assert(table_rt->plugin.key2ex_hash==NULL);
table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines,
table_rt->plugin.cache_line_num, _feather->logger);
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return 0;
}
MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_table_desc *table_desc=_feather->p_table_info[table_id];
struct Maat_table_runtime *table_rt= NULL;
struct plugin_table_desc* plugin_desc=&(table_desc->plugin);
struct wrap_plugin_EX_data* wrap_data=NULL;
MAAT_RULE_EX_DATA exdata=NULL;
if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata==0)
{
assert(0);
return NULL;
}
if(_feather->scanner==NULL)
{
return NULL;
}
table_rt= _feather->scanner->table_rt[table_id];
pthread_rwlock_rdlock(&table_rt->plugin.rwlock);
wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash,
(const unsigned char*)key, strlen(key));
if(wrap_data!=NULL)
{
plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata),
plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
}
pthread_rwlock_unlock(&table_rt->plugin.rwlock);
return exdata;
}
int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
,enum MAAT_CHARSET charset,const char* data,int data_len

7
src/entry/Maat_command.cpp
View File

@@ -221,7 +221,7 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_
assert(0);
}
ret=get_column_pos(const char* line, int column_seq, &offset, &len);
ret=get_column_pos(line, column_seq, &offset, &len);
if(ret<0||offset>=strlen(line)||(line[offset]!='1'&&line[offset]!='0'))// 0 is also a valid value for some non-MAAT producer.
{
return -1;
@@ -2131,6 +2131,11 @@ int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value,
int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value, size_t size, enum MAAT_OPERATION op)
{
struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather;
redisContext* ctx=_feather->mr_ctx.write_ctx;
if(ctx==NULL)
{
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, "%s failed: Redis is not connected.", __FUNCTION__);
return -1;
}
const char *arg_vec[3];
size_t len_vec[3];

94
src/entry/Maat_rule.cpp
View File

@@ -529,7 +529,7 @@ int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_h
char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0};
char *token=NULL,*sub_token=NULL,*saveptr;
struct expr_table_desc* p=&(table->expr);
sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id)
sscanf(line,"%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id)
,table->table_name[0]
,table_type
,src_charset
@@ -602,31 +602,21 @@ int _read_integer_arrary(char* string, int *array, int size)
return i;
}
int read_plugin_table_info(const char* line, struct Maat_table_desc* p)
int read_plugin_table_description(const char* line, struct Maat_table_desc* p)
{
int i=0,ret=0;
size_t offset=0, len=0;
cJSON* json=NULL, *tmp=NULL, *array_item=NULL;
char* copy_line=NULL, *plug_info=NULL;
char *token=NULL,*sub_token=NULL,*saveptr;
struct plugin_table_desc* plugin_desc=&(p->plugin);
plugin_desc->valid_flag_column=-1;
copy_line=_maat_strdup(line);
for (token = copy_line, i=0; ; token= NULL, i++)
{
sub_token= strtok_r(token,"\t ", &saveptr);
if (sub_token == NULL)
break;
if(i==3)
{
break;
}
}
if(i<3)
ret=get_column_pos(copy_line, 4, &offset, &len);
if(i<0)
{
goto error_out;
}
plug_info=sub_token;
copy_line[offset+len+1]='\0';
plug_info=copy_line+offset;
if(strlen(plug_info)<4)//For old version compatible.
{
@@ -693,7 +683,7 @@ error_out:
free(copy_line);
return -1;
}
int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger)
int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger)
{
FILE*fp=NULL;
char line[MAX_TABLE_LINE_SIZE];
@@ -750,7 +740,7 @@ int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* ta
}
p=table_info_new(max_thread_num);
ret=sscanf(line,"%hu\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id)
ret=sscanf(line,"%d\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id)
,p->table_name[0]
,table_type_str
,not_care);
@@ -781,7 +771,7 @@ int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* ta
}
break;
case TABLE_TYPE_PLUGIN:
ret=read_plugin_table_info(line, p);
ret=read_plugin_table_description(line, p);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i);
@@ -1193,29 +1183,36 @@ void destroy_digest_rule(GIE_digest_t*rule)
}
struct Maat_table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_thread_num)
struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num)
{
struct Maat_table_runtime* p= ALLOC(struct Maat_table_runtime, 1);
p->table_type=type;
switch(type)
struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1);
table_rt->table_type=table_desc->table_type;
switch(table_desc->table_type)
{
case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY:
p->similar.update_q=MESA_lqueue_create(0,0);
table_rt->similar.update_q=MESA_lqueue_create(0,0);
break;
case TABLE_TYPE_PLUGIN:
p->plugin.cache_lines=dynamic_array_create(1, 1024);
table_rt->plugin.cache_lines=dynamic_array_create(1, 1024);
if(table_desc->plugin.have_exdata)
{
table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size,
table_desc->plugin.ex_desc.key2index_func);
pthread_rwlock_init(&(table_rt->plugin.rwlock), NULL);
}
break;
default:
break;
}
p->scan_cnt=alignment_int64_array_alloc(max_thread_num);
p->scan_cpu_time=alignment_int64_array_alloc(max_thread_num);
p->input_bytes=alignment_int64_array_alloc(max_thread_num);
p->stream_num=alignment_int64_array_alloc(max_thread_num);
p->hit_cnt=alignment_int64_array_alloc(max_thread_num);
return p;
table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num);
table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num);
table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num);
table_rt->stream_num=alignment_int64_array_alloc(max_thread_num);
table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num);
return table_rt;
}
void table_runtime_free(struct Maat_table_runtime* p)
{
@@ -1344,7 +1341,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
{
continue;
}
table_rt=table_runtime_new(pp_table_desc[i]->table_type, feather->scan_thread_num);
table_rt=table_runtime_new(pp_table_desc[i], feather->scan_thread_num);
if(pp_table_desc[i]->table_type==TABLE_TYPE_EXPR||pp_table_desc[i]->table_type==TABLE_TYPE_EXPR_PLUS)
{
expr_desc=&(pp_table_desc[i]->expr);
@@ -3178,7 +3175,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
char *p=NULL;
char* copy=NULL;
size_t is_valid_offset=0, valid_len=0;
char *token=NULL,*sub_token=NULL,*saveptr;
if(plugin_desc->rule_tag_column>0&&n_tags>0)
{
@@ -3213,13 +3210,32 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M
}
table_rt->plugin.acc_line_num++;
if(plugin_desc->cb_plug_cnt>0)
if(plugin_desc->have_exdata || plugin_desc->cb_plug_cnt>0)
{
for(i=0;i<plugin_desc->cb_plug_cnt;i++)
if(plugin_desc->have_exdata)
{
plugin_desc->cb_plug[i].update(table->table_id,table_line,plugin_desc->cb_plug[i].u_para);
ret=get_column_pos(table_line, plugin_desc->valid_flag_column, &is_valid_offset, &valid_len);
pthread_rwlock_wrlock(&(table_rt->plugin.rwlock));
if(atoi(table_line+is_valid_offset)==1)
{
plugin_EX_data_new(table, table_line, table_rt->plugin.key2ex_hash, logger);
}
else
{
plugin_EX_data_free(table, table_line, table_rt->plugin.key2ex_hash, logger);
}
pthread_rwlock_unlock(&(table_rt->plugin.rwlock));
}
if(plugin_desc->cb_plug_cnt>0)
{
for(i=0;i<plugin_desc->cb_plug_cnt;i++)
{
plugin_desc->cb_plug[i].update(table->table_id,table_line,plugin_desc->cb_plug[i].u_para);
}
}
}
else
{
@@ -3227,7 +3243,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M
memcpy(p,table_line,len);
table_rt->plugin.cache_size+=len;
dynamic_array_write(table_rt->plugin.cache_lines,table_rt->plugin.cache_line_num,p);
table_rt->plugin.cache_line_num++;
table_rt->plugin.cache_line_num++;
}
}
void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger)

2
src/entry/Maat_utils.cpp
View File

@@ -179,7 +179,7 @@ int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len
{
*offset=subtoken-dup_line;
*len=strlen(subtoken);
ret=0
ret=0;
break;
}
i++;

1
src/entry/json2iris.cpp
View File

@@ -12,6 +12,7 @@
#include "cJSON.h"
#include "hiredis.h"
#include "map_str2int.h"
#include "Maat_table_description.h"
#include "Maat_rule_internal.h"
#include "Maat_utils.h"

172
src/inc_internal/Maat_rule_internal.h
View File

@@ -3,6 +3,7 @@
#include "Maat_rule.h"
#include "Maat_command.h"
#include "Maat_table_description.h"
#include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h>
@@ -22,185 +23,24 @@
extern const char *maat_module;
#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411)
#define atomic_inc(x) __sync_add_and_fetch((x),1)
#define atomic_dec(x) __sync_sub_and_fetch((x),1)
#define atomic_add(x,y) __sync_add_and_fetch((x),(y))
#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y))
typedef int atomic_t;
#define ATOMIC_INIT(i) { (i) }
#define atomic_read(x) __sync_add_and_fetch((x),0)
#define atomic_set(x,y) __sync_lock_test_and_set((x),y)
#else
#include <alsa/iatomic.h>
#endif
#define TRUE 1
#define FALSE 0
#define MAX_TABLE_NUM 256
#define MAX_CONJUNCTION_TABLE_NUM 8
#define MAX_CHARSET_NUM 16
#define MAX_TABLE_NAME_LEN 256
#define MAX_TABLE_LINE_SIZE (1024*16)
#define MAX_EXPR_KEYLEN 1024
#define MAX_DISTRICT_LEN 64
#define MAX_PLUGIN_PER_TABLE 32
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_SCANNER_HIT_NUM 64
#define MAX_COMPILE_EX_DATA_NUM 2
#define MAX_GROUP_CACHE 128
#define MAX_FAILED_NUM 128
#define MAX_MAAT_STAT_NUM 64
#ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
#endif
#ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif
#ifndef container_of
#define container_of(ptr, type, member) ({ \
const typeof( ((type *)0)->member ) *__mptr = (ptr); \
(type *)( (char *)__mptr - offsetof(type,member) );})
#endif
typedef void* rule_scanner_t;
enum MAAT_TABLE_TYPE
{
TABLE_TYPE_EXPR=0,
TABLE_TYPE_IP,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_DIGEST,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_GROUP,
TABLE_TYPE_COMPILE,
TABLE_TYPE_PLUGIN
};
enum USER_REGION_ENCODE
{
USER_REGION_ENCODE_NONE=0,
USER_REGION_ENCODE_ESCAPE,
USER_REGION_ENCODE_BASE64
};
struct compile_ex_data_idx
{
Maat_rule_EX_new_func_t *new_func;
Maat_rule_EX_free_func_t* free_func;
Maat_rule_EX_dup_func_t* dup_func;
long argl;
void *argp;
int idx;
int table_id;
};
struct compile_table_desc
{
enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num;
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
};
struct plugin_table_callback_desc
{
Maat_start_callback_t *start;
Maat_update_callback_t *update;
Maat_finish_callback_t *finish;
void* u_para;
};
struct plugin_table_ex_data_desc
{
Maat_plugin_EX_new_func_t* new_func;
Maat_plugin_EX_free_func_t* free_func;
Maat_plugin_EX_dup_func_t* dup_func;
Maat_plugin_EX_key2index_func_t* key2index_func;
long argl;
void *argp;
MESA_htable_handle key2ex_hash;
};
struct plugin_table_desc
{
int key_column;
int valid_flag_column;
int rule_tag_column;
int n_foreign;
int foreign_columns[MAX_FOREIGN_CLMN_NUM];
int cb_plug_cnt;
long long estimate_size;
struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE];
struct plugin_table_ex_data_desc ex_desc;
};
struct expr_table_desc
{
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cross_cache_size;
int quick_expr_switch;
long long iconv_err_cnt;
};
struct ip_table_desc
{
int ipv4_rule_cnt;
int ipv6_rule_cnt;
};
struct Maat_table_desc
{
unsigned short table_id;
unsigned short conj_cnt;
unsigned short updating_name;
char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
enum MAAT_TABLE_TYPE table_type;
union
{
struct compile_table_desc compile;
struct expr_table_desc expr;
struct ip_table_desc ip;
struct plugin_table_desc plugin;
void* others;//group, interval and digest don't have sperate description info.
};
/*
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cross_cache_size;
int quick_expr_switch;
union
{
int expr_rule_cnt; //expr_type=0,1,3
int ipv4_rule_cnt;
};
union
{
int regex_rule_cnt; //expr_type=2
int ipv6_rule_cnt;
};
struct _plugin_table_info *cb_info;
int valid_flag_column; //for plugin table
int rule_tag_column; //for plugin table;
int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table;
int n_foreign;
//for compile table
enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num;
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
*/
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
unsigned long long unmatch_tag_cnt;
int stat_line_id;
};
struct db_str_rule_t
{
@@ -380,6 +220,7 @@ struct plugin_runtime
long long cache_line_num;
long long acc_line_num;
long long cache_size;
pthread_rwlock_t rwlock;
MESA_htable_handle key2ex_hash;
};
struct expr_runtime
@@ -595,7 +436,7 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set);
int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
void maat_start_cb(long long new_version,int update_type,void*u_para);
int maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para);
@@ -620,6 +461,9 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p);
void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len);
MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc);
void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc);
MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index);
int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger);
int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger);
void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout);

113
src/inc_internal/Maat_table_description.h Normal file
View File

@@ -0,0 +1,113 @@
#pragma once
#include "Maat_rule.h"
#define MAX_COMPILE_EX_DATA_NUM 2
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_PLUGIN_PER_TABLE 32
#define MAX_CHARSET_NUM 16
#define MAX_CONJUNCTION_TABLE_NUM 8
#define MAX_TABLE_NAME_LEN 256
enum USER_REGION_ENCODE
{
USER_REGION_ENCODE_NONE=0,
USER_REGION_ENCODE_ESCAPE,
USER_REGION_ENCODE_BASE64
};
enum MAAT_TABLE_TYPE
{
TABLE_TYPE_EXPR=0,
TABLE_TYPE_IP,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_DIGEST,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_GROUP,
TABLE_TYPE_COMPILE,
TABLE_TYPE_PLUGIN
};
struct compile_ex_data_idx
{
Maat_rule_EX_new_func_t *new_func;
Maat_rule_EX_free_func_t* free_func;
Maat_rule_EX_dup_func_t* dup_func;
long argl;
void *argp;
int idx;
int table_id;
};
struct compile_table_desc
{
enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num;
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
};
struct plugin_table_callback_desc
{
Maat_start_callback_t *start;
Maat_update_callback_t *update;
Maat_finish_callback_t *finish;
void* u_para;
};
struct plugin_table_ex_data_desc
{
Maat_plugin_EX_new_func_t* new_func;
Maat_plugin_EX_free_func_t* free_func;
Maat_plugin_EX_dup_func_t* dup_func;
Maat_plugin_EX_key2index_func_t* key2index_func;
long argl;
void *argp;
};
struct plugin_table_desc
{
int key_column;
int valid_flag_column;
int rule_tag_column;
int n_foreign;
int foreign_columns[MAX_FOREIGN_CLMN_NUM];
int cb_plug_cnt;
int have_exdata;
long long estimate_size;
struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE];
struct plugin_table_ex_data_desc ex_desc;
};
struct expr_table_desc
{
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cross_cache_size;
int quick_expr_switch;
long long iconv_err_cnt;
};
struct ip_table_desc
{
int ipv4_rule_cnt;
int ipv6_rule_cnt;
};
struct Maat_table_desc
{
int table_id;
int conj_cnt;
int updating_name;
char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
enum MAAT_TABLE_TYPE table_type;
union
{
struct compile_table_desc compile;
struct expr_table_desc expr;
struct ip_table_desc ip;
struct plugin_table_desc plugin;
void* others;//group, interval and digest don't have sperate description info.
};
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
unsigned long long unmatch_tag_cnt;
int stat_line_id;
};

27
src/inc_internal/Maat_utils.h
View File

@@ -9,9 +9,20 @@
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
#define FREE(p) {free(*p);*p=NULL;}
#define ATOMIC_INC(x) __atomic_fetch_add(x,1,__ATOMIC_RELAXED)
#define ATOMIC_READ(x) __atomic_fetch_add(x,0,__ATOMIC_RELAXED)
#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411)
#define atomic_inc(x) __sync_add_and_fetch((x),1)
#define atomic_dec(x) __sync_sub_and_fetch((x),1)
#define atomic_add(x,y) __sync_add_and_fetch((x),(y))
#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y))
typedef int atomic_t;
#define ATOMIC_INIT(i) { (i) }
#define atomic_read(x) __sync_add_and_fetch((x),0)
#define atomic_set(x,y) __sync_lock_test_and_set((x),y)
#else
#include <alsa/iatomic.h>
#endif
#define TRUE 1
#define FALSE 0
#ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
@@ -21,6 +32,16 @@
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif
#ifndef container_of
#define container_of(ptr, type, member) ({ \
const typeof( ((type *)0)->member ) *__mptr = (ptr); \
(type *)( (char *)__mptr - offsetof(type,member) );})
#endif
#define UNUSED __attribute__((unused))
const char* module_name_str(const char*name);
#define maat_module (module_name_str("MAAT_Frame"))