gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完成单元测试。

Browse Source
This commit is contained in:
zhengchao
2018-12-05 18:00:55 +08:00
parent 7b5baacf62
commit 899a61e371
11 changed files with 466 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
inc/Maat_rule.h
View File

@@ -259,8 +259,8 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa
//Following functions are similar to Maat_rule_get_ex_data, except they are effective on plugin table.
typedef void* MAAT_PLUGIN_EX_DATA;
typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, size_t key_sz, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);
typedef void Maat_plugin_EX_free_func_t(int table_id, const char* key, size_t key_sz, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);
typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);
typedef void Maat_plugin_EX_free_func_t(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);
typedef void Maat_plugin_EX_dup_func_t(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp);
typedef int Maat_plugin_EX_key2index_func_t(const char* key);
@@ -271,7 +271,7 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp);
//Data is duplicated by dup_func of Maat_plugin_EX_register, caller is responsible to free the data.
MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key, int thread_num);
MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key);
enum MAAT_RULE_OPT
{

176
src/entry/Maat_api.cpp
View File

@@ -443,7 +443,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
return NULL;
}
_Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1);
feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger);
feather->table_cnt=read_table_description(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger);
feather->map_tablename2id=map_create();
int i=0,j=0,ret=0;
for(i=0;i<MAX_TABLE_NUM;i++)
@@ -930,7 +930,6 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
{
return -1;
}
plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin);
//plugin table register blocks background update.
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
idx=plugin_desc->cb_plug_cnt;
@@ -944,6 +943,12 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
plugin_desc->cb_plug[idx].update=update;
plugin_desc->cb_plug[idx].finish=finish;
plugin_desc->cb_plug[idx].u_para=u_para;
if(_feather->scanner==NULL)
{
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return 1;
}
plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin);
if(plugin_aux->cache_line_num>0)
{
if(start!=NULL)
@@ -1051,46 +1056,128 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa
pthread_rwlock_unlock(&(compile_inner->rwlock));
return ad;
}
MESA_htable_handle plugin_EX_htable_new(const struct plugin_table_desc* plugin_desc,
struct dynamic_array_t* lines, size_t line_cnt)
struct wrap_plugin_EX_data
{
MAAT_RULE_EX_DATA exdata;
const struct Maat_table_desc* ref_plugin_table;
};
void wrap_plugin_EX_data_free(void *data)
{
struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data;
const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc);
ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp);
wrap_data->ref_plugin_table=NULL;
free(wrap_data);
return;
}
MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index)
{
MESA_htable_handle key2ex_hash=NULL;
unsigned int slot_size=1;
const char* line=NULL;
size_t key_offset=0, key_len=0;
long long estimate_size=plugin_desc->estimate_size, i=0;
int ret=0;
MAAT_RULE_EX_DATA exdata=NULL;
while(estimate_size>0)
while(estimate_size!=0)
{
estimate_size=estimate_size<<1;
estimate_size=estimate_size>>1;
slot_size*=2;
}
if(slot_size==1)
{
slot_size=4096;
}
MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs));
hargs.thread_safe=1;
hargs.thread_safe=0;
hargs.hash_slot_size = slot_size;
hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
hargs.expire_time = 0;
hargs.key_comp = NULL;
hargs.key2index = plugin_desc->ex_desc.key2index_func;
hargs.key2index = NULL; //Not supported yet.
hargs.recursive = 1;
hargs.data_free = NULL ;
hargs.data_free = wrap_plugin_EX_data_free;
hargs.data_expire_with_condition = NULL;
key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(key2ex_hash, 0);
return key2ex_hash;
}
int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line,
MESA_htable_handle key2ex_hash, void *logger)
{
size_t key_offset=0, key_len=0;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
int ret=0;
ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data del error: cannot find column %d of %s",
plugin_desc->key_column, line);
return -1;
}
ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data del error: no such key %.*s of %s",
key_len, line+key_offset, line);
return -1;
}
return 0;
}
int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line,
MESA_htable_handle key2ex_hash, void *logger)
{
char* key=NULL;
size_t key_offset=0, key_len=0;
MAAT_RULE_EX_DATA exdata=NULL;
struct wrap_plugin_EX_data* wrap_data=NULL;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
int ret=0;
ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data add error: cannot find column %d of %s",
plugin_desc->key_column, line);
return -1;
}
key=ALLOC(char, key_len+1);
memcpy(key, line+key_offset, key_len);
plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata,
plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
wrap_data=ALLOC(struct wrap_plugin_EX_data, 1);
wrap_data->exdata=exdata;
wrap_data->ref_plugin_table=plugin_table;
ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data);
free(key);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Plugin EX data add error: duplicated key %.*s of %s",
key_len, line+key_offset, line);
wrap_plugin_EX_data_free(wrap_data);
return -1;
}
return 0;
}
MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table,
struct dynamic_array_t* lines, size_t line_cnt, void* logger)
{
MESA_htable_handle key2ex_hash=NULL;
size_t i=0;
const char* line=NULL;
const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func);
for(i=0; i< line_cnt; i++)
{
line=(const char*)dynamic_array_read(lines, i);
ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
exdata=plugin_desc->ex_desc.new_func(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);;
MESA_htable_add(key2ex_hash, line+key_offset, key_len, const void * data)
plugin_EX_data_new(plugin_table, line, key2ex_hash, logger);
}
return key2ex_hash;
}
int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
@@ -1103,26 +1190,71 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_table_desc *table_desc=_feather->p_table_info[table_id];
struct plugin_table_desc* plugin_desc=&(table_desc->plugin);
struct Maat_table_runtime* table_rt=_feather->scanner->table_rt[table_id];
MESA_htable_handle key2ex_hash=NULL;
if(table_desc->table_type!=TABLE_TYPE_PLUGIN || new_func==NULL || free_func==NULL || dup_func==NULL)
struct Maat_table_runtime* table_rt=NULL;
if(new_func==NULL || free_func==NULL || dup_func==NULL )
{
assert(0);
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
return -1;
}
if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata
|| plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0)
{
assert(0);
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module,
"%s failed: key or valid flag column are not specified", __FUNCTION__);
return -1;
}
plugin_desc->ex_desc.new_func=new_func;
plugin_desc->ex_desc.free_func=free_func;
plugin_desc->ex_desc.dup_func=dup_func;
plugin_desc->ex_desc.key2index_func=key2index_func;
plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used.
plugin_desc->ex_desc.argl=argl;
plugin_desc->ex_desc.argp=argp;
plugin_desc->have_exdata=1;
if(_feather->scanner==NULL)
{
return 0;
}
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
table_rt->plugin.key2ex_hash=plugin_EX_htable_new(plugin_desc,table_rt->plugin.cache_lines, table_rt->plugin.cache_line_num);
table_rt=_feather->scanner->table_rt[table_id];
assert(table_rt->plugin.key2ex_hash==NULL);
table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines,
table_rt->plugin.cache_line_num, _feather->logger);
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return 0;
}
MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_table_desc *table_desc=_feather->p_table_info[table_id];
struct Maat_table_runtime *table_rt= NULL;
struct plugin_table_desc* plugin_desc=&(table_desc->plugin);
struct wrap_plugin_EX_data* wrap_data=NULL;
MAAT_RULE_EX_DATA exdata=NULL;
if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata==0)
{
assert(0);
return NULL;
}
if(_feather->scanner==NULL)
{
return NULL;
}
table_rt= _feather->scanner->table_rt[table_id];
pthread_rwlock_rdlock(&table_rt->plugin.rwlock);
wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash,
(const unsigned char*)key, strlen(key));
if(wrap_data!=NULL)
{
plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata),
plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
}
pthread_rwlock_unlock(&table_rt->plugin.rwlock);
return exdata;
}
int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
,enum MAAT_CHARSET charset,const char* data,int data_len

7
src/entry/Maat_command.cpp
View File

@@ -221,7 +221,7 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_
assert(0);
}
ret=get_column_pos(const char* line, int column_seq, &offset, &len);
ret=get_column_pos(line, column_seq, &offset, &len);
if(ret<0||offset>=strlen(line)||(line[offset]!='1'&&line[offset]!='0'))// 0 is also a valid value for some non-MAAT producer.
{
return -1;
@@ -2131,6 +2131,11 @@ int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value,
int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value, size_t size, enum MAAT_OPERATION op)
{
struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather;
redisContext* ctx=_feather->mr_ctx.write_ctx;
if(ctx==NULL)
{
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, "%s failed: Redis is not connected.", __FUNCTION__);
return -1;
}
const char *arg_vec[3];
size_t len_vec[3];

84
src/entry/Maat_rule.cpp
View File

@@ -529,7 +529,7 @@ int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_h
char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0};
char *token=NULL,*sub_token=NULL,*saveptr;
struct expr_table_desc* p=&(table->expr);
sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id)
sscanf(line,"%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id)
,table->table_name[0]
,table_type
,src_charset
@@ -602,31 +602,21 @@ int _read_integer_arrary(char* string, int *array, int size)
return i;
}
int read_plugin_table_info(const char* line, struct Maat_table_desc* p)
int read_plugin_table_description(const char* line, struct Maat_table_desc* p)
{
int i=0,ret=0;
size_t offset=0, len=0;
cJSON* json=NULL, *tmp=NULL, *array_item=NULL;
char* copy_line=NULL, *plug_info=NULL;
char *token=NULL,*sub_token=NULL,*saveptr;
struct plugin_table_desc* plugin_desc=&(p->plugin);
plugin_desc->valid_flag_column=-1;
copy_line=_maat_strdup(line);
for (token = copy_line, i=0; ; token= NULL, i++)
{
sub_token= strtok_r(token,"\t ", &saveptr);
if (sub_token == NULL)
break;
if(i==3)
{
break;
}
}
if(i<3)
ret=get_column_pos(copy_line, 4, &offset, &len);
if(i<0)
{
goto error_out;
}
plug_info=sub_token;
copy_line[offset+len+1]='\0';
plug_info=copy_line+offset;
if(strlen(plug_info)<4)//For old version compatible.
{
@@ -693,7 +683,7 @@ error_out:
free(copy_line);
return -1;
}
int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger)
int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger)
{
FILE*fp=NULL;
char line[MAX_TABLE_LINE_SIZE];
@@ -750,7 +740,7 @@ int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* ta
}
p=table_info_new(max_thread_num);
ret=sscanf(line,"%hu\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id)
ret=sscanf(line,"%d\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id)
,p->table_name[0]
,table_type_str
,not_care);
@@ -781,7 +771,7 @@ int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* ta
}
break;
case TABLE_TYPE_PLUGIN:
ret=read_plugin_table_info(line, p);
ret=read_plugin_table_description(line, p);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i);
@@ -1193,29 +1183,36 @@ void destroy_digest_rule(GIE_digest_t*rule)
}
struct Maat_table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_thread_num)
struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num)
{
struct Maat_table_runtime* p= ALLOC(struct Maat_table_runtime, 1);
p->table_type=type;
switch(type)
struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1);
table_rt->table_type=table_desc->table_type;
switch(table_desc->table_type)
{
case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY:
p->similar.update_q=MESA_lqueue_create(0,0);
table_rt->similar.update_q=MESA_lqueue_create(0,0);
break;
case TABLE_TYPE_PLUGIN:
p->plugin.cache_lines=dynamic_array_create(1, 1024);
table_rt->plugin.cache_lines=dynamic_array_create(1, 1024);
if(table_desc->plugin.have_exdata)
{
table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size,
table_desc->plugin.ex_desc.key2index_func);
pthread_rwlock_init(&(table_rt->plugin.rwlock), NULL);
}
break;
default:
break;
}
p->scan_cnt=alignment_int64_array_alloc(max_thread_num);
p->scan_cpu_time=alignment_int64_array_alloc(max_thread_num);
p->input_bytes=alignment_int64_array_alloc(max_thread_num);
p->stream_num=alignment_int64_array_alloc(max_thread_num);
p->hit_cnt=alignment_int64_array_alloc(max_thread_num);
return p;
table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num);
table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num);
table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num);
table_rt->stream_num=alignment_int64_array_alloc(max_thread_num);
table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num);
return table_rt;
}
void table_runtime_free(struct Maat_table_runtime* p)
{
@@ -1344,7 +1341,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
{
continue;
}
table_rt=table_runtime_new(pp_table_desc[i]->table_type, feather->scan_thread_num);
table_rt=table_runtime_new(pp_table_desc[i], feather->scan_thread_num);
if(pp_table_desc[i]->table_type==TABLE_TYPE_EXPR||pp_table_desc[i]->table_type==TABLE_TYPE_EXPR_PLUS)
{
expr_desc=&(pp_table_desc[i]->expr);
@@ -3178,7 +3175,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
char *p=NULL;
char* copy=NULL;
size_t is_valid_offset=0, valid_len=0;
char *token=NULL,*sub_token=NULL,*saveptr;
if(plugin_desc->rule_tag_column>0&&n_tags>0)
{
@@ -3213,7 +3210,24 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M
}
table_rt->plugin.acc_line_num++;
if(plugin_desc->have_exdata || plugin_desc->cb_plug_cnt>0)
{
if(plugin_desc->have_exdata)
{
ret=get_column_pos(table_line, plugin_desc->valid_flag_column, &is_valid_offset, &valid_len);
pthread_rwlock_wrlock(&(table_rt->plugin.rwlock));
if(atoi(table_line+is_valid_offset)==1)
{
plugin_EX_data_new(table, table_line, table_rt->plugin.key2ex_hash, logger);
}
else
{
plugin_EX_data_free(table, table_line, table_rt->plugin.key2ex_hash, logger);
}
pthread_rwlock_unlock(&(table_rt->plugin.rwlock));
}
if(plugin_desc->cb_plug_cnt>0)
{
for(i=0;i<plugin_desc->cb_plug_cnt;i++)
@@ -3221,6 +3235,8 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M
plugin_desc->cb_plug[i].update(table->table_id,table_line,plugin_desc->cb_plug[i].u_para);
}
}
}
else
{
p=ALLOC(char, len);

2
src/entry/Maat_utils.cpp
View File

@@ -179,7 +179,7 @@ int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len
{
*offset=subtoken-dup_line;
*len=strlen(subtoken);
ret=0
ret=0;
break;
}
i++;

1
src/entry/json2iris.cpp
View File

@@ -12,6 +12,7 @@
#include "cJSON.h"
#include "hiredis.h"
#include "map_str2int.h"
#include "Maat_table_description.h"
#include "Maat_rule_internal.h"
#include "Maat_utils.h"

172
src/inc_internal/Maat_rule_internal.h
View File

@@ -3,6 +3,7 @@
#include "Maat_rule.h"
#include "Maat_command.h"
#include "Maat_table_description.h"
#include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h>
@@ -22,185 +23,24 @@
extern const char *maat_module;
#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411)
#define atomic_inc(x) __sync_add_and_fetch((x),1)
#define atomic_dec(x) __sync_sub_and_fetch((x),1)
#define atomic_add(x,y) __sync_add_and_fetch((x),(y))
#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y))
typedef int atomic_t;
#define ATOMIC_INIT(i) { (i) }
#define atomic_read(x) __sync_add_and_fetch((x),0)
#define atomic_set(x,y) __sync_lock_test_and_set((x),y)
#else
#include <alsa/iatomic.h>
#endif
#define TRUE 1
#define FALSE 0
#define MAX_TABLE_NUM 256
#define MAX_CONJUNCTION_TABLE_NUM 8
#define MAX_CHARSET_NUM 16
#define MAX_TABLE_NAME_LEN 256
#define MAX_TABLE_LINE_SIZE (1024*16)
#define MAX_EXPR_KEYLEN 1024
#define MAX_DISTRICT_LEN 64
#define MAX_PLUGIN_PER_TABLE 32
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_SCANNER_HIT_NUM 64
#define MAX_COMPILE_EX_DATA_NUM 2
#define MAX_GROUP_CACHE 128
#define MAX_FAILED_NUM 128
#define MAX_MAAT_STAT_NUM 64
#ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
#endif
#ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif
#ifndef container_of
#define container_of(ptr, type, member) ({ \
const typeof( ((type *)0)->member ) *__mptr = (ptr); \
(type *)( (char *)__mptr - offsetof(type,member) );})
#endif
typedef void* rule_scanner_t;
enum MAAT_TABLE_TYPE
{
TABLE_TYPE_EXPR=0,
TABLE_TYPE_IP,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_DIGEST,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_GROUP,
TABLE_TYPE_COMPILE,
TABLE_TYPE_PLUGIN
};
enum USER_REGION_ENCODE
{
USER_REGION_ENCODE_NONE=0,
USER_REGION_ENCODE_ESCAPE,
USER_REGION_ENCODE_BASE64
};
struct compile_ex_data_idx
{
Maat_rule_EX_new_func_t *new_func;
Maat_rule_EX_free_func_t* free_func;
Maat_rule_EX_dup_func_t* dup_func;
long argl;
void *argp;
int idx;
int table_id;
};
struct compile_table_desc
{
enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num;
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
};
struct plugin_table_callback_desc
{
Maat_start_callback_t *start;
Maat_update_callback_t *update;
Maat_finish_callback_t *finish;
void* u_para;
};
struct plugin_table_ex_data_desc
{
Maat_plugin_EX_new_func_t* new_func;
Maat_plugin_EX_free_func_t* free_func;
Maat_plugin_EX_dup_func_t* dup_func;
Maat_plugin_EX_key2index_func_t* key2index_func;
long argl;
void *argp;
MESA_htable_handle key2ex_hash;
};
struct plugin_table_desc
{
int key_column;
int valid_flag_column;
int rule_tag_column;
int n_foreign;
int foreign_columns[MAX_FOREIGN_CLMN_NUM];
int cb_plug_cnt;
long long estimate_size;
struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE];
struct plugin_table_ex_data_desc ex_desc;
};
struct expr_table_desc
{
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cross_cache_size;
int quick_expr_switch;
long long iconv_err_cnt;
};
struct ip_table_desc
{
int ipv4_rule_cnt;
int ipv6_rule_cnt;
};
struct Maat_table_desc
{
unsigned short table_id;
unsigned short conj_cnt;
unsigned short updating_name;
char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
enum MAAT_TABLE_TYPE table_type;
union
{
struct compile_table_desc compile;
struct expr_table_desc expr;
struct ip_table_desc ip;
struct plugin_table_desc plugin;
void* others;//group, interval and digest don't have sperate description info.
};
/*
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cross_cache_size;
int quick_expr_switch;
union
{
int expr_rule_cnt; //expr_type=0,1,3
int ipv4_rule_cnt;
};
union
{
int regex_rule_cnt; //expr_type=2
int ipv6_rule_cnt;
};
struct _plugin_table_info *cb_info;
int valid_flag_column; //for plugin table
int rule_tag_column; //for plugin table;
int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table;
int n_foreign;
//for compile table
enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num;
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
*/
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
unsigned long long unmatch_tag_cnt;
int stat_line_id;
};
struct db_str_rule_t
{
@@ -380,6 +220,7 @@ struct plugin_runtime
long long cache_line_num;
long long acc_line_num;
long long cache_size;
pthread_rwlock_t rwlock;
MESA_htable_handle key2ex_hash;
};
struct expr_runtime
@@ -595,7 +436,7 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set);
int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
void maat_start_cb(long long new_version,int update_type,void*u_para);
int maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para);
@@ -620,6 +461,9 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p);
void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len);
MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc);
void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc);
MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index);
int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger);
int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger);
void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout);

113
src/inc_internal/Maat_table_description.h Normal file
View File

@@ -0,0 +1,113 @@
#pragma once
#include "Maat_rule.h"
#define MAX_COMPILE_EX_DATA_NUM 2
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_PLUGIN_PER_TABLE 32
#define MAX_CHARSET_NUM 16
#define MAX_CONJUNCTION_TABLE_NUM 8
#define MAX_TABLE_NAME_LEN 256
enum USER_REGION_ENCODE
{
USER_REGION_ENCODE_NONE=0,
USER_REGION_ENCODE_ESCAPE,
USER_REGION_ENCODE_BASE64
};
enum MAAT_TABLE_TYPE
{
TABLE_TYPE_EXPR=0,
TABLE_TYPE_IP,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_DIGEST,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_GROUP,
TABLE_TYPE_COMPILE,
TABLE_TYPE_PLUGIN
};
struct compile_ex_data_idx
{
Maat_rule_EX_new_func_t *new_func;
Maat_rule_EX_free_func_t* free_func;
Maat_rule_EX_dup_func_t* dup_func;
long argl;
void *argp;
int idx;
int table_id;
};
struct compile_table_desc
{
enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num;
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
};
struct plugin_table_callback_desc
{
Maat_start_callback_t *start;
Maat_update_callback_t *update;
Maat_finish_callback_t *finish;
void* u_para;
};
struct plugin_table_ex_data_desc
{
Maat_plugin_EX_new_func_t* new_func;
Maat_plugin_EX_free_func_t* free_func;
Maat_plugin_EX_dup_func_t* dup_func;
Maat_plugin_EX_key2index_func_t* key2index_func;
long argl;
void *argp;
};
struct plugin_table_desc
{
int key_column;
int valid_flag_column;
int rule_tag_column;
int n_foreign;
int foreign_columns[MAX_FOREIGN_CLMN_NUM];
int cb_plug_cnt;
int have_exdata;
long long estimate_size;
struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE];
struct plugin_table_ex_data_desc ex_desc;
};
struct expr_table_desc
{
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cross_cache_size;
int quick_expr_switch;
long long iconv_err_cnt;
};
struct ip_table_desc
{
int ipv4_rule_cnt;
int ipv6_rule_cnt;
};
struct Maat_table_desc
{
int table_id;
int conj_cnt;
int updating_name;
char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
enum MAAT_TABLE_TYPE table_type;
union
{
struct compile_table_desc compile;
struct expr_table_desc expr;
struct ip_table_desc ip;
struct plugin_table_desc plugin;
void* others;//group, interval and digest don't have sperate description info.
};
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
unsigned long long unmatch_tag_cnt;
int stat_line_id;
};

27
src/inc_internal/Maat_utils.h
View File

@@ -9,9 +9,20 @@
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
#define FREE(p) {free(*p);*p=NULL;}
#define ATOMIC_INC(x) __atomic_fetch_add(x,1,__ATOMIC_RELAXED)
#define ATOMIC_READ(x) __atomic_fetch_add(x,0,__ATOMIC_RELAXED)
#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411)
#define atomic_inc(x) __sync_add_and_fetch((x),1)
#define atomic_dec(x) __sync_sub_and_fetch((x),1)
#define atomic_add(x,y) __sync_add_and_fetch((x),(y))
#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y))
typedef int atomic_t;
#define ATOMIC_INIT(i) { (i) }
#define atomic_read(x) __sync_add_and_fetch((x),0)
#define atomic_set(x,y) __sync_lock_test_and_set((x),y)
#else
#include <alsa/iatomic.h>
#endif
#define TRUE 1
#define FALSE 0
#ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
@@ -21,6 +32,16 @@
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif
#ifndef container_of
#define container_of(ptr, type, member) ({ \
const typeof( ((type *)0)->member ) *__mptr = (ptr); \
(type *)( (char *)__mptr - offsetof(type,member) );})
#endif
#define UNUSED __attribute__((unused))
const char* module_name_str(const char*name);
#define maat_module (module_name_str("MAAT_Frame"))

3
test/table_info.conf
View File

@@ -28,5 +28,6 @@
9 SIM_URL similar --
10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff
11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} --
12 TEST_FOREIGN_KEY plugin {"key":2,"valid":4,"foreign":[6,8],"tag":3, "estimate_size":4096} --
12 TEST_FOREIGN_KEY plugin {"valid":4,"foreign":[6,8],"tag":3} --
13 COMPILE_ALIAS compile escape --
14 TEST_PLUGIN_EXDATA_TABLE plugin {"key":2,"valid":4,"tag":5,"estimate_size":1024} --

106
test/test_maatframe.cpp
View File

@@ -640,7 +640,7 @@ struct rule_ex_param
pthread_mutex_t lock;
};
void ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
void compile_ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
MAAT_RULE_EX_DATA* ad, long argl, void *argp)
{
int *counter=(int*)argp;
@@ -657,7 +657,7 @@ void ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_l
*ad=param;
return;
}
void ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp)
void compile_ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp)
{
if(*ad==NULL)
{
@@ -674,7 +674,7 @@ void ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_
free(param);
return;
}
void ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp)
void compile_ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp)
{
struct rule_ex_param* from_param=*((struct rule_ex_param**)from);
pthread_mutex_lock(&(from_param->lock));
@@ -686,9 +686,9 @@ void ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long
TEST(EX_DATA_INDEX, all)
TEST(EX_DATA_INDEX, MaatRuleEXData)
{
#define EX_data_index
#define rule_EX_data_index
int ret=0;
int table_id=0, ex_data_counter=0;
@@ -701,7 +701,7 @@ TEST(EX_DATA_INDEX, all)
ASSERT_GT(table_id, 0);
int ex_param_idx=Maat_rule_get_ex_new_index(g_feather, "COMPILE_ALIAS",
ex_param_new, ex_param_free, ex_param_dup,
compile_ex_param_new, compile_ex_param_free, compile_ex_param_dup,
0, &ex_data_counter);
ASSERT_TRUE(ex_param_idx>=0);
EXPECT_EQ(ex_data_counter, 1);
@@ -716,7 +716,7 @@ TEST(EX_DATA_INDEX, all)
struct rule_ex_param* param=(struct rule_ex_param*)ex_data;
EXPECT_EQ(param->id, 7799);
EXPECT_EQ(strcmp(param->name, expect_name),0);
ex_param_free(0, NULL, NULL, &ex_data, 0, NULL);
compile_ex_param_free(0, NULL, NULL, &ex_data, 0, NULL);
Maat_clean_status(&mid);
return;
}
@@ -1418,6 +1418,98 @@ TEST_F(MaatCmdTest, SetFile)
return;
}
struct user_info
{
char name[256];
char ip_addr[32];
int id;
int ref_cnt;
};
void plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{
int *counter=(int *)argp;
struct user_info* u=(struct user_info*)calloc(sizeof(struct user_info), 1);
int valid=0, tag=0, ret;
ret=sscanf(table_line, "%d\t%s\t%s%d\t%d", &(u->id), u->ip_addr, u->name, &valid, &tag);
EXPECT_EQ(ret, 5);
u->ref_cnt=1;
*ad=u;
(*counter)++;
return;
}
void plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{
struct user_info* u=(struct user_info*)(*ad);
u->ref_cnt--;
if(u->ref_cnt>0) return;
free(u);
*ad=NULL;
}
void plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
{
struct user_info* u=(struct user_info*)(*from);
u->ref_cnt++;
*to=u;
}
TEST_F(MaatCmdTest, PluginEXData)
{
#define plugin_EX_data_index
Maat_feather_t feather=MaatCmdTest::_shared_feather;
int ret=0, i=0;
int table_id=0, ex_data_counter=0;
const char* table_name="TEST_PLUGIN_EXDATA_TABLE";
const int TEST_CMD_LINE_NUM=4;
const struct Maat_line_t *p_line[TEST_CMD_LINE_NUM];
struct Maat_line_t line_rule[TEST_CMD_LINE_NUM];
const char* table_line[TEST_CMD_LINE_NUM]={"1\t192.168.0.1\tmahuateng\t1\t0",
"2\t192.168.0.2\tliuqiangdong\t1\t0",
"3\t192.168.0.3\tmayun\t1\t0",
"4\t192.168.0.4\tliyanhong\t1\t0"};
table_id=Maat_table_register(feather, table_name);
ASSERT_GT(table_id, 0);
memset(&line_rule,0,sizeof(line_rule));
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
line_rule[i].label_id=0;
line_rule[i].rule_id=(int)Maat_cmd_incrby(feather,"TEST_PLUG_SEQ", 1);
line_rule[i].table_name=table_name;
line_rule[i].table_line=table_line[i];
line_rule[i].expire_after=0;
p_line[i]=line_rule+i;
}
ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
EXPECT_GT(ret, 0);
usleep(WAIT_FOR_EFFECTIVE_US);
ret=Maat_plugin_EX_register(feather, table_id,
plugin_EX_new_cb,
plugin_EX_free_cb,
plugin_EX_dup_cb,
NULL,
0, &ex_data_counter);
ASSERT_TRUE(ret>=0);
EXPECT_EQ(ex_data_counter, TEST_CMD_LINE_NUM);
struct user_info* uinfo=NULL;
uinfo=(struct user_info*)Maat_plugin_get_EX_data(feather, table_id, "192.168.0.2");
ASSERT_TRUE(uinfo!=NULL);
EXPECT_EQ(0, strcmp(uinfo->name, "liuqiangdong"));
EXPECT_EQ(uinfo->id, 2);
plugin_EX_free_cb(table_id, (void**)&uinfo, 0, NULL);
ret=Maat_cmd_set_lines(feather, p_line+1,1, MAAT_OP_DEL);
EXPECT_GT(ret, 0);
usleep(WAIT_FOR_EFFECTIVE_US);
uinfo=(struct user_info*)Maat_plugin_get_EX_data(feather, table_id, "192.168.0.2");
ASSERT_TRUE(uinfo==NULL);
return;
}
int main(int argc, char ** argv)
{