gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

支持内容外键，即某一列指向redis中的一个key，将其变成文件路径。

Browse Source
This commit is contained in:
zhengchao
2018-09-24 18:49:18 +08:00
parent 16ff0886c9
commit e2f4a583ad
11 changed files with 656 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
inc/Maat_rule.h
View File

@@ -141,10 +141,10 @@ enum MAAT_INIT_OPT
MAAT_OPT_FULL_CFG_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default.
MAAT_OPT_INC_CFG_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default.
MAAT_OPT_JSON_FILE_PATH, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default.
MAAT_OPT_STAT_ON, //VALUE is NULL,SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF.
MAAT_OPT_PERF_ON, //VALUE is NULL,SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF.
MAAT_OPT_STAT_ON, //VALUE is NULL, SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF.
MAAT_OPT_PERF_ON, //VALUE is NULL, SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF.
MAAT_OPT_STAT_FILE_PATH, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. DEFAULT: no default.
MAAT_OPT_SCAN_DETAIL, //VALUE is interger *, SIZE=sizeof(int). 0: not return any detail;1: return hit pos, not include regex grouping;
MAAT_OPT_SCAN_DETAIL, //VALUE is interger *, SIZE=sizeof(int). 0: not return any detail;1: return hit pos, not include regex grouping.
// 2 return hit pos and regex grouping pos;DEFAULT:0
MAAT_OPT_INSTANCE_NAME, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1, no more than 11 bytes.DEFAULT: MAAT_$tableinfo_path$.
MAAT_OPT_DECRYPT_KEY, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. No DEFAULT.
@@ -157,8 +157,10 @@ enum MAAT_INIT_OPT
MAAT_OPT_LOAD_VERSION_FROM, //VALUE is a long long, SIZE=sizeof(long long). Default: Load the Latest. Only valid in redis mode, and maybe failed for too old.
//This option also disables background update.
MAAT_OPT_ENABLE_UPDATE, //VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled. DEFAULT: Backgroud update is enabled. Runtime setting is allowed.
MAAT_OPT_ACCEPT_TAGS //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]}
};
MAAT_OPT_ACCEPT_TAGS, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]}
MAAT_OPT_FOREIGN_CONT_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files
MAAT_OPT_FOREIGN_CONT_LINGER //VALUE is interger *, SIZE=sizeof(int). Greater than 0: delete after VALUE seconds; 0: delete foreign content right after the notification callbacks; Less than 0: NEVER delete. Default: 0.
};
//return -1 if failed, return 0 on success;
int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size);
enum MAAT_STATE_OPT

2
src/CMakeLists.txt
View File

@@ -9,7 +9,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/UniversalBoolMatch.cpp)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/UniversalBoolMatch.cpp)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
include_directories(/opt/MESA/include/MESA/)

15
src/entry/Maat_api.cpp
View File

@@ -8,6 +8,7 @@
#include "UniversalBoolMatch.h"
#include "Maat_rule.h"
#include "Maat_rule_internal.h"
#include "Maat_utils.h"
#include "dynamic_array.h"
#include "aligment_int64.h"
#include "config_monitor.h"
@@ -486,6 +487,8 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
feather->connect_timeout.tv_sec=0;
feather->connect_timeout.tv_usec=100*1000; // 100 ms
feather->backgroud_update_enabled=1;
feather->foreign_cont_linger=0;
snprintf(feather->foreign_cont_dir, sizeof(feather->foreign_cont_dir), "%s_files", table_info_path);
pthread_mutex_init(&(feather->backgroud_update_mutex),NULL);
pthread_mutex_init(&(feather->redis_write_lock),NULL);
snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path);
@@ -673,6 +676,16 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
return -1;
}
break;
case MAAT_OPT_FOREIGN_CONT_DIR:
strncpy(_feather->foreign_cont_dir, (char*)value, sizeof(_feather->foreign_cont_dir));
if(_feather->foreign_cont_dir[size-1]=='/')
{
_feather->foreign_cont_dir[size-1]='\0';
}
break;
case MAAT_OPT_FOREIGN_CONT_LINGER:
_feather->foreign_cont_linger=*(int*)value;
break;
default:
return -1;
}
@@ -904,7 +917,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
//plugin table register blocks background update.
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
idx=p_table->cb_info->cb_plug_cnt;
if(idx==MAX_PLUGING_NUM)
if(idx==MAX_PLUGIN_PER_TABLE)
{
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return -1;

256
src/entry/Maat_command.cpp
View File

@@ -1,6 +1,7 @@
#include "Maat_command.h"
#include "Maat_rule.h"
#include "Maat_rule_internal.h"
#include "Maat_utils.h"
#include "config_monitor.h"
#include "map_str2int.h"
#include "hiredis.h"
@@ -292,7 +293,15 @@ void empty_serial_rules(struct serial_rule_t* rule)
if(rule->table_line!=NULL)
{
free(rule->table_line);
rule->table_line=NULL;
}
if(rule->n_foreign>0)
{
for(int i=0; i<rule->n_foreign; i++)
{
free(rule->f_keys[i].filename);
free(rule->f_keys[i].key);
}
free(rule->f_keys);
}
memset(rule,0,sizeof(struct serial_rule_t));
return;
@@ -1145,7 +1154,6 @@ void _exec_serial_rule(redisContext* ctx, long long version, struct serial_rule_
return;
}
#define MAX_REDIS_OP_PER_SRULE 8
int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_rule_num, long long server_time, void* logger)
{
int max_redis_batch=1*1024,batch_cnt=0;
@@ -1154,6 +1162,7 @@ int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_r
unsigned int i=0;
unsigned int multi_cmd_cnt=0;
const int MAX_REDIS_OP_PER_SRULE=8;
unsigned int max_multi_cmd_num=MAX_REDIS_OP_PER_SRULE*serial_rule_num+2;// 2 for operation in _exec_serial_rule_end()
struct expected_reply_t *expected_reply=(struct expected_reply_t*)calloc(sizeof(struct expected_reply_t), max_multi_cmd_num);
@@ -1371,6 +1380,208 @@ void cleanup_update_status(redisContext *ctx, void *logger)
,entry_num);
}
const char* find_Nth_column(const char* line, int Nth, int* column_len)
{
int i=0, j=0;
int start=0, end=0;
for(i=0;i<(int)strlen(line);i++)
{
if(line[i]==' '||line[i]=='\t')
{
j++;
}
if(j==Nth-1)
{
start=i+1;
}
if(j==Nth)
{
end=i+1;
break;
}
}
if(start==0)
{
return NULL;
}
if(end==0)
{
end=i;
}
*column_len=end-start;
return line+start;
}
char* get_foreign_cont_filename(const char* table_name, int rule_id, const char* foreign_key, const char* dir)
{
char* filename=NULL;
char buffer[512];
snprintf(buffer, sizeof(buffer),"%s/%s/%d%s",dir, table_name, rule_id, foreign_key);
filename=(char*)calloc(sizeof(char), strlen(buffer)+1);
memcpy(filename, buffer, strlen(buffer));
return filename;
}
void rewrite_table_line_with_foreign(struct serial_rule_t*p)
{
int origin_column_size=0;
const char* origin_column=NULL, *pos_origin_line=NULL;
char* pos_rewrite_line=NULL;
char* rewrite_line=NULL;
size_t fn_size=0;
int i=0;
for(i=0; i<p->n_foreign; i++)
{
fn_size+=strlen(p->f_keys[i].filename);
}
rewrite_line=(char*)calloc(sizeof(char), strlen(p->table_line)+fn_size);
pos_origin_line=p->table_line;
pos_rewrite_line=rewrite_line;
for(i=0; i<p->n_foreign; i++)
{
origin_column=find_Nth_column(p->table_line, p->f_keys[i].column, &origin_column_size);
strncat(pos_rewrite_line, pos_origin_line, origin_column-pos_origin_line);
pos_rewrite_line+=origin_column-pos_origin_line;
pos_origin_line+=origin_column_size;
strncat(pos_rewrite_line, p->f_keys[i].filename, strlen(p->f_keys[i].filename));
pos_rewrite_line+=strlen(p->f_keys[i].filename);
}
strncat(pos_rewrite_line, pos_origin_line, strlen(p->table_line)-(pos_origin_line-p->table_line));
pos_rewrite_line+=strlen(p->f_keys[i].filename);
free(p->table_line);
p->table_line=rewrite_line;
return;
}
int get_foreign_keys(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger)
{
int ret=0, table_id=0, i=0, j=0;
int foregin_key_size=0;
int rule_with_foreign_key=0;
const char* foreign_source_prefix="redis://";
const char* foreign_key_prefix="__FILE_";
const char* p_foregin=NULL;
struct _Maat_table_info_t* p_table=NULL;
for(i=0; i<rule_num; i++)
{
if(rule_list[i].table_line==NULL)
{
continue;
}
ret=map_str2int(feather->map_tablename2id, rule_list[i].table_name, &table_id);
if(ret<0)
{
continue;
}
p_table=feather->p_table_info[table_id];
if(p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->n_foreign==0)
{
continue;
}
rule_list[i].n_foreign=p_table->n_foreign;
rule_list[i].f_keys=(struct foreign_key*)calloc(sizeof(struct foreign_key), rule_list[i].n_foreign);
for(j=0;j<p_table->n_foreign;j++)
{
p_foregin=find_Nth_column(rule_list[i].table_line, p_table->foreign_columns[j], &foregin_key_size);
if(p_foregin==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor
, "Get %s,%d foreign keys failed: No %dth column."
, rule_list[i].table_name, rule_list[i].rule_id, p_table->foreign_columns[j]);
continue;
}
if(0!=strncasecmp(p_foregin, foreign_source_prefix, strlen(foreign_source_prefix)))
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor
,"Get %s,%d foreign key failed: Invalid source prefix %s."
, rule_list[i].table_name, rule_list[i].rule_id, p_foregin);
continue;
}
rule_list[i].f_keys[j].column=p_table->foreign_columns[j];
foregin_key_size=foregin_key_size+1-strlen(foreign_source_prefix);
p_foregin+=strlen(foreign_source_prefix);
if(0!=strncasecmp(p_foregin, foreign_key_prefix, strlen(foreign_key_prefix)))
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor
,"%s,%d foreign key prefix %s is discouraged."
, rule_list[i].table_name, rule_list[i].rule_id, p_foregin);
}
rule_list[i].f_keys[j].key=(char*)calloc(sizeof(char),foregin_key_size);
memcpy(rule_list[i].f_keys[j].key, p_foregin, foregin_key_size);
rule_list[i].f_keys[j].filename=get_foreign_cont_filename(rule_list[i].table_name, rule_list[i].rule_id, p_foregin, dir);
}
rule_with_foreign_key++;
}
return rule_with_foreign_key;
}
void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, void *logger)
{
int i=0, j=0, ret=0;
int key_num=0;
char redis_cmd[256];
redisReply* reply=NULL;
struct serial_rule_t*p=NULL;
char** filenames[MAX_FOREIGN_CLMN_NUM];
FILE* fp=NULL;
for(i=0;i<rule_num;i++)
{
p=rule_list+i;
if(p->op==MAAT_OP_DEL||p->n_foreign==0)
{
continue;
}
for(j=0; j<p->n_foreign; j++)
{
snprintf(redis_cmd,sizeof(redis_cmd),"GET %s", p->f_keys[j].key);
ret=redisAppendCommand(ctx, redis_cmd);
key_num++;
assert(ret==REDIS_OK);
}
}
for(i=0;i<rule_num;i++)
{
p=rule_list+i;
if(p->op==MAAT_OP_DEL||p->n_foreign==0)
{
continue;
}
for(j=0; j<p->n_foreign; j++)
{
ret=_wrap_redisGetReply(ctx,&reply);
if(reply->type!=REDIS_REPLY_STRING)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor
,"Get %s,%d foreign key %s content failed."
,rule_list[i].table_name
,rule_list[i].rule_id
,p->f_keys[j].key);
continue;
}
else
{
fp=fopen(p->f_keys[j].filename, "w");
if(fp==NULL)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor
, "Write foreign content failed: fopen %s error."
, filenames[j]);
}
else
{
fwrite(reply->str, 1, reply->len, fp);
fclose(fp);
fp=NULL;
}
}
freeReplyObject(reply);
}
}
return;
}
void redis_monitor_traverse(long long version,redisContext *c,
void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para
@@ -1380,7 +1591,7 @@ void redis_monitor_traverse(long long version,redisContext *c,
const unsigned char* dec_key,
_Maat_feather_t* feather)
{
int table_id=0,i=0,rule_num=0,empty_value_num=0;
int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0;
int ret=0;
struct serial_rule_t* rule_list=NULL;
int update_type=CM_UPDATE_TYPE_INC;
@@ -1432,6 +1643,11 @@ void redis_monitor_traverse(long long version,redisContext *c,
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor,"%d of %d rules are empty.",empty_value_num,rule_num);
}
ret=get_foreign_keys(c, rule_list, rule_num, feather, feather->foreign_cont_dir, logger);
if(ret>0)
{
get_foreign_conts(c, rule_list, rule_num, logger);
}
}
start(new_version,update_type,u_para);
@@ -1458,11 +1674,41 @@ void redis_monitor_traverse(long long version,redisContext *c,
,rule_list[i].table_line);
continue;
}
}
if(rule_list[i].n_foreign>0)
{
rewrite_table_line_with_foreign(rule_list+i);
}
update(rule_list[i].table_name,rule_list[i].table_line,u_para);
update(rule_list[i].table_name,rule_list[i].table_line,u_para);
if(rule_list[i].n_foreign&&rule_list[i].op==MAAT_OP_DEL)
{
for(j=0; j<rule_list[i].n_foreign; j++)
{
if(feather->foreign_cont_linger==0)
{
ret=system_cmd_rm(rule_list[i].f_keys[j].filename);
if(ret==-1)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
"Foreign content file %s remove failed.",
rule_list[i].f_keys[j].filename);
}
}
else if(feather->foreign_cont_linger>0)
{
garbage_bagging_with_timeout(GARBAGE_FOREIGN_FILE, rule_list[i].f_keys[j].filename, feather->foreign_cont_linger, feather->garbage_q);
rule_list[i].f_keys[j].filename=NULL;//transfer owner to garbage collection.
}
else
{
//Less than 0, don't delete.
}
}
}
}
finish(u_para);
finish(u_para);
clean_up:
for(i=0;i<rule_num;i++)

292
src/entry/Maat_rule.cpp
View File

@@ -19,6 +19,7 @@
#include "Maat_rule.h"
#include "Maat_rule_internal.h"
#include "Maat_utils.h"
#include "json2iris.h"
#include "cJSON.h"
#include "dynamic_array.h"
@@ -35,57 +36,7 @@ int MAAT_FRAME_VERSION_2_2_20180921=1;
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};
pid_t gettid()
{
return syscall(SYS_gettid);
}
const char* module_name_str(const char*name)
{
static __thread char module[64];
snprintf(module,sizeof(module),"%s(%d)", name, gettid());
return module;
}
int converHextoint(char srctmp)
{
if(isdigit(srctmp))
{
return srctmp-'0';
}
else
{
char temp=toupper(srctmp);
temp=temp-'A'+10;
return temp;
}
}
int hex2bin(char *hex,int hex_len,char *binary,int size)
{
int i=0;
int resultlen=0;
int high,low;
for(i=0;i<hex_len&&size>resultlen; i+=2,resultlen++)
{
high=converHextoint(hex[i]);
low=converHextoint(hex[i+1]);
binary[resultlen]=high*16+low;
}
size=resultlen;
binary[resultlen]='\0';
return resultlen;
}
//functioned as strdup, for dictator compatible.
char* _maat_strdup(const char* s)
{
char*d=NULL;
if(s==NULL)
{
return NULL;
}
d=(char*)malloc(strlen(s)+1);
memcpy(d,s,strlen(s)+1);
return d;
}
int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type)
{
switch(expr_type)
@@ -320,113 +271,6 @@ error_out:
return ret;
}
char* str_tolower(char* string)
{
int i=0;
for(i=0;i<(int)strlen(string);i++)
{
string[i]=(char)tolower(string[i]);
}
return string;
}
char * strchr_esc(char* s,const char delim)
{
char *token;
if(s==NULL)
return NULL;
for(token=s;*token!='\0';token++)
{
if(*token=='\\')
{
token++;
continue;
}
if(*token==delim)
break;
}
if (*token == '\0')
{
return NULL;
}
else
{
return token;
}
}
char *strtok_r_esc(char *s, const char delim, char **save_ptr) {
char *token;
if (s == NULL) s = *save_ptr;
/* Scan leading delimiters. */
token=strchr_esc(s,delim);
if(token==NULL)
{
*save_ptr=token;
return s;
}
/* Find the end of the token. */
*token='\0';
token++;
*save_ptr=token;
return s;
}
char *str_unescape_and(char*s)
{
int i=0,j=0;
for(i=0,j=0;i<(int)strlen(s);i++)
{
if(s[i]=='\\'&&s[i+1]=='&')
{
s[j]='&';
i++;
j++;
}
else{
s[j]=s[i];
j++;
}
}
s[j]='\0';
return s;
}
char* str_unescape(char* s)
{
int i=0,j=0;
int len=strlen(s);
for(i=0,j=0;i<len;i++)
{
if(s[i]=='\\')
{
switch(s[i+1])
{
case '&':
s[j]='&';
break;
case 'b':
s[j]=' ';//space,0x20;
break;
case '\\':
s[j]='\\';
break;
default:
s[j]=s[i];
i--; //undo the followed i++
break;
}
i++;
j++;
}
else
{
s[j]=s[i];
j++;
}
}
s[j]='\0';
return s;
}
char* Maat_str_escape(char* dst,int size,const char*src)
{
int i=0,j=0;
@@ -460,6 +304,7 @@ char* Maat_str_escape(char* dst,int size,const char*src)
dst[j]='\0';
return dst;
}
int cnt_maskbits(struct in6_addr mask)
{
unsigned int i=0;
@@ -689,7 +534,7 @@ void destroy_table_info(struct _Maat_table_info_t*p)
free(p);
return;
}
int read_expr_table_info(const char* line,int line_num,struct _Maat_table_info_t* p,MESA_htable_handle string2int_map)
int read_expr_table_info(const char* line, struct _Maat_table_info_t* p, MESA_htable_handle string2int_map)
{
int j=0,ret[4]={0};
char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0};
@@ -743,6 +588,91 @@ int read_expr_table_info(const char* line,int line_num,struct _Maat_table_info_t
}
return 0;
}
int _read_integer_arrary(char* string, int *array, int size)
{
char *token=NULL,*sub_token=NULL,*saveptr;
int i=0;
for (token = string, i=0; i<size ; token= NULL, i++)
{
sub_token= strtok_r(token,",", &saveptr);
if (sub_token == NULL)
break;
sscanf(sub_token, "%d", array+i);
}
return i;
}
int read_plugin_table_info(const char* line, struct _Maat_table_info_t* p)
{
int i=0,ret=0;
size_t j=0;
char* copy_line=NULL, *plug_info=NULL;
char *token=NULL,*sub_token=NULL,*saveptr;
const char* plug_info_prefix[]={"valid","tag","foreign"};
const int VALID_PREFIX_IDX=0, TAG_PREFIX_IDX=1, FOREIGN_PREFIX_IDX=2;
p->valid_flag_column=-1;
copy_line=_maat_strdup(line);
for (token = copy_line, i=0; ; token= NULL, i++)
{
sub_token= strtok_r(token,"\t", &saveptr);
if (sub_token == NULL)
break;
if(i==3)
{
break;
}
}
if(i<3)
{
goto error_out;
}
plug_info=sub_token;
if(strlen(plug_info)<4)//For old version compatible.
{
ret=sscanf(plug_info, "%d", &(p->valid_flag_column));
if(ret==0||ret==EOF)
{
p->valid_flag_column=-1;
}
free(copy_line);
return 0;
}
for (token = plug_info; ; token= NULL)
{
sub_token= strtok_r(token,";", &saveptr);
if (sub_token == NULL)
break;
for(j=0; j< sizeof(plug_info_prefix)/sizeof(const char*); j++)
{
if(0==strncasecmp(sub_token, plug_info_prefix[j], strlen(plug_info_prefix[j])))
{
break;
}
}
switch(j)
{
case VALID_PREFIX_IDX:
sscanf(sub_token+strlen(plug_info_prefix[j])+1,"%d",&(p->valid_flag_column));
break;
case TAG_PREFIX_IDX:
sscanf(sub_token+strlen(plug_info_prefix[j])+1,"%d",&(p->rule_tag_column));
break;
case FOREIGN_PREFIX_IDX:
p->n_foreign=_read_integer_arrary(sub_token+strlen(plug_info_prefix[j])+1, p->foreign_columns, MAX_FOREIGN_CLMN_NUM);
break;
default:
break;
}
}
free(copy_line);
return 0;
error_out:
free(copy_line);
return -1;
}
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger)
{
FILE*fp=NULL;
@@ -821,22 +751,25 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
{
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
ret=read_expr_table_info(line, i, p, string2int_map);
ret=read_expr_table_info(line, p, string2int_map);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:unknown column.\n",table_info_path,i);
fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:unknown column.",table_info_path,i);
"Maat read table info %s line %d error:illegal column.",table_info_path,i);
goto error_jump;
}
break;
case TABLE_TYPE_PLUGIN:
p->cb_info=(struct _plugin_table_info*)calloc(sizeof(struct _plugin_table_info),1);
p->cb_info->cache_lines=dynamic_array_create(1024,1024);
ret=sscanf(not_care,"%d\t%d\t%[0-9,]",&(p->valid_flag_column), &p->rule_tag_column, tmp_str);
if(ret==0||ret==EOF)
ret=read_plugin_table_info(line, p);
if(ret<0)
{
p->valid_flag_column=-1;
fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal plugin info.",table_info_path,i);
goto error_jump;
}
break;
case TABLE_TYPE_COMPILE:
@@ -2997,7 +2930,7 @@ error_out:
free(digest_rule);
digest_rule=NULL;
}
void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q)
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q)
{
if(p==NULL)
{
@@ -3008,25 +2941,40 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag
bag->type=type;
bag->create_time=time(NULL);
bag->ok_times=0;
bag->expire_after=timeout;
MESA_lqueue_join_tail(garbage_q,&bag,sizeof(void*));
return;
}
void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q)
{
garbage_bagging_with_timeout(type, p, -1, garbage_q);
return;
}
void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger)
{
MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK;
_maat_garbage_t* bag=NULL;
long data_size=0;
const long q_cnt=MESA_lqueue_get_count(garbage_q);
int i=0,bury_cnt=0;
int i=0,bury_cnt=0, ret=0;
long long ref_cnt=0;
int have_timeout=0;
int override_timeout=0;
time_t now=time(NULL);
for(i=0;i<q_cnt;i++)
{
data_size=sizeof(void*);
q_ret=(MESA_queue_errno_t)MESA_lqueue_get_head(garbage_q,&bag,&data_size);
assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK);
if(now-bag->create_time<timeout)
if(bag->expire_after<0)
{
override_timeout=timeout;
}
else
{
override_timeout=bag->expire_after;
}
if(now-bag->create_time<override_timeout)
{
MESA_lqueue_join_tail(garbage_q,&bag,sizeof(void*));
continue;
@@ -3052,7 +3000,6 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger)
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
"scanner %p version %d force destroyed,ref_cnt %lld.",
bag->scanner,bag->scanner->version,ref_cnt);
}
destroy_maat_scanner(bag->scanner);
break;
@@ -3062,6 +3009,15 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger)
case GARBAGE_MAP_STR2INT:
map_destroy(bag->str2int_map);
break;
case GARBAGE_FOREIGN_FILE:
ret=system_cmd_rm(bag->filename);
if(ret==-1)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
"Foreign content file %s remove failed.",
bag->filename);
}
break;
default:
assert(0);
}

186
src/entry/Maat_utils.cpp Normal file
View File

@@ -0,0 +1,186 @@
#include <string.h>
#include <stdio.h>
#include <ctype.h>
#include <stdlib.h>
#include "Maat_utils.h"
pid_t gettid()
{
return syscall(SYS_gettid);
}
const char* module_name_str(const char*name)
{
static __thread char module[64];
snprintf(module,sizeof(module),"%s(%d)", name, gettid());
return module;
}
int converHextoint(char srctmp)
{
if(isdigit(srctmp))
{
return srctmp-'0';
}
else
{
char temp=toupper(srctmp);
temp=temp-'A'+10;
return temp;
}
}
int hex2bin(char *hex,int hex_len,char *binary,int size)
{
int i=0;
int resultlen=0;
int high,low;
for(i=0;i<hex_len&&size>resultlen; i+=2,resultlen++)
{
high=converHextoint(hex[i]);
low=converHextoint(hex[i+1]);
binary[resultlen]=high*16+low;
}
size=resultlen;
binary[resultlen]='\0';
return resultlen;
}
//functioned as strdup, for dictator compatible.
char* _maat_strdup(const char* s)
{
char*d=NULL;
if(s==NULL)
{
return NULL;
}
d=(char*)malloc(strlen(s)+1);
memcpy(d,s,strlen(s)+1);
return d;
}
char* str_tolower(char* string)
{
int i=0;
for(i=0;i<(int)strlen(string);i++)
{
string[i]=(char)tolower(string[i]);
}
return string;
}
char * strchr_esc(char* s,const char delim)
{
char *token;
if(s==NULL)
return NULL;
for(token=s;*token!='\0';token++)
{
if(*token=='\\')
{
token++;
continue;
}
if(*token==delim)
break;
}
if (*token == '\0')
{
return NULL;
}
else
{
return token;
}
}
char *strtok_r_esc(char *s, const char delim, char **save_ptr)
{
char *token;
if (s == NULL) s = *save_ptr;
/* Scan leading delimiters. */
token=strchr_esc(s,delim);
if(token==NULL)
{
*save_ptr=token;
return s;
}
/* Find the end of the token. */
*token='\0';
token++;
*save_ptr=token;
return s;
}
char *str_unescape_and(char*s)
{
int i=0,j=0;
for(i=0,j=0;i<(int)strlen(s);i++)
{
if(s[i]=='\\'&&s[i+1]=='&')
{
s[j]='&';
i++;
j++;
}
else{
s[j]=s[i];
j++;
}
}
s[j]='\0';
return s;
}
char* str_unescape(char* s)
{
int i=0,j=0;
int len=strlen(s);
for(i=0,j=0;i<len;i++)
{
if(s[i]=='\\')
{
switch(s[i+1])
{
case '&':
s[j]='&';
break;
case 'b':
s[j]=' ';//space,0x20;
break;
case '\\':
s[j]='\\';
break;
default:
s[j]=s[i];
i--; //undo the followed i++
break;
}
i++;
j++;
}
else
{
s[j]=s[i];
j++;
}
}
s[j]='\0';
return s;
}
#define MAX_SYSTEM_CMD_LEN 512
int system_cmd_mkdir(const char* path)
{
char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
snprintf(cmd, sizeof(cmd),"mkdir -p %s", path);
return system(cmd) ;
}
int system_cmd_mv(const char* src_file,const char*dst_file)
{
char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
snprintf(cmd,sizeof(cmd), "mv %s %s", src_file, dst_file);
return system(cmd);
}
int system_cmd_rm(const char* src_file)
{
char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
snprintf(cmd,sizeof(cmd), "rm %s -f", src_file);
return system(cmd);
}

8
src/entry/config_monitor.cpp
View File

@@ -1,6 +1,6 @@
#include "MESA_handle_logger.h"
#include "config_monitor.h"
#include "Maat_rule_internal.h"
#include "Maat_utils.h"
#include <dirent.h>
#include <stdlib.h>
#include <stdio.h>
@@ -18,13 +18,7 @@
#define MAX_CONFIG_FN_LEN 256
#define MAX_CONFIG_LINE (1024*16)
#ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
#endif
#ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
//#define USING_DICTATOR 1
extern "C" void __real_free(void*p);
struct cm_table_info_t

2
src/entry/json2iris.cpp
View File

@@ -13,6 +13,8 @@
#include "hiredis.h"
#include "map_str2int.h"
#include "Maat_rule_internal.h"
#include "Maat_utils.h"
#define maat_json (module_name_str("MAAT_JSON"))
const char* untitled_group_name="Untitled";

39
src/inc_internal/Maat_rule_internal.h
View File

@@ -43,8 +43,8 @@ typedef int atomic_t;
#define MAX_TABLE_LINE_SIZE (1024*16)
#define MAX_EXPR_KEYLEN 1024
#define MAX_DISTRICT_LEN 64
#define MAX_PLUGING_NUM 32
#define MAX_PLUGIN_PER_TABLE 32
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_SCANNER_HIT_NUM 64
#define MAX_GROUP_CACHE 128
@@ -210,7 +210,7 @@ struct _callback_plugin
struct _plugin_table_info
{
int cb_plug_cnt;
struct _callback_plugin cb_plug[MAX_PLUGING_NUM];
struct _callback_plugin cb_plug[MAX_PLUGIN_PER_TABLE];
dynamic_array_t *cache_lines;
int cache_line_num;
int acc_line_num;
@@ -262,6 +262,8 @@ struct _Maat_table_info_t
struct _plugin_table_info *cb_info;
int valid_flag_column; //for plugin table
int rule_tag_column; //for plugin table;
int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table;
int n_foreign;
int user_region_encoding; //for compile table, USER_REGION_ENCODE_xx
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
@@ -298,7 +300,8 @@ enum maat_garbage_type
GARBAGE_GROUP_RULE,
GARBAGE_COMPILE_RULE,
GARBAGE_BOOL_MATCHER,
GARBAGE_MAP_STR2INT
GARBAGE_MAP_STR2INT,
GARBAGE_FOREIGN_FILE
};
struct iconv_handle_t
{
@@ -409,6 +412,8 @@ struct _Maat_feather_t
struct rule_tag *accept_tags;
int n_tags;
char foreign_cont_dir[MAX_TABLE_NAME_LEN];
int foreign_cont_linger;
//internal states
long long new_version;
int active_plugin_table_num;
@@ -441,6 +446,7 @@ struct _maat_garbage_t
{
enum maat_garbage_type type;
time_t create_time;
int expire_after;
int ok_times;
union
{
@@ -450,8 +456,15 @@ struct _maat_garbage_t
void* bool_matcher;
void * raw;
MESA_htable_handle str2int_map;
char* filename;
};
};
struct foreign_key
{
int column;
char* key;
char* filename;
};
struct serial_rule_t //rm= Redis Maat
{
enum MAAT_OPERATION op;//0: delete, 1: add.
@@ -460,9 +473,12 @@ struct serial_rule_t //rm= Redis Maat
long long timeout; // absolute unix time.
char table_name[256];
char* table_line;
int n_foreign;
struct foreign_key* f_keys;
};
int parse_accept_tag(const char* value, struct rule_tag** result, void* logger);
void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set);
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
@@ -471,15 +487,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para);
void *thread_rule_monitor(void *arg);
unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge);
inline void ipv6_ntoh(unsigned int *v6_addr)
{
unsigned int i=0;
for(i=0;i<4;i++)
{
v6_addr[i]=ntohl(v6_addr[i]);
}
return;
}
void * HASH_fetch_by_id(MESA_htable_handle hash,int id);
int HASH_add_by_id(MESA_htable_handle hash,int id,void*data);
int HASH_delete_by_id(MESA_htable_handle hash,int id);
@@ -487,8 +495,7 @@ void maat_read_full_config(_Maat_feather_t* _feather);
void maat_stat_init(struct _Maat_feather_t* feather);
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num);
void maat_stat_output(struct _Maat_feather_t* feather);
char* _maat_strdup(const char* s);
char* str_unescape(char* s);
redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...);
int get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, long long* new_version, struct serial_rule_t** list,int *update_type, void* logger, int cumulative_off);
int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger,int print_process);
@@ -505,8 +512,6 @@ void redis_monitor_traverse(long long version,redisContext *c,
const unsigned char* dec_key,
_Maat_feather_t* feather);
const char* module_name_str(const char*name);
#define maat_module (module_name_str("MAAT_Frame"))
#endif

43
src/inc_internal/Maat_utils.h Normal file
View File

@@ -0,0 +1,43 @@
#pragma once
#include <arpa/inet.h>
#include <unistd.h>
#include <sys/syscall.h>
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
#define FREE(p) {free(*p);*p=NULL;}
#define ATOMIC_INC(x) __atomic_fetch_add(x,1,__ATOMIC_RELAXED)
#define ATOMIC_READ(x) __atomic_fetch_add(x,0,__ATOMIC_RELAXED)
#ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
#endif
#ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
const char* module_name_str(const char*name);
#define maat_module (module_name_str("MAAT_Frame"))
char* _maat_strdup(const char* s);
char* str_unescape(char* s);
inline void ipv6_ntoh(unsigned int *v6_addr)
{
unsigned int i=0;
for(i=0;i<4;i++)
{
v6_addr[i]=ntohl(v6_addr[i]);
}
return;
}
int hex2bin(char *hex,int hex_len,char *binary,int size);
char* str_tolower(char* string);
char *strtok_r_esc(char *s, const char delim, char **save_ptr);
char *str_unescape_and(char*s);
char* str_unescape(char* s);
pid_t gettid(void);
int system_cmd_mkdir(const char* path);
int system_cmd_rm(const char* src_file);

7
test/table_info.conf
View File

@@ -7,10 +7,10 @@
#do_merege [yes/no]
#cross cache [number]
#quick mode [quickon/quickoff], default [quickoff]
#For ip/intval/digest/compile/group
#For ip/intval/digest/compile/group table.
#id name type
#
#For plugin table
#For plugin table. The first column's id is 1. 0 as not speicified.
#id name type valid_column tag_column
#
#For expr/expr_plus Table
@@ -27,4 +27,5 @@
8 HTTP_REGION expr_plus GBK GBK no 0
9 SIM_URL similar --
10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff
11 TEST_EFFECTIVE_RANGE_TABLE plugin 4 5 --
11 TEST_EFFECTIVE_RANGE_TABLE plugin valid_pos=4;tag_pos=5; --
12 TEST_EXETERN_LINK plugin valid_pos=3;foreign_pos=6,7;