diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 61be910..8980546 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -141,10 +141,10 @@ enum MAAT_INIT_OPT MAAT_OPT_FULL_CFG_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default. MAAT_OPT_INC_CFG_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default. MAAT_OPT_JSON_FILE_PATH, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default. - MAAT_OPT_STAT_ON, //VALUE is NULL,SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF. - MAAT_OPT_PERF_ON, //VALUE is NULL,SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF. + MAAT_OPT_STAT_ON, //VALUE is NULL, SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF. + MAAT_OPT_PERF_ON, //VALUE is NULL, SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF. MAAT_OPT_STAT_FILE_PATH, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. DEFAULT: no default. - MAAT_OPT_SCAN_DETAIL, //VALUE is interger *, SIZE=sizeof(int). 0: not return any detail;1: return hit pos, not include regex grouping; + MAAT_OPT_SCAN_DETAIL, //VALUE is interger *, SIZE=sizeof(int). 0: not return any detail;1: return hit pos, not include regex grouping. // 2 return hit pos and regex grouping pos;DEFAULT:0 MAAT_OPT_INSTANCE_NAME, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1, no more than 11 bytes.DEFAULT: MAAT_$tableinfo_path$. MAAT_OPT_DECRYPT_KEY, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. No DEFAULT. @@ -157,8 +157,10 @@ enum MAAT_INIT_OPT MAAT_OPT_LOAD_VERSION_FROM, //VALUE is a long long, SIZE=sizeof(long long). Default: Load the Latest. Only valid in redis mode, and maybe failed for too old. //This option also disables background update. MAAT_OPT_ENABLE_UPDATE, //VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled. DEFAULT: Backgroud update is enabled. Runtime setting is allowed. - MAAT_OPT_ACCEPT_TAGS //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]} -}; + MAAT_OPT_ACCEPT_TAGS, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]} + MAAT_OPT_FOREIGN_CONT_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files + MAAT_OPT_FOREIGN_CONT_LINGER //VALUE is interger *, SIZE=sizeof(int). Greater than 0: delete after VALUE seconds; 0: delete foreign content right after the notification callbacks; Less than 0: NEVER delete. Default: 0. + }; //return -1 if failed, return 0 on success; int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size); enum MAAT_STATE_OPT diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9099334..a5e6477 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,7 +9,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) -set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/UniversalBoolMatch.cpp) +set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/UniversalBoolMatch.cpp) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) include_directories(/opt/MESA/include/MESA/) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index eec2285..6888062 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -8,6 +8,7 @@ #include "UniversalBoolMatch.h" #include "Maat_rule.h" #include "Maat_rule_internal.h" +#include "Maat_utils.h" #include "dynamic_array.h" #include "aligment_int64.h" #include "config_monitor.h" @@ -486,6 +487,8 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* feather->connect_timeout.tv_sec=0; feather->connect_timeout.tv_usec=100*1000; // 100 ms feather->backgroud_update_enabled=1; + feather->foreign_cont_linger=0; + snprintf(feather->foreign_cont_dir, sizeof(feather->foreign_cont_dir), "%s_files", table_info_path); pthread_mutex_init(&(feather->backgroud_update_mutex),NULL); pthread_mutex_init(&(feather->redis_write_lock),NULL); snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path); @@ -673,6 +676,16 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo return -1; } break; + case MAAT_OPT_FOREIGN_CONT_DIR: + strncpy(_feather->foreign_cont_dir, (char*)value, sizeof(_feather->foreign_cont_dir)); + if(_feather->foreign_cont_dir[size-1]=='/') + { + _feather->foreign_cont_dir[size-1]='\0'; + } + break; + case MAAT_OPT_FOREIGN_CONT_LINGER: + _feather->foreign_cont_linger=*(int*)value; + break; default: return -1; } @@ -904,7 +917,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, //plugin table register blocks background update. pthread_mutex_lock(&(_feather->backgroud_update_mutex)); idx=p_table->cb_info->cb_plug_cnt; - if(idx==MAX_PLUGING_NUM) + if(idx==MAX_PLUGIN_PER_TABLE) { pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); return -1; diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 2dbcec5..27360a4 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -1,6 +1,7 @@ #include "Maat_command.h" #include "Maat_rule.h" #include "Maat_rule_internal.h" +#include "Maat_utils.h" #include "config_monitor.h" #include "map_str2int.h" #include "hiredis.h" @@ -292,7 +293,15 @@ void empty_serial_rules(struct serial_rule_t* rule) if(rule->table_line!=NULL) { free(rule->table_line); - rule->table_line=NULL; + } + if(rule->n_foreign>0) + { + for(int i=0; in_foreign; i++) + { + free(rule->f_keys[i].filename); + free(rule->f_keys[i].key); + } + free(rule->f_keys); } memset(rule,0,sizeof(struct serial_rule_t)); return; @@ -1145,7 +1154,6 @@ void _exec_serial_rule(redisContext* ctx, long long version, struct serial_rule_ return; } -#define MAX_REDIS_OP_PER_SRULE 8 int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_rule_num, long long server_time, void* logger) { int max_redis_batch=1*1024,batch_cnt=0; @@ -1154,6 +1162,7 @@ int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_r unsigned int i=0; unsigned int multi_cmd_cnt=0; + const int MAX_REDIS_OP_PER_SRULE=8; unsigned int max_multi_cmd_num=MAX_REDIS_OP_PER_SRULE*serial_rule_num+2;// 2 for operation in _exec_serial_rule_end() struct expected_reply_t *expected_reply=(struct expected_reply_t*)calloc(sizeof(struct expected_reply_t), max_multi_cmd_num); @@ -1371,6 +1380,208 @@ void cleanup_update_status(redisContext *ctx, void *logger) ,entry_num); } +const char* find_Nth_column(const char* line, int Nth, int* column_len) +{ + int i=0, j=0; + int start=0, end=0; + for(i=0;i<(int)strlen(line);i++) + { + if(line[i]==' '||line[i]=='\t') + { + j++; + } + if(j==Nth-1) + { + start=i+1; + } + if(j==Nth) + { + end=i+1; + break; + } + } + if(start==0) + { + return NULL; + } + if(end==0) + { + end=i; + } + *column_len=end-start; + return line+start; +} +char* get_foreign_cont_filename(const char* table_name, int rule_id, const char* foreign_key, const char* dir) +{ + char* filename=NULL; + char buffer[512]; + snprintf(buffer, sizeof(buffer),"%s/%s/%d%s",dir, table_name, rule_id, foreign_key); + filename=(char*)calloc(sizeof(char), strlen(buffer)+1); + memcpy(filename, buffer, strlen(buffer)); + return filename; +} +void rewrite_table_line_with_foreign(struct serial_rule_t*p) +{ + int origin_column_size=0; + const char* origin_column=NULL, *pos_origin_line=NULL; + char* pos_rewrite_line=NULL; + char* rewrite_line=NULL; + size_t fn_size=0; + int i=0; + for(i=0; in_foreign; i++) + { + fn_size+=strlen(p->f_keys[i].filename); + } + + rewrite_line=(char*)calloc(sizeof(char), strlen(p->table_line)+fn_size); + pos_origin_line=p->table_line; + pos_rewrite_line=rewrite_line; + + for(i=0; in_foreign; i++) + { + origin_column=find_Nth_column(p->table_line, p->f_keys[i].column, &origin_column_size); + strncat(pos_rewrite_line, pos_origin_line, origin_column-pos_origin_line); + pos_rewrite_line+=origin_column-pos_origin_line; + pos_origin_line+=origin_column_size; + + strncat(pos_rewrite_line, p->f_keys[i].filename, strlen(p->f_keys[i].filename)); + pos_rewrite_line+=strlen(p->f_keys[i].filename); + } + strncat(pos_rewrite_line, pos_origin_line, strlen(p->table_line)-(pos_origin_line-p->table_line)); + pos_rewrite_line+=strlen(p->f_keys[i].filename); + + free(p->table_line); + p->table_line=rewrite_line; + return; +} +int get_foreign_keys(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger) +{ + int ret=0, table_id=0, i=0, j=0; + int foregin_key_size=0; + int rule_with_foreign_key=0; + const char* foreign_source_prefix="redis://"; + const char* foreign_key_prefix="__FILE_"; + const char* p_foregin=NULL; + struct _Maat_table_info_t* p_table=NULL; + for(i=0; imap_tablename2id, rule_list[i].table_name, &table_id); + if(ret<0) + { + continue; + } + p_table=feather->p_table_info[table_id]; + if(p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->n_foreign==0) + { + continue; + } + rule_list[i].n_foreign=p_table->n_foreign; + rule_list[i].f_keys=(struct foreign_key*)calloc(sizeof(struct foreign_key), rule_list[i].n_foreign); + for(j=0;jn_foreign;j++) + { + p_foregin=find_Nth_column(rule_list[i].table_line, p_table->foreign_columns[j], &foregin_key_size); + if(p_foregin==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor + , "Get %s,%d foreign keys failed: No %dth column." + , rule_list[i].table_name, rule_list[i].rule_id, p_table->foreign_columns[j]); + continue; + } + if(0!=strncasecmp(p_foregin, foreign_source_prefix, strlen(foreign_source_prefix))) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor + ,"Get %s,%d foreign key failed: Invalid source prefix %s." + , rule_list[i].table_name, rule_list[i].rule_id, p_foregin); + continue; + } + rule_list[i].f_keys[j].column=p_table->foreign_columns[j]; + foregin_key_size=foregin_key_size+1-strlen(foreign_source_prefix); + p_foregin+=strlen(foreign_source_prefix); + if(0!=strncasecmp(p_foregin, foreign_key_prefix, strlen(foreign_key_prefix))) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor + ,"%s,%d foreign key prefix %s is discouraged." + , rule_list[i].table_name, rule_list[i].rule_id, p_foregin); + } + rule_list[i].f_keys[j].key=(char*)calloc(sizeof(char),foregin_key_size); + memcpy(rule_list[i].f_keys[j].key, p_foregin, foregin_key_size); + + rule_list[i].f_keys[j].filename=get_foreign_cont_filename(rule_list[i].table_name, rule_list[i].rule_id, p_foregin, dir); + } + rule_with_foreign_key++; + } + return rule_with_foreign_key; +} + +void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, void *logger) +{ + int i=0, j=0, ret=0; + int key_num=0; + + char redis_cmd[256]; + redisReply* reply=NULL; + struct serial_rule_t*p=NULL; + char** filenames[MAX_FOREIGN_CLMN_NUM]; + FILE* fp=NULL; + for(i=0;iop==MAAT_OP_DEL||p->n_foreign==0) + { + continue; + } + for(j=0; jn_foreign; j++) + { + snprintf(redis_cmd,sizeof(redis_cmd),"GET %s", p->f_keys[j].key); + ret=redisAppendCommand(ctx, redis_cmd); + key_num++; + assert(ret==REDIS_OK); + } + } + for(i=0;iop==MAAT_OP_DEL||p->n_foreign==0) + { + continue; + } + for(j=0; jn_foreign; j++) + { + ret=_wrap_redisGetReply(ctx,&reply); + if(reply->type!=REDIS_REPLY_STRING) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor + ,"Get %s,%d foreign key %s content failed." + ,rule_list[i].table_name + ,rule_list[i].rule_id + ,p->f_keys[j].key); + continue; + } + else + { + fp=fopen(p->f_keys[j].filename, "w"); + if(fp==NULL) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor + , "Write foreign content failed: fopen %s error." + , filenames[j]); + } + else + { + fwrite(reply->str, 1, reply->len, fp); + fclose(fp); + fp=NULL; + } + } + freeReplyObject(reply); + } + } + return; +} void redis_monitor_traverse(long long version,redisContext *c, void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para @@ -1380,7 +1591,7 @@ void redis_monitor_traverse(long long version,redisContext *c, const unsigned char* dec_key, _Maat_feather_t* feather) { - int table_id=0,i=0,rule_num=0,empty_value_num=0; + int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0; int ret=0; struct serial_rule_t* rule_list=NULL; int update_type=CM_UPDATE_TYPE_INC; @@ -1432,6 +1643,11 @@ void redis_monitor_traverse(long long version,redisContext *c, { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor,"%d of %d rules are empty.",empty_value_num,rule_num); } + ret=get_foreign_keys(c, rule_list, rule_num, feather, feather->foreign_cont_dir, logger); + if(ret>0) + { + get_foreign_conts(c, rule_list, rule_num, logger); + } } start(new_version,update_type,u_para); MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Start %s update: %lld->%lld (%d entries).", @@ -1458,11 +1674,41 @@ void redis_monitor_traverse(long long version,redisContext *c, continue; } } + if(rule_list[i].n_foreign>0) + { + rewrite_table_line_with_foreign(rule_list+i); + } update(rule_list[i].table_name,rule_list[i].table_line,u_para); - //printf("%s %s,%d\n", rule_list[i].op==MAAT_OP_DEL?"DEL":"ADD", rule_list[i].table_name, rule_list[i].rule_id); + if(rule_list[i].n_foreign&&rule_list[i].op==MAAT_OP_DEL) + { + + for(j=0; jforeign_cont_linger==0) + { + ret=system_cmd_rm(rule_list[i].f_keys[j].filename); + if(ret==-1) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, + "Foreign content file %s remove failed.", + rule_list[i].f_keys[j].filename); + } + } + else if(feather->foreign_cont_linger>0) + { + garbage_bagging_with_timeout(GARBAGE_FOREIGN_FILE, rule_list[i].f_keys[j].filename, feather->foreign_cont_linger, feather->garbage_q); + rule_list[i].f_keys[j].filename=NULL;//transfer owner to garbage collection. + } + else + { + //Less than 0, don't delete. + } + } + + } } finish(u_para); - //printf("Finish %s update: %lld->%lld\n",update_type==CM_UPDATE_TYPE_INC?"INC":"FULL",version,new_version); + clean_up: for(i=0;iresultlen; i+=2,resultlen++) - { - high=converHextoint(hex[i]); - low=converHextoint(hex[i+1]); - binary[resultlen]=high*16+low; - } - size=resultlen; - binary[resultlen]='\0'; - return resultlen; -} -//functioned as strdup, for dictator compatible. -char* _maat_strdup(const char* s) -{ - char*d=NULL; - if(s==NULL) - { - return NULL; - } - d=(char*)malloc(strlen(s)+1); - memcpy(d,s,strlen(s)+1); - return d; -} int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type) { switch(expr_type) @@ -320,113 +271,6 @@ error_out: return ret; } -char* str_tolower(char* string) -{ - int i=0; - for(i=0;i<(int)strlen(string);i++) - { - string[i]=(char)tolower(string[i]); - } - return string; -} -char * strchr_esc(char* s,const char delim) -{ - char *token; - if(s==NULL) - return NULL; - for(token=s;*token!='\0';token++) - { - if(*token=='\\') - { - token++; - continue; - } - if(*token==delim) - break; - } - if (*token == '\0') - { - return NULL; - } - else - { - return token; - } -} -char *strtok_r_esc(char *s, const char delim, char **save_ptr) { - char *token; - - if (s == NULL) s = *save_ptr; - - /* Scan leading delimiters. */ - token=strchr_esc(s,delim); - if(token==NULL) - { - *save_ptr=token; - return s; - } - /* Find the end of the token. */ - *token='\0'; - token++; - *save_ptr=token; - - return s; -} -char *str_unescape_and(char*s) -{ - int i=0,j=0; - for(i=0,j=0;i<(int)strlen(s);i++) - { - if(s[i]=='\\'&&s[i+1]=='&') - { - s[j]='&'; - i++; - j++; - } - else{ - s[j]=s[i]; - j++; - } - } - s[j]='\0'; - return s; -} -char* str_unescape(char* s) -{ - int i=0,j=0; - int len=strlen(s); - for(i=0,j=0;ivalid_flag_column=-1; + copy_line=_maat_strdup(line); + for (token = copy_line, i=0; ; token= NULL, i++) + { + sub_token= strtok_r(token,"\t", &saveptr); + if (sub_token == NULL) + break; + if(i==3) + { + break; + } + + } + if(i<3) + { + goto error_out; + } + plug_info=sub_token; + + if(strlen(plug_info)<4)//For old version compatible. + { + ret=sscanf(plug_info, "%d", &(p->valid_flag_column)); + if(ret==0||ret==EOF) + { + p->valid_flag_column=-1; + } + free(copy_line); + return 0; + } + + for (token = plug_info; ; token= NULL) + { + sub_token= strtok_r(token,";", &saveptr); + if (sub_token == NULL) + break; + for(j=0; j< sizeof(plug_info_prefix)/sizeof(const char*); j++) + { + if(0==strncasecmp(sub_token, plug_info_prefix[j], strlen(plug_info_prefix[j]))) + { + break; + } + } + switch(j) + { + case VALID_PREFIX_IDX: + sscanf(sub_token+strlen(plug_info_prefix[j])+1,"%d",&(p->valid_flag_column)); + break; + case TAG_PREFIX_IDX: + sscanf(sub_token+strlen(plug_info_prefix[j])+1,"%d",&(p->rule_tag_column)); + break; + case FOREIGN_PREFIX_IDX: + p->n_foreign=_read_integer_arrary(sub_token+strlen(plug_info_prefix[j])+1, p->foreign_columns, MAX_FOREIGN_CLMN_NUM); + break; + default: + break; + } + + } + free(copy_line); + return 0; +error_out: + free(copy_line); + return -1; +} int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) { FILE*fp=NULL; @@ -821,22 +751,25 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - ret=read_expr_table_info(line, i, p, string2int_map); + ret=read_expr_table_info(line, p, string2int_map); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:unknown column.\n",table_info_path,i); + fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:unknown column.",table_info_path,i); + "Maat read table info %s line %d error:illegal column.",table_info_path,i); goto error_jump; } break; case TABLE_TYPE_PLUGIN: p->cb_info=(struct _plugin_table_info*)calloc(sizeof(struct _plugin_table_info),1); p->cb_info->cache_lines=dynamic_array_create(1024,1024); - ret=sscanf(not_care,"%d\t%d\t%[0-9,]",&(p->valid_flag_column), &p->rule_tag_column, tmp_str); - if(ret==0||ret==EOF) + ret=read_plugin_table_info(line, p); + if(ret<0) { - p->valid_flag_column=-1; + fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:illegal plugin info.",table_info_path,i); + goto error_jump; } break; case TABLE_TYPE_COMPILE: @@ -2997,7 +2930,7 @@ error_out: free(digest_rule); digest_rule=NULL; } -void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q) +void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q) { if(p==NULL) { @@ -3008,25 +2941,40 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag bag->type=type; bag->create_time=time(NULL); bag->ok_times=0; + bag->expire_after=timeout; MESA_lqueue_join_tail(garbage_q,&bag,sizeof(void*)); return; } +void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q) +{ + garbage_bagging_with_timeout(type, p, -1, garbage_q); + return; +} void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger) { MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; _maat_garbage_t* bag=NULL; long data_size=0; const long q_cnt=MESA_lqueue_get_count(garbage_q); - int i=0,bury_cnt=0; + int i=0,bury_cnt=0, ret=0; long long ref_cnt=0; int have_timeout=0; + int override_timeout=0; time_t now=time(NULL); for(i=0;icreate_timeexpire_after<0) + { + override_timeout=timeout; + } + else + { + override_timeout=bag->expire_after; + } + if(now-bag->create_timescanner,bag->scanner->version,ref_cnt); - } destroy_maat_scanner(bag->scanner); break; @@ -3062,6 +3009,15 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger) case GARBAGE_MAP_STR2INT: map_destroy(bag->str2int_map); break; + case GARBAGE_FOREIGN_FILE: + ret=system_cmd_rm(bag->filename); + if(ret==-1) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, + "Foreign content file %s remove failed.", + bag->filename); + } + break; default: assert(0); } diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp new file mode 100644 index 0000000..c2b5732 --- /dev/null +++ b/src/entry/Maat_utils.cpp @@ -0,0 +1,186 @@ +#include +#include +#include +#include + +#include "Maat_utils.h" +pid_t gettid() +{ + return syscall(SYS_gettid); +} +const char* module_name_str(const char*name) +{ + static __thread char module[64]; + snprintf(module,sizeof(module),"%s(%d)", name, gettid()); + return module; +} + +int converHextoint(char srctmp) +{ + if(isdigit(srctmp)) + { + return srctmp-'0'; + } + else + { + char temp=toupper(srctmp); + temp=temp-'A'+10; + return temp; + } +} +int hex2bin(char *hex,int hex_len,char *binary,int size) +{ + int i=0; + int resultlen=0; + int high,low; + for(i=0;iresultlen; i+=2,resultlen++) + { + high=converHextoint(hex[i]); + low=converHextoint(hex[i+1]); + binary[resultlen]=high*16+low; + } + size=resultlen; + binary[resultlen]='\0'; + return resultlen; +} +//functioned as strdup, for dictator compatible. +char* _maat_strdup(const char* s) +{ + char*d=NULL; + if(s==NULL) + { + return NULL; + } + d=(char*)malloc(strlen(s)+1); + memcpy(d,s,strlen(s)+1); + return d; +} +char* str_tolower(char* string) +{ + int i=0; + for(i=0;i<(int)strlen(string);i++) + { + string[i]=(char)tolower(string[i]); + } + return string; +} +char * strchr_esc(char* s,const char delim) +{ + char *token; + if(s==NULL) + return NULL; + for(token=s;*token!='\0';token++) + { + if(*token=='\\') + { + token++; + continue; + } + if(*token==delim) + break; + } + if (*token == '\0') + { + return NULL; + } + else + { + return token; + } +} +char *strtok_r_esc(char *s, const char delim, char **save_ptr) +{ + char *token; + + if (s == NULL) s = *save_ptr; + + /* Scan leading delimiters. */ + token=strchr_esc(s,delim); + if(token==NULL) + { + *save_ptr=token; + return s; + } + /* Find the end of the token. */ + *token='\0'; + token++; + *save_ptr=token; + + return s; +} +char *str_unescape_and(char*s) +{ + int i=0,j=0; + for(i=0,j=0;i<(int)strlen(s);i++) + { + if(s[i]=='\\'&&s[i+1]=='&') + { + s[j]='&'; + i++; + j++; + } + else{ + s[j]=s[i]; + j++; + } + } + s[j]='\0'; + return s; +} +char* str_unescape(char* s) +{ + int i=0,j=0; + int len=strlen(s); + for(i=0,j=0;i #include #include @@ -18,13 +18,7 @@ #define MAX_CONFIG_FN_LEN 256 #define MAX_CONFIG_LINE (1024*16) -#ifndef MAX -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) -#endif -#ifndef MIN -#define MIN(a, b) (((a) < (b)) ? (a) : (b)) -#endif //#define USING_DICTATOR 1 extern "C" void __real_free(void*p); struct cm_table_info_t diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index c70ebc3..29a538a 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -13,6 +13,8 @@ #include "hiredis.h" #include "map_str2int.h" #include "Maat_rule_internal.h" +#include "Maat_utils.h" + #define maat_json (module_name_str("MAAT_JSON")) const char* untitled_group_name="Untitled"; diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 11e6ae9..3621e41 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -43,9 +43,9 @@ typedef int atomic_t; #define MAX_TABLE_LINE_SIZE (1024*16) #define MAX_EXPR_KEYLEN 1024 #define MAX_DISTRICT_LEN 64 -#define MAX_PLUGING_NUM 32 - -#define MAX_SCANNER_HIT_NUM 64 +#define MAX_PLUGIN_PER_TABLE 32 +#define MAX_FOREIGN_CLMN_NUM 8 +#define MAX_SCANNER_HIT_NUM 64 #define MAX_GROUP_CACHE 128 @@ -210,7 +210,7 @@ struct _callback_plugin struct _plugin_table_info { int cb_plug_cnt; - struct _callback_plugin cb_plug[MAX_PLUGING_NUM]; + struct _callback_plugin cb_plug[MAX_PLUGIN_PER_TABLE]; dynamic_array_t *cache_lines; int cache_line_num; int acc_line_num; @@ -262,6 +262,8 @@ struct _Maat_table_info_t struct _plugin_table_info *cb_info; int valid_flag_column; //for plugin table int rule_tag_column; //for plugin table; + int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table; + int n_foreign; int user_region_encoding; //for compile table, USER_REGION_ENCODE_xx //for stat>>>>>>>> unsigned long long udpate_err_cnt; @@ -298,7 +300,8 @@ enum maat_garbage_type GARBAGE_GROUP_RULE, GARBAGE_COMPILE_RULE, GARBAGE_BOOL_MATCHER, - GARBAGE_MAP_STR2INT + GARBAGE_MAP_STR2INT, + GARBAGE_FOREIGN_FILE }; struct iconv_handle_t { @@ -409,6 +412,8 @@ struct _Maat_feather_t struct rule_tag *accept_tags; int n_tags; + char foreign_cont_dir[MAX_TABLE_NAME_LEN]; + int foreign_cont_linger; //internal states long long new_version; int active_plugin_table_num; @@ -441,6 +446,7 @@ struct _maat_garbage_t { enum maat_garbage_type type; time_t create_time; + int expire_after; int ok_times; union { @@ -450,8 +456,15 @@ struct _maat_garbage_t void* bool_matcher; void * raw; MESA_htable_handle str2int_map; + char* filename; }; }; +struct foreign_key +{ + int column; + char* key; + char* filename; +}; struct serial_rule_t //rm= Redis Maat { enum MAAT_OPERATION op;//0: delete, 1: add. @@ -460,9 +473,12 @@ struct serial_rule_t //rm= Redis Maat long long timeout; // absolute unix time. char table_name[256]; char* table_line; + int n_foreign; + struct foreign_key* f_keys; }; int parse_accept_tag(const char* value, struct rule_tag** result, void* logger); void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); +void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set); int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); @@ -471,15 +487,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para); void maat_finish_cb(void* u_para); void *thread_rule_monitor(void *arg); unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge); -inline void ipv6_ntoh(unsigned int *v6_addr) -{ - unsigned int i=0; - for(i=0;i<4;i++) - { - v6_addr[i]=ntohl(v6_addr[i]); - } - return; -} + void * HASH_fetch_by_id(MESA_htable_handle hash,int id); int HASH_add_by_id(MESA_htable_handle hash,int id,void*data); int HASH_delete_by_id(MESA_htable_handle hash,int id); @@ -487,8 +495,7 @@ void maat_read_full_config(_Maat_feather_t* _feather); void maat_stat_init(struct _Maat_feather_t* feather); void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num); void maat_stat_output(struct _Maat_feather_t* feather); -char* _maat_strdup(const char* s); -char* str_unescape(char* s); + redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...); int get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, long long* new_version, struct serial_rule_t** list,int *update_type, void* logger, int cumulative_off); int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger,int print_process); @@ -505,8 +512,6 @@ void redis_monitor_traverse(long long version,redisContext *c, const unsigned char* dec_key, _Maat_feather_t* feather); -const char* module_name_str(const char*name); -#define maat_module (module_name_str("MAAT_Frame")) #endif diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h new file mode 100644 index 0000000..6f86504 --- /dev/null +++ b/src/inc_internal/Maat_utils.h @@ -0,0 +1,43 @@ +#pragma once +#include +#include +#include + +#define ALLOC(type, number) ((type *)calloc(sizeof(type), number)) +#define FREE(p) {free(*p);*p=NULL;} + +#define ATOMIC_INC(x) __atomic_fetch_add(x,1,__ATOMIC_RELAXED) +#define ATOMIC_READ(x) __atomic_fetch_add(x,0,__ATOMIC_RELAXED) + + +#ifndef MAX +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#endif + +#ifndef MIN +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif + +const char* module_name_str(const char*name); +#define maat_module (module_name_str("MAAT_Frame")) + +char* _maat_strdup(const char* s); +char* str_unescape(char* s); +inline void ipv6_ntoh(unsigned int *v6_addr) +{ + unsigned int i=0; + for(i=0;i<4;i++) + { + v6_addr[i]=ntohl(v6_addr[i]); + } + return; +} +int hex2bin(char *hex,int hex_len,char *binary,int size); +char* str_tolower(char* string); +char *strtok_r_esc(char *s, const char delim, char **save_ptr); +char *str_unescape_and(char*s); +char* str_unescape(char* s); +pid_t gettid(void); +int system_cmd_mkdir(const char* path); +int system_cmd_rm(const char* src_file); + diff --git a/test/table_info.conf b/test/table_info.conf index 617e8fe..13a4433 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -7,10 +7,10 @@ #do_merege [yes/no] #cross cache [number] #quick mode [quickon/quickoff], default [quickoff] -#For ip/intval/digest/compile/group +#For ip/intval/digest/compile/group table. #id name type # -#For plugin table +#For plugin table. The first column's id is 1. 0 as not speicified. #id name type valid_column tag_column # #For expr/expr_plus Table @@ -27,4 +27,5 @@ 8 HTTP_REGION expr_plus GBK GBK no 0 9 SIM_URL similar -- 10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff -11 TEST_EFFECTIVE_RANGE_TABLE plugin 4 5 -- \ No newline at end of file +11 TEST_EFFECTIVE_RANGE_TABLE plugin valid_pos=4;tag_pos=5; -- +12 TEST_EXETERN_LINK plugin valid_pos=3;foreign_pos=6,7; \ No newline at end of file