diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 5d17446..2648ecf 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -532,8 +532,10 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo case MAAT_OPT_JSON_FILE_PATH: ret=json2iris((const char*)value ,_feather->compile_tn,_feather->group_tn + ,NULL ,_feather->full_dir - ,sizeof(_feather->full_dir),_feather->logger); + ,sizeof(_feather->full_dir) + ,_feather->logger); if(ret<0) { return -1; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index c80d6de..560bf94 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -30,7 +30,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_1_20171221_dev=1; +int MAAT_FRAME_VERSION_2_1_20180126=1; const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; diff --git a/src/entry/Maat_rule_internal.h b/src/entry/Maat_rule_internal.h index 363ed1e..9b1f41d 100644 --- a/src/entry/Maat_rule_internal.h +++ b/src/entry/Maat_rule_internal.h @@ -473,7 +473,11 @@ char* str_unescape(char* s); redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...); int get_rm_key_list(long long version,redisContext *c,struct serial_rule_t** list,void* logger, long long* new_version,int *update_type, int cumulative_off); int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger,int print_process); +void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout); void empty_serial_rules(struct serial_rule_t* rule); +int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_rule_num, long long server_time, void* logger); +long long redis_server_time(redisContext* ctx); + void redis_monitor_traverse(long long version,redisContext *c, void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para diff --git a/src/entry/config_monitor.cpp b/src/entry/config_monitor.cpp index 07a1e0e..9a0b63b 100644 --- a/src/entry/config_monitor.cpp +++ b/src/entry/config_monitor.cpp @@ -492,12 +492,18 @@ void config_monitor_traverse(long long version,const char*idx_dir, } table_filename=path2filename(idx_path_array[i]); sscanf(table_filename,"%[a-zA-Z]_config_index.%lld",str_not_care,&new_version); - start(new_version,update_type,u_para); + if(start!=NULL) + { + start(new_version,update_type,u_para); + } for(j=0;j #include #include "cJSON.h" +#include "hiredis.h" #include "map_str2int.h" #include "Maat_rule_internal.h" #define maat_json (module_name_str("MAAT_JSON")) @@ -42,7 +43,7 @@ struct iris_description_t MESA_htable_handle group_name_map; MESA_htable_handle iris_table_map; MESA_htable_handle str2int_map; - + redisContext *redis_write_ctx; }; struct traslate_command_t { @@ -54,7 +55,42 @@ struct traslate_command_t const char* default_string; int default_int; }; -int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,const char* group_tn,struct iris_description_t *iris_cfg,void * logger) +static int get_group_seq(struct iris_description_t* iris_cfg) +{ + redisReply* data_reply=NULL; + int sequence=0; + if(iris_cfg->redis_write_ctx==NULL) + { + sequence=iris_cfg->group_cnt; + } + else + { + data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_GROUP 1"); + sequence=(int)data_reply->integer-1; + freeReplyObject(data_reply); + } + iris_cfg->group_cnt++; + return sequence; +} +static int get_region_seq(struct iris_description_t* iris_cfg) +{ + redisReply* data_reply=NULL; + int sequence=0; + if(iris_cfg->redis_write_ctx==NULL) + { + sequence=iris_cfg->region_cnt; + } + else + { + data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_REGION 1"); + sequence=(int)data_reply->integer-1; + freeReplyObject(data_reply); + } + iris_cfg->region_cnt++; + return sequence; +} + +int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,const char* group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger) { memset(iris_cfg,0,sizeof(struct iris_description_t)); snprintf(iris_cfg->tmp_iris_dir,sizeof(iris_cfg->tmp_iris_dir),"%s_iris_tmp",json_file); @@ -69,7 +105,8 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn, memcpy(iris_cfg->group_table.table_name,group_tn,strlen(group_tn)); snprintf(iris_cfg->group_table.table_path,sizeof(iris_cfg->group_table.table_name), "%s/%s.local",iris_cfg->tmp_iris_dir,iris_cfg->group_table.table_name); - + + iris_cfg->redis_write_ctx=redis_write_ctx; MESA_htable_create_args_t hargs; memset(&hargs,0,sizeof(hargs)); hargs.thread_safe=1; @@ -617,7 +654,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri return -1; } } - region_id=p_iris->region_cnt++; + region_id=get_region_seq(p_iris); cJSON_AddNumberToObject(table_content, "region_id", region_id); cJSON_AddNumberToObject(table_content, "group_id", group_id); cJSON_AddNumberToObject(table_content, "is_valid", 1); @@ -866,12 +903,12 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger) if(0==strncasecmp(group_name,untitled_group_name,strlen(untitled_group_name))) { group_info=&untitled_group; - group_info->group_id=p_iris->group_cnt++; + group_info->group_id=get_group_seq(p_iris); } else { group_info=(struct group_info_t*)malloc(sizeof(struct group_info_t)); - group_info->group_id=p_iris->group_cnt++; + group_info->group_id=get_group_seq(p_iris); MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info); } r_rules=cJSON_GetObjectItem(group_rule,"regions"); @@ -916,10 +953,10 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger) } return 0; } -int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,char* iris_dir_buf,int buf_len,void* logger) +int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger) { FILE* json_fp=NULL; - cJSON *json=NULL; + cJSON *json=NULL, *tmp_obj=NULL; struct stat fstat_buf; int ret=-1; char* json_buff=NULL; @@ -960,7 +997,17 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,c MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,"Error before: %-200.200s",cJSON_GetErrorPtr()); goto error_out; } - ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,&iris_cfg,logger); + if(compile_tn==NULL) + { + tmp_obj=cJSON_GetObjectItem(json,"compile_table"); + compile_tn=tmp_obj->valuestring; + } + if(group_tn==NULL) + { + tmp_obj=cJSON_GetObjectItem(json,"group_table"); + group_tn=tmp_obj->valuestring; + } + ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,redis_write_ctx,&iris_cfg,logger); if(ret<0) { goto error_out; diff --git a/src/entry/json2iris.h b/src/entry/json2iris.h index b6b08f3..f61f6e5 100644 --- a/src/entry/json2iris.h +++ b/src/entry/json2iris.h @@ -1,6 +1,6 @@ #ifndef H_MAAT_JSON2IRIS_H_INCLUDE #define H_MAAT_JSON2IRIS_H_INCLUDE -int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,char* iris_dir_buf,int buf_len,void* logger); +int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger); int set_file_rulenum(const char* path,int rulenum,void* logger); #endif diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp index 194493b..4304117 100644 --- a/tools/maat_redis_tool.cpp +++ b/tools/maat_redis_tool.cpp @@ -1,15 +1,18 @@ #include "Maat_rule.h" #include "Maat_command.h" #include "Maat_rule_internal.h" +#include "cJSON.h" #include "json2iris.h" #include "config_monitor.h" #include "hiredis.h" +#include #include #include #include #include #include const char* redis_dump_dir="./redis_dump"; +const char* default_table_info="./table_info.conf"; void maat_tool_print_usage(void) { printf("maat_redis_tool manipulate rules from redis.\n"); @@ -17,8 +20,10 @@ void maat_tool_print_usage(void) printf("\t-h [host], redis IP, 127.0.0.1 as default.\n"); printf("\t-p [port], redis port, 6379 as default.\n"); printf("\t-d [dir], dump rules from redis to [dir], %s as default.\n",redis_dump_dir); + printf("\t-j [payload.json], add or delete rules as maat json. Must have field compile_table field, and plugin table's valid flag must be in the last column.\n"); + printf("\t-t [timeout], timeout config after t seconds, default 0, not timeout.\n"); printf("example: ./maat_redis_tool -h 127.0.0.1 -p 6379 -d %s\n",redis_dump_dir); - + printf(" ./maat_redis_tool -h 127.0.0.1 -p 6379 -j payload.json -t 300\n"); return; } static int compare_serial_rule(const void *a, const void *b) @@ -31,7 +36,23 @@ static int compare_serial_rule(const void *a, const void *b) snprintf(q_str,sizeof(q_str),"%s.%d",rb->table_name,rb->rule_id); return strcmp(p_str,q_str); } -void read_rule_from_redis(const char*redis_ip, int redis_port, int redis_db,const char* output_path ,void*logger) +static redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db) +{ + struct timeval connect_timeout; + connect_timeout.tv_sec=0; + connect_timeout.tv_usec=100*1000; // 100 ms + + redisContext * ctx; + ctx=redisConnectWithTimeout(redis_ip, redis_port,connect_timeout); + if(ctx==NULL) + { + printf("Unable to connect %s:%d db%d\n",redis_ip,redis_port,redis_db); + return NULL; + } + return ctx; + +} +void read_rule_from_redis(redisContext * ctx,const char* output_path ,void*logger) { struct serial_rule_t* rule_list; int rule_num=0,line_count=0; @@ -43,19 +64,6 @@ void read_rule_from_redis(const char*redis_ip, int redis_port, int redis_db,cons char table_path[256],index_path[256]; FILE *table_fp=NULL, *index_fp=NULL; - struct timeval connect_timeout; - connect_timeout.tv_sec=0; - connect_timeout.tv_usec=100*1000; // 100 ms - - redisContext * ctx; - ctx=redisConnectWithTimeout(redis_ip, redis_port,connect_timeout); - if(ctx==NULL) - { - printf("Unable to connect %s:%d db%d\n",redis_ip,redis_port,redis_db); - return; - } - - printf("Reading key list from %s:%d db%d.\n",redis_ip,redis_port,redis_db); rule_num=get_rm_key_list(0, ctx, &rule_list, logger,&version, &update_type,0); if(rule_num==0) { @@ -146,41 +154,137 @@ clean_up: } return; } +int count_line_num(const char* table_name,const char* line,void *u_para) +{ + (*((int *)u_para))++; + return 0; +} +int line_idx=0; +long long server_time=0,absolute_expire_time=0; +char compile_table_name[128]; +int make_serial_rule(const char* table_name,const char* line,void *u_para) +{ + struct serial_rule_t* s_rule=(struct serial_rule_t*)u_para; + int rule_id=0,is_valid=0; + char *str1=NULL, *token=NULL; + char *saveptr1=NULL,*last1=NULL; + int j=0; + char *buff=(char*)malloc(strlen(line)+1); + memcpy(buff,line, strlen(line)+1); + while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t') + { + buff[strlen(buff)-1]='\0'; + } + + for (j = 0,str1=buff; ; j++, str1 = NULL) { + token = strtok_r(str1, "\t ", &saveptr1); + if (token == NULL) + break; + if(j==0) + { + sscanf(token,"%d",&rule_id); + } + last1=token; + } + sscanf(last1,"%d",&is_valid); + + memcpy(buff,line, strlen(line)+1); + while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t') + { + buff[strlen(buff)-1]='\0'; + } + + set_serial_rule(s_rule+line_idx, (enum MAAT_OPERATION)is_valid, rule_id, 0, table_name, buff, absolute_expire_time); + line_idx++; + free(str1); + return 0; +} + +#define WORK_MODE_DUMP 0 +#define WORK_MODE_JSON 1 int main(int argc, char * argv[]) { - int oc=0; - //char model='?'; + int oc=0,ret=0, i=0,success_cnt=0; + int model=0; char redis_ip[64]; int redis_port=6379; int redis_db=0; strncpy(redis_ip,"127.0.0.1",sizeof(redis_ip)); char table_info[128]; strncpy(table_info,"./table_info.conf",sizeof(table_info)); - char dump_dir[128]; + char dump_dir[128], json_file[128], tmp_iris_path[128]; strncpy(dump_dir,redis_dump_dir,sizeof(dump_dir)); - while((oc=getopt(argc,argv,"mh:p:t:d:f:"))!=-1) + redisContext * ctx=NULL; + int total_line_cnt=0; + int timeout=0; + FILE* json_fp=NULL; + cJSON *json=NULL, *tmp_obj=NULL; + struct stat fstat_buf; + unsigned long json_file_size=0,read_size=0; + char* json_buff=NULL; + + while((oc=getopt(argc,argv,"h:p:d:f:j:t:"))!=-1) { switch(oc) { - case 'm': - //model=oc; - break; case 'h': strncpy(redis_ip,optarg,sizeof(redis_ip)); break; case 'p': sscanf(optarg,"%d",&redis_port); break; - case 't': - strncpy(table_info,optarg,sizeof(table_info)); - break; case 'd': + model=WORK_MODE_DUMP; strncpy(dump_dir,optarg,sizeof(dump_dir)); if(dump_dir[strlen(dump_dir)-1]=='/') { dump_dir[strlen(dump_dir)-1]='\0'; } break; + case 'j': + strncpy(json_file, optarg,sizeof(json_file)); + model=WORK_MODE_JSON; + ret=stat(json_file, &fstat_buf); + if(ret!=0) + { + printf("fstat file %s error.\n",json_file); + return -1; + } + json_fp=fopen(json_file,"r"); + if(json_fp==NULL) + { + printf("fopen file %s error %s.\n",json_file,strerror(errno)); + return -1; + } + json_file_size=fstat_buf.st_size; + json_buff=(char*)calloc(1,json_file_size); + read_size=fread(json_buff,1,json_file_size,json_fp); + if(read_size!=json_file_size) + { + printf("fread file %s error.",json_file); + return -1; + } + + json=cJSON_Parse(json_buff); + if (!json) + { + printf("%s format is error before: %-200.200s",json_file,cJSON_GetErrorPtr()); + return -1; + } + tmp_obj=cJSON_GetObjectItem(json,"compile_table"); + if(tmp_obj==NULL) + { + printf("No \"compile_table\" in %s.\n",json_file); + return -1; + } + strncpy(compile_table_name,tmp_obj->valuestring,sizeof(compile_table_name)); + free(json_buff); + cJSON_Delete(json); + fclose(json_fp); + break; + case 't': + sscanf(optarg,"%d",&timeout); + break; case '?': default: maat_tool_print_usage(); @@ -188,5 +292,47 @@ int main(int argc, char * argv[]) break; } } - read_rule_from_redis(redis_ip,redis_port, redis_db,dump_dir, NULL); + ctx=connect_redis(redis_ip,redis_port, redis_db); + if(ctx==NULL) + { + return -1; + } + if(model==WORK_MODE_DUMP) + { + printf("Reading key list from %s:%d db%d.\n",redis_ip,redis_port,redis_db); + read_rule_from_redis(ctx,dump_dir, NULL); + } + else if(model==WORK_MODE_JSON) + { + ret=json2iris(json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL); + if(ret<0) + { + printf("Invalid json format.\n"); + } + total_line_cnt=0; + config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num, NULL, &total_line_cnt,NULL, NULL); + printf("Serialize %s to %d lines.\nNote that group reuse is NOT supported.\n",json_file, total_line_cnt); + struct serial_rule_t * s_rule=(struct serial_rule_t *)calloc(sizeof(struct serial_rule_t),total_line_cnt); + server_time=redis_server_time(ctx); + if(timeout>0) + { + absolute_expire_time=server_time+timeout; + } + config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule,NULL, NULL); + ret=0; + while(success_cnt