maat_redis_tool支持通过命令行导入json格式的配置，注意不支持分组复用。

src/entry/Maat_api.cpp

@@ -532,8 +532,10 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
 		case MAAT_OPT_JSON_FILE_PATH:
 			ret=json2iris((const char*)value
 				,_feather->compile_tn,_feather->group_tn
+				,NULL
 				,_feather->full_dir
-				,sizeof(_feather->full_dir),_feather->logger);
+				,sizeof(_feather->full_dir)
+				,_feather->logger);
 			if(ret<0)
 			{
 				return -1;

src/entry/Maat_rule.cpp

@@ -30,7 +30,7 @@
 #include "stream_fuzzy_hash.h"
 #include "gram_index_engine.h"
 
-int MAAT_FRAME_VERSION_2_1_20171221_dev=1;
+int MAAT_FRAME_VERSION_2_1_20180126=1;
 
 const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
 							"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};

src/entry/Maat_rule_internal.h

@@ -473,7 +473,11 @@ char* str_unescape(char* s);
 redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...);
 int get_rm_key_list(long long version,redisContext *c,struct serial_rule_t** list,void* logger, long long* new_version,int *update_type, int cumulative_off);
 int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger,int print_process);
+void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout);
 void empty_serial_rules(struct serial_rule_t* rule);
+int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_rule_num, long long server_time, void* logger);
+long long redis_server_time(redisContext* ctx);
+
 void redis_monitor_traverse(long        long version,redisContext *c,
 		void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para
 		int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para

src/entry/config_monitor.cpp

@@ -492,12 +492,18 @@ void config_monitor_traverse(long long version,const char*idx_dir,
 			}
 			table_filename=path2filename(idx_path_array[i]);
 			sscanf(table_filename,"%[a-zA-Z]_config_index.%lld",str_not_care,&new_version);
-			start(new_version,update_type,u_para);
+			if(start!=NULL)
+			{
+				start(new_version,update_type,u_para);
+			}
 			for(j=0;j<table_num;j++)
 			{
 				cm_read_table_file(table_array+j,update,u_para,dec_key,logger);
 			}
-			finish(u_para);
+			if(finish!=NULL)
+			{
+				finish(u_para);
+			}
 		}
 
 	}

src/entry/json2iris.cpp

@@ -10,6 +10,7 @@
 #include <stdlib.h>
 #include <MESA/MESA_htable.h>
 #include "cJSON.h"
+#include "hiredis.h"
 #include "map_str2int.h"
 #include "Maat_rule_internal.h"
 #define maat_json (module_name_str("MAAT_JSON"))
@@ -42,7 +43,7 @@ struct iris_description_t
 	MESA_htable_handle group_name_map;
 	MESA_htable_handle iris_table_map;
 	MESA_htable_handle str2int_map;
-	
+	redisContext *redis_write_ctx;
 };
 struct traslate_command_t
 {
@@ -54,7 +55,42 @@ struct traslate_command_t
 	const char* default_string;
 	int default_int;
 };
-int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,const char* group_tn,struct iris_description_t *iris_cfg,void * logger)
+static int get_group_seq(struct iris_description_t* iris_cfg)
+{
+	redisReply* data_reply=NULL;
+	int sequence=0;
+	if(iris_cfg->redis_write_ctx==NULL)
+	{
+		sequence=iris_cfg->group_cnt;
+	}
+	else
+	{
+		data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_GROUP 1");
+		sequence=(int)data_reply->integer-1;
+		freeReplyObject(data_reply);
+	}
+	iris_cfg->group_cnt++;
+	return sequence;
+}
+static int get_region_seq(struct iris_description_t* iris_cfg)
+{
+	redisReply* data_reply=NULL;
+	int sequence=0;
+	if(iris_cfg->redis_write_ctx==NULL)
+	{
+		sequence=iris_cfg->region_cnt;
+	}
+	else
+	{
+		data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_REGION 1");
+		sequence=(int)data_reply->integer-1;
+		freeReplyObject(data_reply);
+	}
+	iris_cfg->region_cnt++;
+	return sequence;
+}
+
+int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,const char* group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger)
 {
 	memset(iris_cfg,0,sizeof(struct iris_description_t));
 	snprintf(iris_cfg->tmp_iris_dir,sizeof(iris_cfg->tmp_iris_dir),"%s_iris_tmp",json_file);
@@ -69,7 +105,8 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,
 	memcpy(iris_cfg->group_table.table_name,group_tn,strlen(group_tn));
 	snprintf(iris_cfg->group_table.table_path,sizeof(iris_cfg->group_table.table_name),
 			"%s/%s.local",iris_cfg->tmp_iris_dir,iris_cfg->group_table.table_name);
-	
+
+	iris_cfg->redis_write_ctx=redis_write_ctx;
 	MESA_htable_create_args_t hargs;
 	memset(&hargs,0,sizeof(hargs));
 	hargs.thread_safe=1;
@@ -617,7 +654,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri
 			return -1;
 		}
 	}
-	region_id=p_iris->region_cnt++;
+	region_id=get_region_seq(p_iris);
 	cJSON_AddNumberToObject(table_content, "region_id", region_id);
 	cJSON_AddNumberToObject(table_content, "group_id", group_id);
 	cJSON_AddNumberToObject(table_content, "is_valid", 1);
@@ -866,12 +903,12 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
 			if(0==strncasecmp(group_name,untitled_group_name,strlen(untitled_group_name)))
 			{
 				group_info=&untitled_group;
-				group_info->group_id=p_iris->group_cnt++;
+				group_info->group_id=get_group_seq(p_iris);
 			}
 			else
 			{
 				group_info=(struct group_info_t*)malloc(sizeof(struct group_info_t));
-				group_info->group_id=p_iris->group_cnt++;
+				group_info->group_id=get_group_seq(p_iris);
 				MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info);
 			}
 			r_rules=cJSON_GetObjectItem(group_rule,"regions");
@@ -916,10 +953,10 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
 	}
 	return 0;
 }
-int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,char* iris_dir_buf,int buf_len,void* logger)
+int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger)
 {
 	FILE* json_fp=NULL;
-	cJSON *json=NULL;
+	cJSON *json=NULL, *tmp_obj=NULL;
 	struct stat fstat_buf;
 	int ret=-1;
 	char* json_buff=NULL;
@@ -960,7 +997,17 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,c
 		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,"Error before: %-200.200s",cJSON_GetErrorPtr());
 		goto error_out;
 	}
-	ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,&iris_cfg,logger);
+	if(compile_tn==NULL)
+	{
+		tmp_obj=cJSON_GetObjectItem(json,"compile_table");
+		compile_tn=tmp_obj->valuestring;
+	}
+	if(group_tn==NULL)
+	{
+		tmp_obj=cJSON_GetObjectItem(json,"group_table");
+		group_tn=tmp_obj->valuestring;
+	}
+	ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,redis_write_ctx,&iris_cfg,logger);
 	if(ret<0)
 	{
 		goto error_out;

src/entry/json2iris.h

@@ -1,6 +1,6 @@
 #ifndef H_MAAT_JSON2IRIS_H_INCLUDE
 #define H_MAAT_JSON2IRIS_H_INCLUDE
-int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,char* iris_dir_buf,int buf_len,void* logger);
+int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger);
 int set_file_rulenum(const char* path,int rulenum,void* logger);
 #endif
 

tools/maat_redis_tool.cpp

@@ -1,15 +1,18 @@
 #include "Maat_rule.h"
 #include "Maat_command.h"
 #include "Maat_rule_internal.h"
+#include "cJSON.h"
 #include "json2iris.h"
 #include "config_monitor.h"
 #include "hiredis.h"
+#include <MESA/MESA_handle_logger.h>
 #include <unistd.h>
 #include <stdio.h>
 #include <assert.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 const char* redis_dump_dir="./redis_dump";
+const char* default_table_info="./table_info.conf";
 void maat_tool_print_usage(void)
 {
 	printf("maat_redis_tool manipulate rules from redis.\n");
@@ -17,8 +20,10 @@ void maat_tool_print_usage(void)
 	printf("\t-h [host], redis IP, 127.0.0.1 as default.\n");
 	printf("\t-p [port], redis port, 6379 as default.\n");
 	printf("\t-d [dir], dump rules from redis to [dir], %s as default.\n",redis_dump_dir);
+	printf("\t-j [payload.json], add or delete rules as maat json. Must have field compile_table field, and plugin table's valid flag must be in the last column.\n");
+	printf("\t-t [timeout], timeout config after t seconds, default 0, not timeout.\n");	
 	printf("example: ./maat_redis_tool -h 127.0.0.1 -p 6379 -d %s\n",redis_dump_dir);
-	
+	printf("	./maat_redis_tool -h 127.0.0.1 -p 6379 -j payload.json -t 300\n");
 	return;
 }
 static int compare_serial_rule(const void *a, const void *b)
@@ -31,7 +36,23 @@ static int compare_serial_rule(const void *a, const void *b)
 	snprintf(q_str,sizeof(q_str),"%s.%d",rb->table_name,rb->rule_id);
 	return strcmp(p_str,q_str);
 }
-void read_rule_from_redis(const char*redis_ip, int redis_port, int redis_db,const char* output_path ,void*logger)
+static redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db)
+{
+	struct timeval connect_timeout;
+	connect_timeout.tv_sec=0;
+	connect_timeout.tv_usec=100*1000; // 100 ms
+
+	redisContext * ctx;
+	ctx=redisConnectWithTimeout(redis_ip, redis_port,connect_timeout);
+	if(ctx==NULL)
+	{
+		printf("Unable to connect %s:%d db%d\n",redis_ip,redis_port,redis_db);
+		return NULL;
+	}
+	return ctx;
+
+}
+void read_rule_from_redis(redisContext * ctx,const char* output_path ,void*logger)
 {
 	struct serial_rule_t* rule_list;
 	int rule_num=0,line_count=0;
@@ -43,19 +64,6 @@ void read_rule_from_redis(const char*redis_ip, int redis_port, int redis_db,cons
 	char table_path[256],index_path[256];
 	FILE *table_fp=NULL, *index_fp=NULL;
 
-	struct timeval connect_timeout;
-	connect_timeout.tv_sec=0;
-	connect_timeout.tv_usec=100*1000; // 100 ms
-
-	redisContext * ctx;
-	ctx=redisConnectWithTimeout(redis_ip, redis_port,connect_timeout);
-	if(ctx==NULL)
-	{
-		printf("Unable to connect %s:%d db%d\n",redis_ip,redis_port,redis_db);
-		return;
-	}
-
-	printf("Reading key list from %s:%d db%d.\n",redis_ip,redis_port,redis_db);
 	rule_num=get_rm_key_list(0, ctx, &rule_list, logger,&version, &update_type,0);
 	if(rule_num==0)
 	{
@@ -146,41 +154,137 @@ clean_up:
 	}
 	return;
 }
+int count_line_num(const char* table_name,const char* line,void *u_para)
+{
+	(*((int *)u_para))++;
+	return 0;
+}
+int line_idx=0;
+long long server_time=0,absolute_expire_time=0;
+char compile_table_name[128];
+int make_serial_rule(const char* table_name,const char* line,void *u_para)
+{
+	struct serial_rule_t* s_rule=(struct serial_rule_t*)u_para;
+	int rule_id=0,is_valid=0;
+	char *str1=NULL, *token=NULL;
+	char *saveptr1=NULL,*last1=NULL;
+	int j=0;
+	char *buff=(char*)malloc(strlen(line)+1);
+	memcpy(buff,line, strlen(line)+1);
+	while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t')
+	{
+		buff[strlen(buff)-1]='\0';
+	}
+
+	for (j = 0,str1=buff; ; j++, str1 = NULL) {
+		 token = strtok_r(str1, "\t ", &saveptr1);
+		 if (token == NULL)
+			 break;
+		 if(j==0)
+		 {
+			sscanf(token,"%d",&rule_id);
+		 }
+		 last1=token;
+ 	 }
+	sscanf(last1,"%d",&is_valid);
+	
+	memcpy(buff,line, strlen(line)+1);
+	while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t')
+	{
+		buff[strlen(buff)-1]='\0';
+	}
+
+	set_serial_rule(s_rule+line_idx, (enum MAAT_OPERATION)is_valid, rule_id, 0, table_name, buff, absolute_expire_time);
+	line_idx++;
+	free(str1);
+	return 0;
+}
+
+#define	WORK_MODE_DUMP	0
+#define WORK_MODE_JSON	1
 int main(int argc, char * argv[])
 {
-	int oc=0;
+	int oc=0,ret=0, i=0,success_cnt=0;
-	//char model='?';
+	int model=0;
 	char redis_ip[64];
 	int redis_port=6379;
 	int redis_db=0;
 	strncpy(redis_ip,"127.0.0.1",sizeof(redis_ip));
 	char table_info[128];
 	strncpy(table_info,"./table_info.conf",sizeof(table_info));
-	char dump_dir[128];
+	char dump_dir[128], json_file[128], tmp_iris_path[128];
 	strncpy(dump_dir,redis_dump_dir,sizeof(dump_dir));
-	while((oc=getopt(argc,argv,"mh:p:t:d:f:"))!=-1)
+	redisContext * ctx=NULL;
+	int total_line_cnt=0;
+	int timeout=0;
+	FILE* json_fp=NULL;
+	cJSON *json=NULL, *tmp_obj=NULL;
+	struct stat fstat_buf;
+	unsigned long json_file_size=0,read_size=0;
+	char* json_buff=NULL;
+
+	while((oc=getopt(argc,argv,"h:p:d:f:j:t:"))!=-1)
 	{
 		switch(oc)
 		{
-			case 'm':
-				//model=oc;
-				break;
 			case 'h':
 				strncpy(redis_ip,optarg,sizeof(redis_ip));
 				break;
 			case 'p':
 				sscanf(optarg,"%d",&redis_port);
 				break;
-			case 't':
-				strncpy(table_info,optarg,sizeof(table_info));
-				break;
 			case 'd':
+				model=WORK_MODE_DUMP;
 				strncpy(dump_dir,optarg,sizeof(dump_dir));
 				if(dump_dir[strlen(dump_dir)-1]=='/')
 				{
 					dump_dir[strlen(dump_dir)-1]='\0';
 				}
 				break;
+			case 'j':
+				strncpy(json_file, optarg,sizeof(json_file));
+				model=WORK_MODE_JSON;
+				ret=stat(json_file, &fstat_buf);
+				if(ret!=0)
+				{
+					printf("fstat file %s error.\n",json_file);
+					return -1;
+				}				
+				json_fp=fopen(json_file,"r");
+				if(json_fp==NULL)
+				{
+					printf("fopen file %s error %s.\n",json_file,strerror(errno));
+					return -1;
+				}
+				json_file_size=fstat_buf.st_size;
+				json_buff=(char*)calloc(1,json_file_size);
+				read_size=fread(json_buff,1,json_file_size,json_fp);
+				if(read_size!=json_file_size)
+				{
+					printf("fread file %s error.",json_file);
+					return -1;
+				}
+
+				json=cJSON_Parse(json_buff);
+				if (!json)
+				{
+					printf("%s format is error before: %-200.200s",json_file,cJSON_GetErrorPtr());
+					return -1;
+				}
+				tmp_obj=cJSON_GetObjectItem(json,"compile_table");
+				if(tmp_obj==NULL)
+				{
+					printf("No \"compile_table\" in %s.\n",json_file);
+					return -1;
+				}
+				strncpy(compile_table_name,tmp_obj->valuestring,sizeof(compile_table_name));
+				free(json_buff);
+				cJSON_Delete(json);
+				fclose(json_fp);
+				break;
+			case 't':
+				sscanf(optarg,"%d",&timeout);
+				break;
 			case '?':
 			default:
 				maat_tool_print_usage();
@@ -188,5 +292,47 @@ int main(int argc, char * argv[])
 				break;
 		}
 	}
-	read_rule_from_redis(redis_ip,redis_port, redis_db,dump_dir, NULL);
+	ctx=connect_redis(redis_ip,redis_port, redis_db);
+	if(ctx==NULL)
+	{
+		return -1;
+	}
+	if(model==WORK_MODE_DUMP)
+	{
+		printf("Reading key list from %s:%d db%d.\n",redis_ip,redis_port,redis_db);
+		read_rule_from_redis(ctx,dump_dir, NULL);
+	}
+	else if(model==WORK_MODE_JSON)
+	{		
+		ret=json2iris(json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL);
+		if(ret<0)
+		{
+			printf("Invalid json format.\n");
+		}
+		total_line_cnt=0;
+		config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num, NULL, &total_line_cnt,NULL, NULL);
+		printf("Serialize %s to %d lines.\nNote that group reuse is NOT supported.\n",json_file, total_line_cnt);
+		struct serial_rule_t * s_rule=(struct serial_rule_t *)calloc(sizeof(struct serial_rule_t),total_line_cnt);
+		server_time=redis_server_time(ctx);
+		if(timeout>0)
+		{
+			absolute_expire_time=server_time+timeout;
+		}
+		config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule,NULL, NULL);
+		ret=0;
+		while(success_cnt<total_line_cnt)
+		{
+			success_cnt+=exec_serial_rule(ctx,s_rule+success_cnt, total_line_cnt-success_cnt, server_time, NULL);
+		}
+		for(i=0;i<total_line_cnt;i++)
+		{
+			empty_serial_rules(s_rule+i);
+		}
+		free(s_rule);
+		
+		redisFree(ctx);
+	}
+
+	return 0;
 }
+