gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

maat_redis_tool支持通过命令行导入json格式的配置，注意不支持分组复用。

Browse Source
This commit is contained in:
zhengchao
2018-01-26 18:47:51 +08:00
parent b360726c32
commit c78be056c0
7 changed files with 245 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/entry/Maat_api.cpp
View File

@@ -532,8 +532,10 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
case MAAT_OPT_JSON_FILE_PATH:
ret=json2iris((const char*)value
,_feather->compile_tn,_feather->group_tn
,NULL
,_feather->full_dir
,sizeof(_feather->full_dir),_feather->logger);
,sizeof(_feather->full_dir)
,_feather->logger);
if(ret<0)
{
return -1;

2
src/entry/Maat_rule.cpp
View File

@@ -30,7 +30,7 @@
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
int MAAT_FRAME_VERSION_2_1_20171221_dev=1;
int MAAT_FRAME_VERSION_2_1_20180126=1;
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};

4
src/entry/Maat_rule_internal.h
View File

@@ -473,7 +473,11 @@ char* str_unescape(char* s);
redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...);
int get_rm_key_list(long long version,redisContext *c,struct serial_rule_t** list,void* logger, long long* new_version,int *update_type, int cumulative_off);
int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger,int print_process);
void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout);
void empty_serial_rules(struct serial_rule_t* rule);
int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_rule_num, long long server_time, void* logger);
long long redis_server_time(redisContext* ctx);
void redis_monitor_traverse(long long version,redisContext *c,
void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para
int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para

6
src/entry/config_monitor.cpp
View File

@@ -492,13 +492,19 @@ void config_monitor_traverse(long long version,const char*idx_dir,
}
table_filename=path2filename(idx_path_array[i]);
sscanf(table_filename,"%[a-zA-Z]_config_index.%lld",str_not_care,&new_version);
if(start!=NULL)
{
start(new_version,update_type,u_para);
}
for(j=0;j<table_num;j++)
{
cm_read_table_file(table_array+j,update,u_para,dec_key,logger);
}
if(finish!=NULL)
{
finish(u_para);
}
}
}

63
src/entry/json2iris.cpp
View File

@@ -10,6 +10,7 @@
#include <stdlib.h>
#include <MESA/MESA_htable.h>
#include "cJSON.h"
#include "hiredis.h"
#include "map_str2int.h"
#include "Maat_rule_internal.h"
#define maat_json (module_name_str("MAAT_JSON"))
@@ -42,7 +43,7 @@ struct iris_description_t
MESA_htable_handle group_name_map;
MESA_htable_handle iris_table_map;
MESA_htable_handle str2int_map;
redisContext *redis_write_ctx;
};
struct traslate_command_t
{
@@ -54,7 +55,42 @@ struct traslate_command_t
const char* default_string;
int default_int;
};
int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,const char* group_tn,struct iris_description_t *iris_cfg,void * logger)
static int get_group_seq(struct iris_description_t* iris_cfg)
{
redisReply* data_reply=NULL;
int sequence=0;
if(iris_cfg->redis_write_ctx==NULL)
{
sequence=iris_cfg->group_cnt;
}
else
{
data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_GROUP 1");
sequence=(int)data_reply->integer-1;
freeReplyObject(data_reply);
}
iris_cfg->group_cnt++;
return sequence;
}
static int get_region_seq(struct iris_description_t* iris_cfg)
{
redisReply* data_reply=NULL;
int sequence=0;
if(iris_cfg->redis_write_ctx==NULL)
{
sequence=iris_cfg->region_cnt;
}
else
{
data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_REGION 1");
sequence=(int)data_reply->integer-1;
freeReplyObject(data_reply);
}
iris_cfg->region_cnt++;
return sequence;
}
int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,const char* group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger)
{
memset(iris_cfg,0,sizeof(struct iris_description_t));
snprintf(iris_cfg->tmp_iris_dir,sizeof(iris_cfg->tmp_iris_dir),"%s_iris_tmp",json_file);
@@ -70,6 +106,7 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,
snprintf(iris_cfg->group_table.table_path,sizeof(iris_cfg->group_table.table_name),
"%s/%s.local",iris_cfg->tmp_iris_dir,iris_cfg->group_table.table_name);
iris_cfg->redis_write_ctx=redis_write_ctx;
MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs));
hargs.thread_safe=1;
@@ -617,7 +654,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri
return -1;
}
}
region_id=p_iris->region_cnt++;
region_id=get_region_seq(p_iris);
cJSON_AddNumberToObject(table_content, "region_id", region_id);
cJSON_AddNumberToObject(table_content, "group_id", group_id);
cJSON_AddNumberToObject(table_content, "is_valid", 1);
@@ -866,12 +903,12 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
if(0==strncasecmp(group_name,untitled_group_name,strlen(untitled_group_name)))
{
group_info=&untitled_group;
group_info->group_id=p_iris->group_cnt++;
group_info->group_id=get_group_seq(p_iris);
}
else
{
group_info=(struct group_info_t*)malloc(sizeof(struct group_info_t));
group_info->group_id=p_iris->group_cnt++;
group_info->group_id=get_group_seq(p_iris);
MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info);
}
r_rules=cJSON_GetObjectItem(group_rule,"regions");
@@ -916,10 +953,10 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
}
return 0;
}
int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,char* iris_dir_buf,int buf_len,void* logger)
int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger)
{
FILE* json_fp=NULL;
cJSON *json=NULL;
cJSON *json=NULL, *tmp_obj=NULL;
struct stat fstat_buf;
int ret=-1;
char* json_buff=NULL;
@@ -960,7 +997,17 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,c
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,"Error before: %-200.200s",cJSON_GetErrorPtr());
goto error_out;
}
ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,&iris_cfg,logger);
if(compile_tn==NULL)
{
tmp_obj=cJSON_GetObjectItem(json,"compile_table");
compile_tn=tmp_obj->valuestring;
}
if(group_tn==NULL)
{
tmp_obj=cJSON_GetObjectItem(json,"group_table");
group_tn=tmp_obj->valuestring;
}
ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,redis_write_ctx,&iris_cfg,logger);
if(ret<0)
{
goto error_out;

2
src/entry/json2iris.h
View File

@@ -1,6 +1,6 @@
#ifndef H_MAAT_JSON2IRIS_H_INCLUDE
#define H_MAAT_JSON2IRIS_H_INCLUDE
int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,char* iris_dir_buf,int buf_len,void* logger);
int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger);
int set_file_rulenum(const char* path,int rulenum,void* logger);
#endif

198
tools/maat_redis_tool.cpp
View File

@@ -1,15 +1,18 @@
#include "Maat_rule.h"
#include "Maat_command.h"
#include "Maat_rule_internal.h"
#include "cJSON.h"
#include "json2iris.h"
#include "config_monitor.h"
#include "hiredis.h"
#include <MESA/MESA_handle_logger.h>
#include <unistd.h>
#include <stdio.h>
#include <assert.h>
#include <sys/types.h>
#include <sys/stat.h>
const char* redis_dump_dir="./redis_dump";
const char* default_table_info="./table_info.conf";
void maat_tool_print_usage(void)
{
printf("maat_redis_tool manipulate rules from redis.\n");
@@ -17,8 +20,10 @@ void maat_tool_print_usage(void)
printf("\t-h [host], redis IP, 127.0.0.1 as default.\n");
printf("\t-p [port], redis port, 6379 as default.\n");
printf("\t-d [dir], dump rules from redis to [dir], %s as default.\n",redis_dump_dir);
printf("\t-j [payload.json], add or delete rules as maat json. Must have field compile_table field, and plugin table's valid flag must be in the last column.\n");
printf("\t-t [timeout], timeout config after t seconds, default 0, not timeout.\n");
printf("example: ./maat_redis_tool -h 127.0.0.1 -p 6379 -d %s\n",redis_dump_dir);
printf(" ./maat_redis_tool -h 127.0.0.1 -p 6379 -j payload.json -t 300\n");
return;
}
static int compare_serial_rule(const void *a, const void *b)
@@ -31,7 +36,23 @@ static int compare_serial_rule(const void *a, const void *b)
snprintf(q_str,sizeof(q_str),"%s.%d",rb->table_name,rb->rule_id);
return strcmp(p_str,q_str);
}
void read_rule_from_redis(const char*redis_ip, int redis_port, int redis_db,const char* output_path ,void*logger)
static redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db)
{
struct timeval connect_timeout;
connect_timeout.tv_sec=0;
connect_timeout.tv_usec=100*1000; // 100 ms
redisContext * ctx;
ctx=redisConnectWithTimeout(redis_ip, redis_port,connect_timeout);
if(ctx==NULL)
{
printf("Unable to connect %s:%d db%d\n",redis_ip,redis_port,redis_db);
return NULL;
}
return ctx;
}
void read_rule_from_redis(redisContext * ctx,const char* output_path ,void*logger)
{
struct serial_rule_t* rule_list;
int rule_num=0,line_count=0;
@@ -43,19 +64,6 @@ void read_rule_from_redis(const char*redis_ip, int redis_port, int redis_db,cons
char table_path[256],index_path[256];
FILE *table_fp=NULL, *index_fp=NULL;
struct timeval connect_timeout;
connect_timeout.tv_sec=0;
connect_timeout.tv_usec=100*1000; // 100 ms
redisContext * ctx;
ctx=redisConnectWithTimeout(redis_ip, redis_port,connect_timeout);
if(ctx==NULL)
{
printf("Unable to connect %s:%d db%d\n",redis_ip,redis_port,redis_db);
return;
}
printf("Reading key list from %s:%d db%d.\n",redis_ip,redis_port,redis_db);
rule_num=get_rm_key_list(0, ctx, &rule_list, logger,&version, &update_type,0);
if(rule_num==0)
{
@@ -146,41 +154,137 @@ clean_up:
}
return;
}
int count_line_num(const char* table_name,const char* line,void *u_para)
{
(*((int *)u_para))++;
return 0;
}
int line_idx=0;
long long server_time=0,absolute_expire_time=0;
char compile_table_name[128];
int make_serial_rule(const char* table_name,const char* line,void *u_para)
{
struct serial_rule_t* s_rule=(struct serial_rule_t*)u_para;
int rule_id=0,is_valid=0;
char *str1=NULL, *token=NULL;
char *saveptr1=NULL,*last1=NULL;
int j=0;
char *buff=(char*)malloc(strlen(line)+1);
memcpy(buff,line, strlen(line)+1);
while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t')
{
buff[strlen(buff)-1]='\0';
}
for (j = 0,str1=buff; ; j++, str1 = NULL) {
token = strtok_r(str1, "\t ", &saveptr1);
if (token == NULL)
break;
if(j==0)
{
sscanf(token,"%d",&rule_id);
}
last1=token;
}
sscanf(last1,"%d",&is_valid);
memcpy(buff,line, strlen(line)+1);
while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t')
{
buff[strlen(buff)-1]='\0';
}
set_serial_rule(s_rule+line_idx, (enum MAAT_OPERATION)is_valid, rule_id, 0, table_name, buff, absolute_expire_time);
line_idx++;
free(str1);
return 0;
}
#define WORK_MODE_DUMP 0
#define WORK_MODE_JSON 1
int main(int argc, char * argv[])
{
int oc=0;
//char model='?';
int oc=0,ret=0, i=0,success_cnt=0;
int model=0;
char redis_ip[64];
int redis_port=6379;
int redis_db=0;
strncpy(redis_ip,"127.0.0.1",sizeof(redis_ip));
char table_info[128];
strncpy(table_info,"./table_info.conf",sizeof(table_info));
char dump_dir[128];
char dump_dir[128], json_file[128], tmp_iris_path[128];
strncpy(dump_dir,redis_dump_dir,sizeof(dump_dir));
while((oc=getopt(argc,argv,"mh:p:t:d:f:"))!=-1)
redisContext * ctx=NULL;
int total_line_cnt=0;
int timeout=0;
FILE* json_fp=NULL;
cJSON *json=NULL, *tmp_obj=NULL;
struct stat fstat_buf;
unsigned long json_file_size=0,read_size=0;
char* json_buff=NULL;
while((oc=getopt(argc,argv,"h:p:d:f:j:t:"))!=-1)
{
switch(oc)
{
case 'm':
//model=oc;
break;
case 'h':
strncpy(redis_ip,optarg,sizeof(redis_ip));
break;
case 'p':
sscanf(optarg,"%d",&redis_port);
break;
case 't':
strncpy(table_info,optarg,sizeof(table_info));
break;
case 'd':
model=WORK_MODE_DUMP;
strncpy(dump_dir,optarg,sizeof(dump_dir));
if(dump_dir[strlen(dump_dir)-1]=='/')
{
dump_dir[strlen(dump_dir)-1]='\0';
}
break;
case 'j':
strncpy(json_file, optarg,sizeof(json_file));
model=WORK_MODE_JSON;
ret=stat(json_file, &fstat_buf);
if(ret!=0)
{
printf("fstat file %s error.\n",json_file);
return -1;
}
json_fp=fopen(json_file,"r");
if(json_fp==NULL)
{
printf("fopen file %s error %s.\n",json_file,strerror(errno));
return -1;
}
json_file_size=fstat_buf.st_size;
json_buff=(char*)calloc(1,json_file_size);
read_size=fread(json_buff,1,json_file_size,json_fp);
if(read_size!=json_file_size)
{
printf("fread file %s error.",json_file);
return -1;
}
json=cJSON_Parse(json_buff);
if (!json)
{
printf("%s format is error before: %-200.200s",json_file,cJSON_GetErrorPtr());
return -1;
}
tmp_obj=cJSON_GetObjectItem(json,"compile_table");
if(tmp_obj==NULL)
{
printf("No \"compile_table\" in %s.\n",json_file);
return -1;
}
strncpy(compile_table_name,tmp_obj->valuestring,sizeof(compile_table_name));
free(json_buff);
cJSON_Delete(json);
fclose(json_fp);
break;
case 't':
sscanf(optarg,"%d",&timeout);
break;
case '?':
default:
maat_tool_print_usage();
@@ -188,5 +292,47 @@ int main(int argc, char * argv[])
break;
}
}
read_rule_from_redis(redis_ip,redis_port, redis_db,dump_dir, NULL);
ctx=connect_redis(redis_ip,redis_port, redis_db);
if(ctx==NULL)
{
return -1;
}
if(model==WORK_MODE_DUMP)
{
printf("Reading key list from %s:%d db%d.\n",redis_ip,redis_port,redis_db);
read_rule_from_redis(ctx,dump_dir, NULL);
}
else if(model==WORK_MODE_JSON)
{
ret=json2iris(json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL);
if(ret<0)
{
printf("Invalid json format.\n");
}
total_line_cnt=0;
config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num, NULL, &total_line_cnt,NULL, NULL);
printf("Serialize %s to %d lines.\nNote that group reuse is NOT supported.\n",json_file, total_line_cnt);
struct serial_rule_t * s_rule=(struct serial_rule_t *)calloc(sizeof(struct serial_rule_t),total_line_cnt);
server_time=redis_server_time(ctx);
if(timeout>0)
{
absolute_expire_time=server_time+timeout;
}
config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule,NULL, NULL);
ret=0;
while(success_cnt<total_line_cnt)
{
success_cnt+=exec_serial_rule(ctx,s_rule+success_cnt, total_line_cnt-success_cnt, server_time, NULL);
}
for(i=0;i<total_line_cnt;i++)
{
empty_serial_rules(s_rule+i);
}
free(s_rule);
redisFree(ctx);
}
return 0;
}