gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

支持maat json文件加密。

Browse Source
This commit is contained in:
zhengchao
2020-01-13 19:05:24 +08:00
parent b8df6112af
commit 94f11e5a7b
12 changed files with 257 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
src/entry/Maat_api.cpp
View File

@@ -508,6 +508,52 @@ int detain_last_data(char* buff,int buff_size,int detained_len,const char* data,
}
return ret_len;
}
int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char* err_str, size_t err_str_sz)
{
int ret=0;
struct stat fstat_buf;
char* json_buff=NULL;
MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,
"Maat initial with JSON file %s, formating..",
maat_json_fn);
if(strlen(feather->decrypt_key)&&strlen(feather->decrypt_algo))
{
ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&json_buff, err_str, err_str_sz);
}
if(json_buff==NULL)//decryption failed or no decryption.
{
ret=load_file_to_memory(maat_json_fn, &json_buff);
}
ret=json2iris(json_buff,
maat_json_fn,
feather->compile_tn, feather->group_tn,
NULL,
feather->json_ctx.iris_file,
sizeof(feather->json_ctx.iris_file),
feather->logger);
free(json_buff);
json_buff=NULL;
if(ret<0)
{
return -1;
}
strncpy(feather->json_ctx.json_file, maat_json_fn, sizeof(feather->json_ctx.json_file));
ret=stat(maat_json_fn, &fstat_buf);
feather->json_ctx.last_md5_time=fstat_buf.st_ctim;
md5_file(feather->json_ctx.json_file, feather->json_ctx.effective_json_md5);
MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO, maat_module,
"JSON file %s md5: %s, generate index file %s OK.",
feather->json_ctx.json_file,
feather->json_ctx.effective_json_md5,
feather->json_ctx.iris_file);
feather->input_mode=SOURCE_JSON_FILE;
return 0;
}
Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger)
{
if(max_thread_num<=0)
@@ -543,6 +589,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
feather->base_rgn_seq=0;
feather->AUTO_NUMBERING_ON=1;
feather->backgroud_update_enabled=1;
snprintf(feather->decrypt_algo, sizeof(feather->decrypt_algo), "aes-256-cbc");
snprintf(feather->foreign_cont_dir, sizeof(feather->foreign_cont_dir), "%s_files", table_info_path);
pthread_mutex_init(&(feather->background_update_mutex),NULL);
snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path);
@@ -554,8 +601,8 @@ failed:
int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size)
{
_Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct stat attrib;
int intval=0,ret=-1;
char err_str[MAX_TABLE_NAME_LEN];
switch(type)
{
case MAAT_OPT_ENABLE_UPDATE:
@@ -630,31 +677,15 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
break;
case MAAT_OPT_JSON_FILE_PATH:
assert(_feather->input_mode==SOURCE_NONE);
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"Maat initial with JSON file %s, formating..",
(const char*)value);
ret=json2iris((const char*)value,
_feather->compile_tn,_feather->group_tn,
NULL,
_feather->json_ctx.iris_file,
sizeof(_feather->json_ctx.iris_file),
_feather->logger);
ret=load_maat_json_file(_feather, (const char *)value, err_str, sizeof(err_str));
if(ret<0)
{
MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module,
"Load maat json file %s failed: %s.",
(const char*)value, err_str);
return -1;
}
memcpy(_feather->json_ctx.json_file, value, size);
stat(_feather->json_ctx.json_file, &attrib);
_feather->json_ctx.last_md5_time=attrib.st_ctime;
md5_file(_feather->json_ctx.json_file, _feather->json_ctx.effective_json_md5);
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"JSON file %s md5: %s, generate index file %s OK.",
_feather->json_ctx.json_file,
_feather->json_ctx.effective_json_md5,
_feather->json_ctx.iris_file);
_feather->input_mode=SOURCE_JSON_FILE;
break;
case MAAT_OPT_STAT_ON:
_feather->stat_on=1;
@@ -667,10 +698,10 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
{
return -1;
}
memcpy(_feather->stat_file,(const char*)value,size);
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"Maat performance statistic output to %s."
,(const char*)value);
memcpy(_feather->stat_file, (const char*)value, size);
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module,
"Maat performance statistic output to %s.",
(const char*)value);
_feather->stat_on=1;
break;
case MAAT_OPT_SCAN_DETAIL:
@@ -678,9 +709,9 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
_feather->rule_scan_type=intval;
break;
case MAAT_OPT_INSTANCE_NAME:
snprintf(_feather->instance_name
,sizeof(_feather->instance_name)
,"%s",
snprintf(_feather->instance_name,
sizeof(_feather->instance_name),
"%s",
(const char*)value);
break;
case MAAT_OPT_DECRYPT_KEY:

2
src/entry/Maat_command.cpp
View File

@@ -1824,7 +1824,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para
void (*finish)(void*),//u_para
void* u_para,
const unsigned char* dec_key,
const char* dec_key,
_Maat_feather_t* feather)
{
int table_id=0, i=0, rule_num=0, empty_value_num=0, valid_column=-1;

24
src/entry/Maat_rule.cpp
View File

@@ -33,7 +33,7 @@
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
int MAAT_FRAME_VERSION_2_8_20191129=1;
int MAAT_FRAME_VERSION_2_8_20200113=1;
int is_valid_table_name(const char* str)
{
@@ -3500,7 +3500,8 @@ void *thread_rule_monitor(void *arg)
int scan_dir_cnt=0;
int ret=0;
char md5_tmp[MD5_DIGEST_LENGTH*2+1]={0};
char tmp_dir[MAX_TABLE_NAME_LEN]={0};
char err_str[MAX_TABLE_NAME_LEN]={0};
struct stat attrib;
size_t total_wait_rule_cnt=0;
@@ -3557,31 +3558,24 @@ void *thread_rule_monitor(void *arg)
break;
case SOURCE_JSON_FILE:
memset(md5_tmp, 0, sizeof(md5_tmp));
memset(tmp_dir, 0, sizeof(tmp_dir));
stat(feather->json_ctx.json_file, &attrib);
if(attrib.st_ctime!=feather->json_ctx.last_md5_time)
if(memcmp(&attrib.st_ctim, &(feather->json_ctx.last_md5_time), sizeof(attrib.st_ctim)))
{
feather->json_ctx.last_md5_time=attrib.st_ctime;
feather->json_ctx.last_md5_time=attrib.st_ctim;
md5_file(feather->json_ctx.json_file, md5_tmp);
if(0!=strcmp(md5_tmp,feather->json_ctx.effective_json_md5))
{
ret=json2iris(feather->json_ctx.json_file,
feather->compile_tn, feather->group_tn,
NULL,
tmp_dir,
sizeof(tmp_dir),
feather->logger);
ret=load_maat_json_file(feather, feather->json_ctx.json_file, err_str, sizeof(err_str));
if(ret<0)
{
MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module ,
"Maat re-initiate with JSON file %s failed, md5: %s",
"Maat re-initiate with JSON file %s (md5=%s)failed: %s",
feather->json_ctx.json_file,
md5_tmp);
md5_tmp,
err_str);
}
else
{
strcpy(feather->json_ctx.effective_json_md5, md5_tmp);
strcpy(feather->json_ctx.iris_file, tmp_dir);
config_monitor_traverse(0,
feather->json_ctx.iris_file,
maat_start_cb,

127
src/entry/Maat_utils.cpp
View File

@@ -1,9 +1,11 @@
#include <string.h>
#include <stdio.h>
#include <ctype.h>
#include <stdlib.h>
#include <openssl/md5.h>
#include <assert.h>
#include <openssl/md5.h>
#include <sys/stat.h>
#include <openssl/evp.h>
#include "Maat_utils.h"
pid_t gettid()
{
@@ -257,4 +259,125 @@ int lqueue_destroy_cb(void *data, long data_len, void *arg)
assert(0);
return 0;
}
#define DECRYPT_BLOCK_SIZE (16*1024)
int decrypt_open(const char* filename, const char* key, const char* algorithm, unsigned char**pp_out, char* err_str, size_t err_str_sz)
{
unsigned char inbuf[DECRYPT_BLOCK_SIZE];
int inlen, out_blk_len=0;
int out_buff_len=0,buff_offset=0;
EVP_CIPHER_CTX *ctx;
unsigned char cipher_key[EVP_MAX_KEY_LENGTH];
unsigned char cipher_iv[EVP_MAX_IV_LENGTH];
memset(cipher_key,0,sizeof(cipher_key));
memset(cipher_iv,0,sizeof(cipher_iv));
const EVP_CIPHER *cipher;
const EVP_MD *dgst=NULL;
const unsigned char *salt=NULL;
int ret=0;
FILE*in=fopen(filename, "r");
if(in==NULL)
{
return -1;
}
OpenSSL_add_all_algorithms();
cipher=EVP_get_cipherbyname(algorithm);
if(cipher==NULL)
{
snprintf(err_str, err_str_sz, "Cipher %s is not supported.",algorithm);
return 0;
}
dgst=EVP_get_digestbyname("md5");
if(dgst==NULL)
{
snprintf(err_str, err_str_sz, "Get MD5 object failed.");
return 0;
}
ret=EVP_BytesToKey(cipher, dgst, salt, (unsigned char*)key, strlen((const char*)key), 1, cipher_key, cipher_iv);
if(ret==0)
{
snprintf(err_str, err_str_sz, "Key and IV generatioin failed.");
return 0;
}
/* Don't set key or IV right away; we want to check lengths */
ctx = EVP_CIPHER_CTX_new();
EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL,0);
OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) % 16==0);
OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16);
/* Now we can set key and IV */
EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, 0);
out_buff_len=DECRYPT_BLOCK_SIZE;
*pp_out=(unsigned char*)malloc(out_buff_len*sizeof(unsigned char));
for (;;)
{
inlen = fread(inbuf, 1, sizeof(inbuf), in);
if (inlen <= 0)
break;
if(out_buff_len-buff_offset<inlen+EVP_CIPHER_block_size(cipher)-1)
{
out_buff_len*=2;
*pp_out=(unsigned char*)realloc(*pp_out, out_buff_len);
}
out_blk_len=out_buff_len-buff_offset;
if (!EVP_CipherUpdate(ctx, *pp_out+buff_offset, &out_blk_len, inbuf, inlen))
{
snprintf(err_str, err_str_sz, "EVP_CipherUpdate failed.");
EVP_CIPHER_CTX_free(ctx);
goto error_out;
}
buff_offset+=out_blk_len;
}
if (!EVP_CipherFinal_ex(ctx, *pp_out+buff_offset, &out_blk_len))
{
snprintf(err_str, err_str_sz, "EVP_CipherFinal_ex failed. Maybe password is wrong?");
EVP_CIPHER_CTX_free(ctx);
goto error_out;
}
buff_offset+=out_blk_len;
EVP_CIPHER_CTX_free(ctx);
fclose(in);
return buff_offset;
error_out:
free(*pp_out);
*pp_out=NULL;
fclose(in);
return 0;
}
int load_file_to_memory(const char* file_name, char**pp_out)
{
int ret=0;
FILE* fp=NULL;
struct stat fstat_buf;
size_t read_size=0;
ret=stat(file_name, &fstat_buf);
if(ret!=0)
{
return -1;
}
fp=fopen(file_name, "r");
if(fp==NULL)
{
return -1;
}
*pp_out=(char*)calloc(1, fstat_buf.st_size+1);
read_size=fread(*pp_out,1, fstat_buf.st_size, fp);
if(read_size!= (size_t)fstat_buf.st_size)
{
free(*pp_out);
pp_out=NULL;
return -1;
}
fclose(fp);
fp=NULL;
return fstat_buf.st_size;
}

157
src/entry/config_monitor.cpp
View File

@@ -28,97 +28,15 @@ struct cm_table_info_t
int cfg_num;
char encryp_algorithm[MAX_CONFIG_FN_LEN];
};
int decrypt_open(FILE* in,const unsigned char* key, const char* algorithm,unsigned char**pp_out,void *logger)
{
unsigned char inbuf[MAX_CONFIG_LINE];
int inlen, out_blk_len=0;
int out_buff_len=0,buff_offset=0;
EVP_CIPHER_CTX *ctx;
unsigned char cipher_key[EVP_MAX_KEY_LENGTH];
unsigned char cipher_iv[EVP_MAX_IV_LENGTH];
memset(cipher_key,0,sizeof(cipher_key));
memset(cipher_iv,0,sizeof(cipher_iv));
const EVP_CIPHER *cipher;
const EVP_MD *dgst=NULL;
const unsigned char *salt=NULL;
int ret=0;
OpenSSL_add_all_algorithms();
cipher=EVP_get_cipherbyname(algorithm);
if(cipher==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"Cipher %s is not supported.",algorithm);
return 0;
}
dgst=EVP_get_digestbyname("md5");
if(dgst==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"Get MD5 object failed.");
return 0;
}
ret=EVP_BytesToKey(cipher,dgst,salt,key,strlen((const char*)key),1,cipher_key,cipher_iv);
if(ret==0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"Key and IV generatioin failed.");
return 0;
}
/* Don't set key or IV right away; we want to check lengths */
ctx = EVP_CIPHER_CTX_new();
EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL,0);
OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) == 16);
OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16);
/* Now we can set key and IV */
EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, 0);
out_buff_len=16*1024;
*pp_out=(unsigned char*)malloc(out_buff_len*sizeof(unsigned char));
for (;;)
{
inlen = fread(inbuf, 1, MAX_CONFIG_LINE, in);
if (inlen <= 0)
break;
if(out_buff_len-buff_offset<inlen+EVP_CIPHER_block_size(cipher)-1)
{
out_buff_len*=2;
*pp_out=(unsigned char*)realloc(*pp_out,out_buff_len);
}
out_blk_len=out_buff_len-buff_offset;
if (!EVP_CipherUpdate(ctx, *pp_out+buff_offset, &out_blk_len, inbuf, inlen))
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"EVP_CipherUpdate failed.");
EVP_CIPHER_CTX_free(ctx);
goto error_out;
}
buff_offset+=out_blk_len;
}
if (!EVP_CipherFinal_ex(ctx, *pp_out+buff_offset, &out_blk_len))
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"EVP_CipherFinal_ex failed. Maybe password is wrong?");
EVP_CIPHER_CTX_free(ctx);
goto error_out;
}
buff_offset+=out_blk_len;
EVP_CIPHER_CTX_free(ctx);
return buff_offset;
error_out:
free(*pp_out);
*pp_out=NULL;
return 0;
}
char* read_nxt_line_from_buff(const unsigned char* buff, int buff_size, int* offset, char*line ,int line_size)
char* read_nxt_line_from_buff(const char* buff, int buff_size, int* offset, char*line ,int line_size)
{
int this_offset=0;
const unsigned char* p;
const char* p;
//search for CRLF, aka '\r', '\n' or "\r\n"
p=(const unsigned char*)memchr(buff+*offset,'\r',buff_size-*offset);
p=(const char*)memchr(buff+*offset,'\r',buff_size-*offset);
if(p==NULL)
{
p=(const unsigned char*)memchr(buff+*offset,'\n',buff_size-*offset);
p=(const char*)memchr(buff+*offset,'\n',buff_size-*offset);
}
else
{
@@ -136,8 +54,9 @@ char* read_nxt_line_from_buff(const unsigned char* buff, int buff_size, int* off
p=buff+buff_size;
}
this_offset=p-(buff+*offset);
memcpy(line,buff+*offset,MIN(this_offset,line_size));
memcpy(line,buff+*offset, MIN(this_offset,line_size-1));
*offset+=this_offset;
line[MIN(this_offset,line_size-1)]='\0';
return line;
}
//replacement of glibc scandir, to adapt dictator malloc wrap
@@ -358,60 +277,56 @@ int cm_read_cfg_index_file(const char* path,struct cm_table_info_t* idx,int size
int cm_read_table_file(struct cm_table_info_t* index,
int (*update)(const char*,const char*,void*),
void* u_para,
const unsigned char* key,
const char* key,
void* logger)
{
int cfg_num=0,i=0;
int ret=0;
char error_string[MAX_CONFIG_FN_LEN];
char line[MAX_CONFIG_LINE]={0},*ret_str=NULL;
unsigned char* decrypt_buff=NULL;
int decrypt_len=0,do_decrypt=0,decrypt_offset=0;
FILE*fp=fopen(index->cfg_path,"r");
if(fp==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error,open %s failed.",index->cfg_path);
return -1;
}
char* table_file_buff=NULL;
int file_sz=0, file_offset=0;
if(strlen(index->encryp_algorithm)>0)
{
if(key==NULL||strlen((const char*)key)==0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, no key to decrypt %s.",index->cfg_path);
fclose(fp);
return -1;
}
decrypt_len=decrypt_open(fp, key,index->encryp_algorithm, &decrypt_buff,logger);
if(decrypt_len==0)
file_sz=decrypt_open(index->cfg_path, key, index->encryp_algorithm, (unsigned char**)&table_file_buff, error_string, sizeof(error_string));
if(file_sz==0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, %s decrypt failed.",index->cfg_path);
fclose(fp);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, %s decrypt failed: %s",
index->cfg_path, error_string);
return -1;
}
read_nxt_line_from_buff(decrypt_buff, decrypt_len, &decrypt_offset, line, sizeof(line));
sscanf(line,"%d\n",&cfg_num);
do_decrypt=1;
}
else
{
fscanf(fp,"%d\n",&cfg_num);
file_sz=load_file_to_memory(index->cfg_path, &table_file_buff);
if(file_sz==0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, %s decrypt failed: %s",
index->cfg_path, error_string);
return -1;
}
}
read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line));
sscanf(line, "%d\n", &cfg_num);
if(cfg_num!=index->cfg_num)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor ,"file %s config num not matched",index->cfg_path);
fclose(fp);
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor, "file %s config num not matched", index->cfg_path);
return -1;
}
for(i=0;i<cfg_num;i++)
{
line[sizeof(line)-1]='\0';
if(do_decrypt==1)
{
ret_str=read_nxt_line_from_buff(decrypt_buff, decrypt_len, &decrypt_offset, line, sizeof(line));
}
else
{
ret_str=fgets(line,sizeof(line),fp);
}
ret_str=read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line));
if(ret_str==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor ,
@@ -432,11 +347,9 @@ int cm_read_table_file(struct cm_table_info_t* index,
break;
}
}
fclose(fp);
if(decrypt_buff!=NULL)
{
free(decrypt_buff);
}
free(table_file_buff);
return 0;
}
const char* path2filename(const char*path)
@@ -456,7 +369,7 @@ void config_monitor_traverse(long long version,const char*idx_dir,
int (*update)(const char*, const char*, void*),//table name ,line ,u_para
void (*finish)(void*),//u_para
void* u_para,
const unsigned char* dec_key,
const char* dec_key,
void* logger)
{
@@ -492,7 +405,7 @@ void config_monitor_traverse(long long version,const char*idx_dir,
}
for(j=0;j<table_num;j++)
{
cm_read_table_file(table_array+j,update,u_para,dec_key,logger);
cm_read_table_file(table_array+j, update, u_para, dec_key, logger);
}
if(finish!=NULL)
{

48
src/entry/json2iris.cpp
View File

@@ -1100,43 +1100,12 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
return 0;
}
// redis_write_ctx is used by maat_redis_tool to write json to redis.
int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger)
int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, void* logger)
{
FILE* json_fp=NULL;
cJSON *json=NULL, *tmp_obj=NULL;
struct stat fstat_buf;
int ret=-1;
char* json_buff=NULL;
unsigned long json_file_size=0,read_size=0;
struct iris_description_t iris_cfg;
memset(&iris_cfg,0,sizeof(iris_cfg));
ret=stat(json_file, &fstat_buf);
if(ret!=0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"fstat file %s error.",json_file);
goto error_out;
}
json_fp=fopen(json_file,"r");
if(json_fp==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"fopen file %s error %s.",json_file,strerror(errno));
goto error_out;
}
json_file_size=fstat_buf.st_size;
json_buff=(char*)calloc(1,json_file_size+1);
read_size=fread(json_buff,1,json_file_size,json_fp);
if(read_size!=json_file_size)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
"fread file %s error.",json_file);
goto error_out;
}
json=cJSON_Parse(json_buff);
if (!json)
@@ -1155,7 +1124,7 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,r
{
group_tn=tmp_obj->valuestring;
}
ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,redis_write_ctx,&iris_cfg,logger);
ret=set_iris_descriptor(json_filename, json, compile_tn, group_tn, redis_write_ctx, &iris_cfg, logger);
if(ret<0)
{
goto error_out;
@@ -1163,31 +1132,24 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,r
ret=create_tmp_dir(&iris_cfg);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json,
"create tmp folder %s error",iris_cfg.tmp_iris_dir);
goto error_out;
}
ret=write_iris(json,&iris_cfg,logger);
ret=write_iris(json ,&iris_cfg, logger);
if(ret<0)
{
goto error_out;
}
memcpy(iris_dir_buf,iris_cfg.tmp_iris_index_dir,MIN(strlen(iris_cfg.tmp_iris_index_dir)+1,(unsigned int)buf_len));
memcpy(iris_dir_buf,iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir)+1, (unsigned int)buf_len));
cJSON_Delete(json);
fclose(json_fp);
free(json_buff);
clear_iris_descriptor(&iris_cfg);
return 0;
error_out:
cJSON_Delete(json);
if(json_fp!=NULL)
{
fclose(json_fp);
}
free(json_buff);
clear_iris_descriptor(&iris_cfg);
return -1;
}

10
src/inc_internal/Maat_rule_internal.h
View File

@@ -21,7 +21,6 @@
#include <iconv.h>
#include <openssl/md5.h>
extern const char *maat_module;
#define mr_region_id_var "SEQUENCE_REGION"
@@ -287,7 +286,7 @@ struct source_json_ctx
char json_file[MAX_TABLE_NAME_LEN];
char iris_file[MAX_TABLE_NAME_LEN];
char effective_json_md5[MD5_DIGEST_LENGTH*2+1];
time_t last_md5_time;
struct timespec last_md5_time;
};
struct source_redis_ctx
{
@@ -333,7 +332,8 @@ struct _Maat_feather_t
char compile_tn[MAX_TABLE_NAME_LEN];
char group_tn[MAX_TABLE_NAME_LEN];
pthread_mutex_t background_update_mutex;
unsigned char decrypt_key[MAX_TABLE_NAME_LEN];
char decrypt_key[MAX_TABLE_NAME_LEN];
char decrypt_algo[MAX_TABLE_NAME_LEN];
pthread_t cfg_mon_t;
int AUTO_NUMBERING_ON;
@@ -450,14 +450,14 @@ void empty_serial_rules(struct serial_rule_t* rule);
int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,unsigned int serial_rule_num, long long server_time, void* logger);
long long redis_server_time(redisContext* ctx);
redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db, void* logger);
char* md5_file(const char* filename, char* md5string);
int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char* err_str, size_t err_str_sz);
void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx,
void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para
int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para
void (*finish)(void*),//u_para
void* u_para,
const unsigned char* dec_key,
const char* dec_key,
_Maat_feather_t* feather);

3
src/inc_internal/Maat_utils.h
View File

@@ -2,6 +2,7 @@
#include "Maat_rule.h"
#include <arpa/inet.h>
#include <unistd.h>
#include <stdio.h>
#include <sys/syscall.h>
#include <sys/types.h>//fstat
#include <sys/types.h>//fstat
@@ -73,5 +74,7 @@ int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len
const char** charset_get_all_name(void);
const char* charset_get_name(enum MAAT_CHARSET charset);
int lqueue_destroy_cb(void *data, long data_len, void *arg);
int decrypt_open(const char* filename, const char* key, const char* algorithm, unsigned char**pp_out, char* err_str, size_t err_str_sz);
int load_file_to_memory(const char* file_name, char**pp_out);

2
src/inc_internal/config_monitor.h
View File

@@ -9,7 +9,7 @@ void config_monitor_traverse(long long version,const char*idx_dir,
int (*update)(const char*, const char*, void*),//table name ,line ,u_para
void (*finish)(void*),//u_para
void* u_para,
const unsigned char* dec_key,
const char* dec_key,
void* logger);
#endif

4
src/inc_internal/json2iris.h
View File

@@ -1,6 +1,6 @@
#ifndef H_MAAT_JSON2IRIS_H_INCLUDE
#define H_MAAT_JSON2IRIS_H_INCLUDE
int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger);
int set_file_rulenum(const char* path,int rulenum,void* logger);
int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, void* logger);
int set_file_rulenum(const char* path, int rulenum, void* logger);
#endif

8
test/test_maatframe.cpp
View File

@@ -54,7 +54,7 @@ void wait_for_cmd_effective(Maat_feather_t feather, long long version_before)
// printf("wait for %lld ms\n", wating_us/1000);
}
void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old)
void scan_with_old_or_new_cfg(Maat_feather_t feather, int is_old)
{
const char* hit_old_data="Hello world! I'm eve.";
const char* hit_new_data="Maat was borned in MESA.";
@@ -72,7 +72,7 @@ void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old)
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, hit_old_data, strlen(hit_old_data),
&result,NULL, 1,
&mid, 0);
if(hit_old)
if(is_old)
{
EXPECT_EQ(ret, 1);
EXPECT_TRUE(result.config_id==1);
@@ -87,7 +87,7 @@ void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old)
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, hit_new_data, strlen(hit_new_data),
&result,NULL, 1,
&mid, 0);
if(!hit_old)
if(!is_old)
{
EXPECT_EQ(ret, 1);
EXPECT_TRUE(result.config_id==2);
@@ -141,7 +141,7 @@ TEST_F(JSONUpdate, NewCfg)
sleep(2);
scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1);
system_cmd_cp(new_json, watched_json);
sleep(2);
sleep(5);
scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 0);
}

8
tools/maat_redis_tool.cpp
View File

@@ -1,4 +1,5 @@
#include "Maat_rule.h"
#include "Maat_utils.h"
#include "Maat_command.h"
#include "Maat_rule_internal.h"
#include "cJSON.h"
@@ -326,7 +327,12 @@ int main(int argc, char * argv[])
}
else if(model==WORK_MODE_JSON)
{
ret=json2iris(json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL);
ret=load_file_to_memory(json_file, &json_buff);
if(ret<0)
{
printf("open %s failed.\n", json_file);
}
ret=json2iris(json_buff, json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL);
if(ret<0)
{
printf("Invalid json format.\n");