diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index ba3999b..75b6082 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -508,6 +508,52 @@ int detain_last_data(char* buff,int buff_size,int detained_len,const char* data, } return ret_len; } +int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char* err_str, size_t err_str_sz) +{ + int ret=0; + struct stat fstat_buf; + char* json_buff=NULL; + + + MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module , + "Maat initial with JSON file %s, formating..", + maat_json_fn); + if(strlen(feather->decrypt_key)&&strlen(feather->decrypt_algo)) + { + ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&json_buff, err_str, err_str_sz); + } + if(json_buff==NULL)//decryption failed or no decryption. + { + ret=load_file_to_memory(maat_json_fn, &json_buff); + } + ret=json2iris(json_buff, + maat_json_fn, + feather->compile_tn, feather->group_tn, + NULL, + feather->json_ctx.iris_file, + sizeof(feather->json_ctx.iris_file), + feather->logger); + free(json_buff); + json_buff=NULL; + if(ret<0) + { + return -1; + } + strncpy(feather->json_ctx.json_file, maat_json_fn, sizeof(feather->json_ctx.json_file)); + + ret=stat(maat_json_fn, &fstat_buf); + feather->json_ctx.last_md5_time=fstat_buf.st_ctim; + + md5_file(feather->json_ctx.json_file, feather->json_ctx.effective_json_md5); + MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO, maat_module, + "JSON file %s md5: %s, generate index file %s OK.", + feather->json_ctx.json_file, + feather->json_ctx.effective_json_md5, + feather->json_ctx.iris_file); + feather->input_mode=SOURCE_JSON_FILE; + + return 0; +} Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger) { if(max_thread_num<=0) @@ -543,6 +589,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* feather->base_rgn_seq=0; feather->AUTO_NUMBERING_ON=1; feather->backgroud_update_enabled=1; + snprintf(feather->decrypt_algo, sizeof(feather->decrypt_algo), "aes-256-cbc"); snprintf(feather->foreign_cont_dir, sizeof(feather->foreign_cont_dir), "%s_files", table_info_path); pthread_mutex_init(&(feather->background_update_mutex),NULL); snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path); @@ -554,8 +601,8 @@ failed: int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size) { _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct stat attrib; int intval=0,ret=-1; + char err_str[MAX_TABLE_NAME_LEN]; switch(type) { case MAAT_OPT_ENABLE_UPDATE: @@ -630,31 +677,15 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo break; case MAAT_OPT_JSON_FILE_PATH: assert(_feather->input_mode==SOURCE_NONE); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Maat initial with JSON file %s, formating..", - (const char*)value); - - ret=json2iris((const char*)value, - _feather->compile_tn,_feather->group_tn, - NULL, - _feather->json_ctx.iris_file, - sizeof(_feather->json_ctx.iris_file), - _feather->logger); + ret=load_maat_json_file(_feather, (const char *)value, err_str, sizeof(err_str)); if(ret<0) { + MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, + "Load maat json file %s failed: %s.", + (const char*)value, err_str); return -1; } - memcpy(_feather->json_ctx.json_file, value, size); - stat(_feather->json_ctx.json_file, &attrib); - _feather->json_ctx.last_md5_time=attrib.st_ctime; - md5_file(_feather->json_ctx.json_file, _feather->json_ctx.effective_json_md5); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "JSON file %s md5: %s, generate index file %s OK.", - _feather->json_ctx.json_file, - _feather->json_ctx.effective_json_md5, - _feather->json_ctx.iris_file); - _feather->input_mode=SOURCE_JSON_FILE; break; case MAAT_OPT_STAT_ON: _feather->stat_on=1; @@ -667,10 +698,10 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo { return -1; } - memcpy(_feather->stat_file,(const char*)value,size); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Maat performance statistic output to %s." - ,(const char*)value); + memcpy(_feather->stat_file, (const char*)value, size); + MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module, + "Maat performance statistic output to %s.", + (const char*)value); _feather->stat_on=1; break; case MAAT_OPT_SCAN_DETAIL: @@ -678,9 +709,9 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo _feather->rule_scan_type=intval; break; case MAAT_OPT_INSTANCE_NAME: - snprintf(_feather->instance_name - ,sizeof(_feather->instance_name) - ,"%s", + snprintf(_feather->instance_name, + sizeof(_feather->instance_name), + "%s", (const char*)value); break; case MAAT_OPT_DECRYPT_KEY: diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 6d061fd..a51b4de 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -1824,7 +1824,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para void (*finish)(void*),//u_para void* u_para, - const unsigned char* dec_key, + const char* dec_key, _Maat_feather_t* feather) { int table_id=0, i=0, rule_num=0, empty_value_num=0, valid_column=-1; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 90373d3..3d93ad8 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -33,7 +33,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_8_20191129=1; +int MAAT_FRAME_VERSION_2_8_20200113=1; int is_valid_table_name(const char* str) { @@ -3500,7 +3500,8 @@ void *thread_rule_monitor(void *arg) int scan_dir_cnt=0; int ret=0; char md5_tmp[MD5_DIGEST_LENGTH*2+1]={0}; - char tmp_dir[MAX_TABLE_NAME_LEN]={0}; + char err_str[MAX_TABLE_NAME_LEN]={0}; + struct stat attrib; size_t total_wait_rule_cnt=0; @@ -3557,31 +3558,24 @@ void *thread_rule_monitor(void *arg) break; case SOURCE_JSON_FILE: memset(md5_tmp, 0, sizeof(md5_tmp)); - memset(tmp_dir, 0, sizeof(tmp_dir)); stat(feather->json_ctx.json_file, &attrib); - if(attrib.st_ctime!=feather->json_ctx.last_md5_time) + if(memcmp(&attrib.st_ctim, &(feather->json_ctx.last_md5_time), sizeof(attrib.st_ctim))) { - feather->json_ctx.last_md5_time=attrib.st_ctime; + feather->json_ctx.last_md5_time=attrib.st_ctim; md5_file(feather->json_ctx.json_file, md5_tmp); if(0!=strcmp(md5_tmp,feather->json_ctx.effective_json_md5)) { - ret=json2iris(feather->json_ctx.json_file, - feather->compile_tn, feather->group_tn, - NULL, - tmp_dir, - sizeof(tmp_dir), - feather->logger); + ret=load_maat_json_file(feather, feather->json_ctx.json_file, err_str, sizeof(err_str)); if(ret<0) { MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module , - "Maat re-initiate with JSON file %s failed, md5: %s", + "Maat re-initiate with JSON file %s (md5=%s)failed: %s", feather->json_ctx.json_file, - md5_tmp); + md5_tmp, + err_str); } else { - strcpy(feather->json_ctx.effective_json_md5, md5_tmp); - strcpy(feather->json_ctx.iris_file, tmp_dir); config_monitor_traverse(0, feather->json_ctx.iris_file, maat_start_cb, diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 3d40912..f4c16dd 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -1,9 +1,11 @@ #include -#include #include #include -#include #include +#include +#include +#include + #include "Maat_utils.h" pid_t gettid() { @@ -257,4 +259,125 @@ int lqueue_destroy_cb(void *data, long data_len, void *arg) assert(0); return 0; } +#define DECRYPT_BLOCK_SIZE (16*1024) +int decrypt_open(const char* filename, const char* key, const char* algorithm, unsigned char**pp_out, char* err_str, size_t err_str_sz) +{ + unsigned char inbuf[DECRYPT_BLOCK_SIZE]; + int inlen, out_blk_len=0; + int out_buff_len=0,buff_offset=0; + EVP_CIPHER_CTX *ctx; + + unsigned char cipher_key[EVP_MAX_KEY_LENGTH]; + unsigned char cipher_iv[EVP_MAX_IV_LENGTH]; + memset(cipher_key,0,sizeof(cipher_key)); + memset(cipher_iv,0,sizeof(cipher_iv)); + + const EVP_CIPHER *cipher; + const EVP_MD *dgst=NULL; + const unsigned char *salt=NULL; + int ret=0; + + FILE*in=fopen(filename, "r"); + if(in==NULL) + { + return -1; + } + + OpenSSL_add_all_algorithms(); + cipher=EVP_get_cipherbyname(algorithm); + if(cipher==NULL) + { + snprintf(err_str, err_str_sz, "Cipher %s is not supported.",algorithm); + return 0; + } + dgst=EVP_get_digestbyname("md5"); + if(dgst==NULL) + { + snprintf(err_str, err_str_sz, "Get MD5 object failed."); + return 0; + } + ret=EVP_BytesToKey(cipher, dgst, salt, (unsigned char*)key, strlen((const char*)key), 1, cipher_key, cipher_iv); + if(ret==0) + { + snprintf(err_str, err_str_sz, "Key and IV generatioin failed."); + return 0; + } + /* Don't set key or IV right away; we want to check lengths */ + ctx = EVP_CIPHER_CTX_new(); + EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL,0); + OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) % 16==0); + OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16); + + /* Now we can set key and IV */ + EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, 0); + out_buff_len=DECRYPT_BLOCK_SIZE; + *pp_out=(unsigned char*)malloc(out_buff_len*sizeof(unsigned char)); + for (;;) + { + inlen = fread(inbuf, 1, sizeof(inbuf), in); + if (inlen <= 0) + break; + + if(out_buff_len-buff_offsetcfg_path,"r"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error,open %s failed.",index->cfg_path); - return -1; - } + char* table_file_buff=NULL; + int file_sz=0, file_offset=0; + if(strlen(index->encryp_algorithm)>0) { if(key==NULL||strlen((const char*)key)==0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, no key to decrypt %s.",index->cfg_path); - fclose(fp); return -1; } - decrypt_len=decrypt_open(fp, key,index->encryp_algorithm, &decrypt_buff,logger); - if(decrypt_len==0) + file_sz=decrypt_open(index->cfg_path, key, index->encryp_algorithm, (unsigned char**)&table_file_buff, error_string, sizeof(error_string)); + if(file_sz==0) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, %s decrypt failed.",index->cfg_path); - fclose(fp); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, %s decrypt failed: %s", + index->cfg_path, error_string); return -1; } - read_nxt_line_from_buff(decrypt_buff, decrypt_len, &decrypt_offset, line, sizeof(line)); - sscanf(line,"%d\n",&cfg_num); - do_decrypt=1; } else { - fscanf(fp,"%d\n",&cfg_num); + file_sz=load_file_to_memory(index->cfg_path, &table_file_buff); + if(file_sz==0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, %s decrypt failed: %s", + index->cfg_path, error_string); + return -1; + } + } + read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line)); + sscanf(line, "%d\n", &cfg_num); + if(cfg_num!=index->cfg_num) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor ,"file %s config num not matched",index->cfg_path); - fclose(fp); + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor, "file %s config num not matched", index->cfg_path); return -1; } for(i=0;ivaluestring; } - ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,redis_write_ctx,&iris_cfg,logger); + ret=set_iris_descriptor(json_filename, json, compile_tn, group_tn, redis_write_ctx, &iris_cfg, logger); if(ret<0) { goto error_out; @@ -1163,31 +1132,24 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,r ret=create_tmp_dir(&iris_cfg); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, "create tmp folder %s error",iris_cfg.tmp_iris_dir); goto error_out; } - ret=write_iris(json,&iris_cfg,logger); + ret=write_iris(json ,&iris_cfg, logger); if(ret<0) { goto error_out; } - memcpy(iris_dir_buf,iris_cfg.tmp_iris_index_dir,MIN(strlen(iris_cfg.tmp_iris_index_dir)+1,(unsigned int)buf_len)); + memcpy(iris_dir_buf,iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir)+1, (unsigned int)buf_len)); cJSON_Delete(json); - fclose(json_fp); - free(json_buff); clear_iris_descriptor(&iris_cfg); return 0; error_out: cJSON_Delete(json); - if(json_fp!=NULL) - { - fclose(json_fp); - } - free(json_buff); clear_iris_descriptor(&iris_cfg); return -1; } diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 8ccdb81..3aed79e 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -21,7 +21,6 @@ #include #include -extern const char *maat_module; #define mr_region_id_var "SEQUENCE_REGION" @@ -287,7 +286,7 @@ struct source_json_ctx char json_file[MAX_TABLE_NAME_LEN]; char iris_file[MAX_TABLE_NAME_LEN]; char effective_json_md5[MD5_DIGEST_LENGTH*2+1]; - time_t last_md5_time; + struct timespec last_md5_time; }; struct source_redis_ctx { @@ -333,7 +332,8 @@ struct _Maat_feather_t char compile_tn[MAX_TABLE_NAME_LEN]; char group_tn[MAX_TABLE_NAME_LEN]; pthread_mutex_t background_update_mutex; - unsigned char decrypt_key[MAX_TABLE_NAME_LEN]; + char decrypt_key[MAX_TABLE_NAME_LEN]; + char decrypt_algo[MAX_TABLE_NAME_LEN]; pthread_t cfg_mon_t; int AUTO_NUMBERING_ON; @@ -450,14 +450,14 @@ void empty_serial_rules(struct serial_rule_t* rule); int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,unsigned int serial_rule_num, long long server_time, void* logger); long long redis_server_time(redisContext* ctx); redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db, void* logger); -char* md5_file(const char* filename, char* md5string); +int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char* err_str, size_t err_str_sz); void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx, void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para void (*finish)(void*),//u_para void* u_para, - const unsigned char* dec_key, + const char* dec_key, _Maat_feather_t* feather); diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index ab233a2..6f7352f 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -2,6 +2,7 @@ #include "Maat_rule.h" #include #include +#include #include #include //fstat #include //fstat @@ -73,5 +74,7 @@ int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len const char** charset_get_all_name(void); const char* charset_get_name(enum MAAT_CHARSET charset); int lqueue_destroy_cb(void *data, long data_len, void *arg); +int decrypt_open(const char* filename, const char* key, const char* algorithm, unsigned char**pp_out, char* err_str, size_t err_str_sz); +int load_file_to_memory(const char* file_name, char**pp_out); diff --git a/src/inc_internal/config_monitor.h b/src/inc_internal/config_monitor.h index faabf50..6167ce4 100644 --- a/src/inc_internal/config_monitor.h +++ b/src/inc_internal/config_monitor.h @@ -9,7 +9,7 @@ void config_monitor_traverse(long long version,const char*idx_dir, int (*update)(const char*, const char*, void*),//table name ,line ,u_para void (*finish)(void*),//u_para void* u_para, - const unsigned char* dec_key, + const char* dec_key, void* logger); #endif diff --git a/src/inc_internal/json2iris.h b/src/inc_internal/json2iris.h index f61f6e5..ab8e5c6 100644 --- a/src/inc_internal/json2iris.h +++ b/src/inc_internal/json2iris.h @@ -1,6 +1,6 @@ #ifndef H_MAAT_JSON2IRIS_H_INCLUDE #define H_MAAT_JSON2IRIS_H_INCLUDE -int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger); -int set_file_rulenum(const char* path,int rulenum,void* logger); +int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, void* logger); +int set_file_rulenum(const char* path, int rulenum, void* logger); #endif diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 469a524..39306b7 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -54,7 +54,7 @@ void wait_for_cmd_effective(Maat_feather_t feather, long long version_before) // printf("wait for %lld ms\n", wating_us/1000); } -void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old) +void scan_with_old_or_new_cfg(Maat_feather_t feather, int is_old) { const char* hit_old_data="Hello world! I'm eve."; const char* hit_new_data="Maat was borned in MESA."; @@ -72,7 +72,7 @@ void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old) ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, hit_old_data, strlen(hit_old_data), &result,NULL, 1, &mid, 0); - if(hit_old) + if(is_old) { EXPECT_EQ(ret, 1); EXPECT_TRUE(result.config_id==1); @@ -87,7 +87,7 @@ void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old) ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, hit_new_data, strlen(hit_new_data), &result,NULL, 1, &mid, 0); - if(!hit_old) + if(!is_old) { EXPECT_EQ(ret, 1); EXPECT_TRUE(result.config_id==2); @@ -141,7 +141,7 @@ TEST_F(JSONUpdate, NewCfg) sleep(2); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1); system_cmd_cp(new_json, watched_json); - sleep(2); + sleep(5); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 0); } diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp index 58ba87e..584ab3e 100644 --- a/tools/maat_redis_tool.cpp +++ b/tools/maat_redis_tool.cpp @@ -1,4 +1,5 @@ #include "Maat_rule.h" +#include "Maat_utils.h" #include "Maat_command.h" #include "Maat_rule_internal.h" #include "cJSON.h" @@ -325,8 +326,13 @@ int main(int argc, char * argv[]) read_rule_from_redis(ctx,desired_version,dump_dir, NULL); } else if(model==WORK_MODE_JSON) - { - ret=json2iris(json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL); + { + ret=load_file_to_memory(json_file, &json_buff); + if(ret<0) + { + printf("open %s failed.\n", json_file); + } + ret=json2iris(json_buff, json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL); if(ret<0) { printf("Invalid json format.\n");