diff --git a/CMakeLists.txt b/CMakeLists.txt index b670b90..664dbe1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ IF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) SET(CMAKE_INSTALL_PREFIX /opt/MESA/ CACHE PATH "default install path" FORCE ) ENDIF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} -Wall) -set(MAAT_DEPEND_DYN_LIB MESA_handle_logger MESA_htable pcre rulescan pthread m pcre MESA_field_stat2 crypto) +set(MAAT_DEPEND_DYN_LIB MESA_handle_logger MESA_htable pcre rulescan pthread m pcre MESA_field_stat2 crypto z) include_directories(${PROJECT_SOURCE_DIR}/inc/) include_directories(/opt/MESA/include/) diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 1bfb93d..dbeefbd 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -159,7 +159,8 @@ enum MAAT_INIT_OPT MAAT_OPT_ENABLE_UPDATE, //VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled. DEFAULT: Backgroud update is enabled. Runtime setting is allowed. MAAT_OPT_ACCEPT_TAGS, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]} MAAT_OPT_FOREIGN_CONT_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files - MAAT_OPT_GARBAGE_COLLECTION_TIMEOUT_MS //VALUE is interger, SIZE=sizeof(int). DEFAULT:10,000 milliseconds. + MAAT_OPT_GARBAGE_COLLECTION_TIMEOUT_MS, //VALUE is interger, SIZE=sizeof(int). DEFAULT:10,000 milliseconds. + MAAT_OPT_JSON_IS_GZIPPED //VALUE is NULL, SIZE is 0. Default: 0, Not compressed by gzip. }; //return -1 if failed, return 0 on success; int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 223bf10..5fd0dd2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.5) set(MAAT_FRAME_MAJOR_VERSION 3) set(MAAT_FRAME_MINOR_VERSION 1) -set(MAAT_FRAME_PATCH_VERSION 17) +set(MAAT_FRAME_PATCH_VERSION 20) set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION}) message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 60095de..72840b2 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -481,21 +481,50 @@ int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char { int ret=0; struct stat fstat_buf; - char* json_buff=NULL; - size_t buff_sz=0; + unsigned char* json_buff=NULL, *decrypted_buff=NULL, *uncompressed_buff=NULL; + size_t json_buff_sz=0, decrypted_buff_sz=0, uncompressed_buff_sz=0; MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module , "Maat initial with JSON file %s, formating..", maat_json_fn); + if(strlen(feather->decrypt_key)&&strlen(feather->decrypt_algo)) { - ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&json_buff, &buff_sz, err_str, err_str_sz); + ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&decrypted_buff, &decrypted_buff_sz, err_str, err_str_sz); + if(ret<0) + { + MESA_handle_runtime_log(feather->logger, RLOG_LV_FATAL, maat_module, + "Decrypt Maat JSON file %s failed.", maat_json_fn); + return -1; + } + json_buff=decrypted_buff; + json_buff_sz=decrypted_buff_sz; + } + if(feather->maat_json_is_gzipped) + { + ret=gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz); + free(json_buff); + if(ret<0) + { + MESA_handle_runtime_log(feather->logger, RLOG_LV_FATAL, maat_module, + "Uncompress Maat JSON file %s failed.", maat_json_fn); + return -1; + } + json_buff=uncompressed_buff; + json_buff_sz=uncompressed_buff_sz; + } if(json_buff==NULL)//decryption failed or no decryption. { - ret=load_file_to_memory(maat_json_fn, (unsigned char**)&json_buff, &buff_sz); + ret=load_file_to_memory(maat_json_fn, &json_buff, &json_buff_sz); + if(ret<0) + { + MESA_handle_runtime_log(feather->logger, RLOG_LV_FATAL, maat_module, + "Read Maat JSON file %s failed.", maat_json_fn); + return -1; + } } - ret=json2iris(json_buff, + ret=json2iris((const char*)json_buff, maat_json_fn, feather->compile_tn, feather->group2compile_tn, feather->group2group_tn, NULL, @@ -562,6 +591,8 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* feather->base_rgn_seq=0; feather->AUTO_NUMBERING_ON=1; feather->backgroud_update_enabled=1; + feather->maat_json_is_gzipped=0; + snprintf(feather->decrypt_algo, sizeof(feather->decrypt_algo), "aes-256-cbc"); snprintf(feather->foreign_cont_dir, sizeof(feather->foreign_cont_dir), "%s_files", table_info_path); pthread_mutex_init(&(feather->background_update_mutex),NULL); @@ -640,7 +671,22 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo } _feather->garbage_collection_timeout_ms=intval; break; - + + case MAAT_OPT_JSON_IS_GZIPPED: + intval=*(const int*)value; + if(size!=sizeof(int)||intval<0) + { + return -1; + } + if(intval>0) + { + _feather->maat_json_is_gzipped=1; + } + else + { + _feather->maat_json_is_gzipped=0; + } + break; case MAAT_OPT_FULL_CFG_DIR: assert(_feather->input_mode==SOURCE_NONE); if(size>(int)sizeof(_feather->iris_ctx.full_dir)) @@ -667,7 +713,6 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo (const char*)value, err_str); return -1; } - break; case MAAT_OPT_STAT_ON: _feather->stat_on=1; diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 745952a..7195ce4 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -5,7 +5,7 @@ #include #include #include - +#include #include "Maat_utils.h" pid_t gettid() { @@ -221,6 +221,13 @@ int system_cmd_cp(const char* src_file,const char*dst_file) snprintf(cmd, sizeof(cmd), "cp -f %s %s", src_file, dst_file); return system(cmd); } +int system_cmd_gzip(const char* src_file, const char* dst_file) +{ + char cmd[MAX_SYSTEM_CMD_LEN] = { 0 }; + snprintf(cmd,sizeof(cmd), "gzip -9 < %s > %s", src_file, dst_file); + return system(cmd); +} + int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password) { char cmd[MAX_SYSTEM_CMD_LEN] = { 0 }; @@ -390,6 +397,71 @@ int decrypt_open(const char* file_name, const char* key, const char* algorithm, file_buff=NULL; return ret; } + +int gzip_uncompress_one_try(const unsigned char *in_compressed_data, size_t in_compressed_sz, unsigned char **out_uncompressed_data, + size_t *out_uncompressed_sz) +{ + z_stream strm; + strm.zalloc = NULL; + strm.zfree = NULL; + strm.opaque = NULL; + + strm.avail_in = in_compressed_sz; + strm.avail_out = *out_uncompressed_sz; + strm.next_in = (Bytef*) in_compressed_data; + strm.next_out = *out_uncompressed_data; + + int ret = -1; + ret = inflateInit2(&strm, MAX_WBITS+16); + if (ret == Z_OK) + { + ret = inflate(&strm, Z_FINISH); + if (ret == Z_STREAM_END) + { + *out_uncompressed_sz = strm.total_out; + ret = inflateEnd(&strm); + return ret; + } + } + inflateEnd(&strm); + return ret; +} +int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz, unsigned char **out_uncompressed_data, + size_t *out_uncompressed_sz) +{ + + + int z_result; + int ret=-1; + size_t buffer_sz=in_compressed_sz*2; + *out_uncompressed_data = (unsigned char*) malloc(buffer_sz); + do{ + *out_uncompressed_sz=buffer_sz; + z_result = gzip_uncompress_one_try( + in_compressed_data, + in_compressed_sz, + out_uncompressed_data, + out_uncompressed_sz); + + switch( z_result ) + { + case Z_OK: + ret=0; + break; + case Z_BUF_ERROR: + buffer_sz*=2; + *out_uncompressed_data=(unsigned char*) realloc(*out_uncompressed_data, buffer_sz); + break; + default: + ret=-1; + break; + } + + }while(z_result==Z_BUF_ERROR); + return ret; +} + + enum MAAT_IP_FORMAT ip_format_str2int(const char* format) { if(0==strcasecmp(format, "range")) diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 0573579..58ce39e 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -313,6 +313,7 @@ struct _Maat_feather_t pthread_mutex_t background_update_mutex; char decrypt_key[MAX_TABLE_NAME_LEN]; char decrypt_algo[MAX_TABLE_NAME_LEN]; + int maat_json_is_gzipped; pthread_t cfg_mon_t; int AUTO_NUMBERING_ON; diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index 4d8b63c..3cd5030 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -72,12 +72,17 @@ int system_cmd_rm(const char* src_file); int system_cmd_mv(const char* src_file, const char*dst_file); int system_cmd_cp(const char* src_file, const char*dst_file); int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password); +int system_cmd_gzip(const char* src_file, const char* dst_file); char* md5_file(const char* filename, char* md5string); int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len); const char** charset_get_all_name(void); const char* charset_get_name(enum MAAT_CHARSET charset); int lqueue_destroy_cb(void *data, long data_len, void *arg); + +//Caller is responsible to free the out_uncompressed_data buffer. +int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz, unsigned char **out_uncompressed_data, size_t *out_uncompressed_sz); + int decrypt_open(const char* file_name, const char* key, const char* algorithm, unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz); int load_file_to_memory(const char* file_name, unsigned char**pp_out, size_t *out_sz); //do_encrypt: 1 for encryption, 0 for decryption. diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 0412e7a..a78f95b 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -33,6 +33,8 @@ extern int my_scandir(const char *dir, struct dirent ***namelist, extern char* md5_file(const char* filename, char* md5string); extern int system_cmd_cp(const char* src_file,const char*dst_file); extern int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password); +int system_cmd_gzip(const char* src_file, const char* dst_file); + Maat_feather_t g_feather=NULL; void *g_logger=NULL; int g_iThreadNum=4; @@ -117,6 +119,7 @@ const char* old_json="./json_update/old.json"; const char* new_json="./json_update/new.json"; const char* corrupted_json="./json_update/corrupted.json"; const char* json_decrypt_key="himaat!"; +const char* tmp_gzipped_file_name="./json_update/tmp_gzipped_json.gz"; class JSONUpdate : public testing::Test { @@ -124,10 +127,13 @@ class JSONUpdate : public testing::Test protected: static void SetUpTestCase() { + int tmp=0; + system_cmd_gzip(old_json, tmp_gzipped_file_name); + system_cmd_encrypt(tmp_gzipped_file_name, watched_json, json_decrypt_key); - system_cmd_encrypt(old_json, watched_json, json_decrypt_key); - - _shared_feather_j=Maat_feather(g_iThreadNum, table_info_path, g_logger); + _shared_feather_j=Maat_feather(g_iThreadNum, table_info_path, g_logger); + tmp=1; + Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_JSON_IS_GZIPPED, &tmp, sizeof(tmp)); Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_DECRYPT_KEY, json_decrypt_key, strlen(json_decrypt_key)+1); Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_JSON_FILE_PATH, watched_json, strlen(watched_json)+1); Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms)); @@ -152,11 +158,13 @@ TEST_F(JSONUpdate, OldCfg) } TEST_F(JSONUpdate, NewCfg) { - system_cmd_encrypt(corrupted_json, watched_json, json_decrypt_key); + system_cmd_gzip(corrupted_json, tmp_gzipped_file_name); + system_cmd_encrypt(tmp_gzipped_file_name, watched_json, json_decrypt_key); sleep(2); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1); - system_cmd_encrypt(new_json, watched_json, json_decrypt_key); + system_cmd_gzip(new_json, tmp_gzipped_file_name); + system_cmd_encrypt(tmp_gzipped_file_name, watched_json, json_decrypt_key); sleep(5); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 0); }