支持加载使用gzip命令压缩的maat JSON文件。允许先压缩、后加密，不支持先加密后压缩。

CMakeLists.txt

@@ -9,7 +9,7 @@ IF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
   SET(CMAKE_INSTALL_PREFIX /opt/MESA/  CACHE PATH "default install path" FORCE )
 ENDIF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
 set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} -Wall)
-set(MAAT_DEPEND_DYN_LIB  MESA_handle_logger MESA_htable pcre rulescan pthread m pcre MESA_field_stat2 crypto)
+set(MAAT_DEPEND_DYN_LIB  MESA_handle_logger MESA_htable pcre rulescan pthread m pcre MESA_field_stat2 crypto z)
 include_directories(${PROJECT_SOURCE_DIR}/inc/)
 include_directories(/opt/MESA/include/)
 

inc/Maat_rule.h

@@ -159,7 +159,8 @@ enum MAAT_INIT_OPT
 	MAAT_OPT_ENABLE_UPDATE,			//VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled.  DEFAULT: Backgroud update is enabled. Runtime setting is allowed.
 	MAAT_OPT_ACCEPT_TAGS,			//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]}
 	MAAT_OPT_FOREIGN_CONT_DIR,		//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files
-	MAAT_OPT_GARBAGE_COLLECTION_TIMEOUT_MS //VALUE is interger, SIZE=sizeof(int). DEFAULT:10,000 milliseconds.
+	MAAT_OPT_GARBAGE_COLLECTION_TIMEOUT_MS, //VALUE is interger, SIZE=sizeof(int). DEFAULT:10,000 milliseconds.
+	MAAT_OPT_JSON_IS_GZIPPED		//VALUE is NULL, SIZE is 0. Default: 0, Not compressed by gzip.
 };
 //return -1 if failed, return 0 on success;
 int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT  type,const void* value,int size);

src/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.5)
 
 set(MAAT_FRAME_MAJOR_VERSION 3)
 set(MAAT_FRAME_MINOR_VERSION 1)
-set(MAAT_FRAME_PATCH_VERSION 17)
+set(MAAT_FRAME_PATCH_VERSION 20)
 set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION})
 
 message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")

src/entry/Maat_api.cpp

@@ -481,21 +481,50 @@ int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char
 {
 	int ret=0;
 	struct stat fstat_buf;
-	char* json_buff=NULL;
-	size_t buff_sz=0;
+	unsigned char* json_buff=NULL, *decrypted_buff=NULL, *uncompressed_buff=NULL;
+	size_t json_buff_sz=0, decrypted_buff_sz=0, uncompressed_buff_sz=0;
 
 	MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,
 					"Maat initial with JSON file %s, formating..",
 					maat_json_fn);
+
 	if(strlen(feather->decrypt_key)&&strlen(feather->decrypt_algo))
 	{
-		ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&json_buff, &buff_sz, err_str, err_str_sz);
+		ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&decrypted_buff, &decrypted_buff_sz, err_str, err_str_sz);
+		if(ret<0)
+		{
+			MESA_handle_runtime_log(feather->logger, RLOG_LV_FATAL, maat_module,
+							"Decrypt Maat JSON file %s failed.", maat_json_fn);
+			return -1;
+		}
+		json_buff=decrypted_buff;
+		json_buff_sz=decrypted_buff_sz;
+
 	}
+	if(feather->maat_json_is_gzipped)
+	{
+		ret=gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz);
+		free(json_buff);
+		if(ret<0)
+		{
+			MESA_handle_runtime_log(feather->logger, RLOG_LV_FATAL, maat_module,
+							"Uncompress Maat JSON file %s failed.", maat_json_fn);
+			return -1;
+		}
+		json_buff=uncompressed_buff;
+		json_buff_sz=uncompressed_buff_sz;
+	}	
 	if(json_buff==NULL)//decryption failed or no decryption.
 	{	
-		ret=load_file_to_memory(maat_json_fn, (unsigned char**)&json_buff, &buff_sz);
+		ret=load_file_to_memory(maat_json_fn, &json_buff, &json_buff_sz);
+		if(ret<0)
+		{
+			MESA_handle_runtime_log(feather->logger, RLOG_LV_FATAL, maat_module,
+							"Read Maat JSON file %s failed.", maat_json_fn);
+			return -1;
+		}
 	}
-	ret=json2iris(json_buff,
+	ret=json2iris((const char*)json_buff,
 			maat_json_fn,
 			feather->compile_tn, feather->group2compile_tn, feather->group2group_tn,
 			NULL,
@@ -562,6 +591,8 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
 	feather->base_rgn_seq=0;
 	feather->AUTO_NUMBERING_ON=1;
 	feather->backgroud_update_enabled=1;
+	feather->maat_json_is_gzipped=0;
+
 	snprintf(feather->decrypt_algo, sizeof(feather->decrypt_algo), "aes-256-cbc");
 	snprintf(feather->foreign_cont_dir, sizeof(feather->foreign_cont_dir), "%s_files", table_info_path);
 	pthread_mutex_init(&(feather->background_update_mutex),NULL);
@@ -640,7 +671,22 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
 			}
 			_feather->garbage_collection_timeout_ms=intval;
 			break;
-
+		
+		case MAAT_OPT_JSON_IS_GZIPPED:
+			intval=*(const int*)value;
+			if(size!=sizeof(int)||intval<0)
+			{
+				return -1;
+			}
+			if(intval>0)
+			{
+				_feather->maat_json_is_gzipped=1;
+			}
+			else
+			{
+				_feather->maat_json_is_gzipped=0;
+			}
+			break;
 		case MAAT_OPT_FULL_CFG_DIR:			
 			assert(_feather->input_mode==SOURCE_NONE);
 			if(size>(int)sizeof(_feather->iris_ctx.full_dir))
@@ -667,7 +713,6 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
 								(const char*)value, err_str);
 				return -1;
 			}
-
 			break;
 		case MAAT_OPT_STAT_ON:
 			_feather->stat_on=1;

src/entry/Maat_utils.cpp

@@ -5,7 +5,7 @@
 #include <openssl/md5.h>
 #include <sys/stat.h>
 #include <openssl/evp.h>
-
+#include <zlib.h>
 #include "Maat_utils.h"
 pid_t gettid()
 {
@@ -221,6 +221,13 @@ int system_cmd_cp(const char* src_file,const char*dst_file)
 	snprintf(cmd, sizeof(cmd), "cp -f %s %s", src_file, dst_file);
 	return  system(cmd);
 }
+int system_cmd_gzip(const char* src_file, const char* dst_file)
+{
+	char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
+	snprintf(cmd,sizeof(cmd), "gzip -9 < %s > %s", src_file, dst_file);
+	return  system(cmd);
+}
+
 int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password)
 {
 	char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
@@ -390,6 +397,71 @@ int decrypt_open(const char* file_name, const char* key, const char* algorithm,
 	file_buff=NULL;
 	return ret;
 }
+
+int gzip_uncompress_one_try(const unsigned char *in_compressed_data, size_t in_compressed_sz, unsigned char **out_uncompressed_data,
+		size_t *out_uncompressed_sz)
+{
+	z_stream strm;
+	strm.zalloc = NULL;
+	strm.zfree = NULL;
+	strm.opaque = NULL;
+
+	strm.avail_in = in_compressed_sz;
+	strm.avail_out = *out_uncompressed_sz;
+	strm.next_in = (Bytef*) in_compressed_data;
+	strm.next_out = *out_uncompressed_data;
+
+	int ret = -1;
+	ret = inflateInit2(&strm, MAX_WBITS+16);
+	if (ret == Z_OK)
+	{
+		ret = inflate(&strm, Z_FINISH);
+		if (ret == Z_STREAM_END)
+		{
+			*out_uncompressed_sz = strm.total_out;
+			ret = inflateEnd(&strm);
+			return ret;
+		}
+	}
+	inflateEnd(&strm);
+	return ret;
+}
+int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz, unsigned char **out_uncompressed_data,
+		size_t *out_uncompressed_sz)
+{
+
+
+	int z_result;
+	int ret=-1;
+	size_t buffer_sz=in_compressed_sz*2;	
+	*out_uncompressed_data = (unsigned char*) malloc(buffer_sz);
+	do{
+		*out_uncompressed_sz=buffer_sz;
+		z_result = gzip_uncompress_one_try(
+			in_compressed_data,			
+			in_compressed_sz,
+			out_uncompressed_data,		
+			out_uncompressed_sz);
+
+		switch( z_result )
+		{
+			case Z_OK:
+				ret=0;
+				break;			
+			case Z_BUF_ERROR:
+				buffer_sz*=2;
+				*out_uncompressed_data=(unsigned char*) realloc(*out_uncompressed_data, buffer_sz);
+				break;
+			default:
+				ret=-1;
+				break;
+		}
+
+	}while(z_result==Z_BUF_ERROR);
+	return ret;
+}
+
+
 enum MAAT_IP_FORMAT ip_format_str2int(const char* format)
 {
 	if(0==strcasecmp(format, "range"))

src/inc_internal/Maat_rule_internal.h

@@ -313,6 +313,7 @@ struct _Maat_feather_t
 	pthread_mutex_t background_update_mutex;
 	char decrypt_key[MAX_TABLE_NAME_LEN];
 	char decrypt_algo[MAX_TABLE_NAME_LEN];
+	int maat_json_is_gzipped;
 	pthread_t cfg_mon_t;
 
 	int AUTO_NUMBERING_ON;

src/inc_internal/Maat_utils.h

@@ -72,12 +72,17 @@ int system_cmd_rm(const char* src_file);
 int system_cmd_mv(const char* src_file, const char*dst_file);
 int system_cmd_cp(const char* src_file, const char*dst_file);
 int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password);
+int system_cmd_gzip(const char* src_file, const char* dst_file);
 
 char* md5_file(const char* filename, char* md5string);
 int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len);
 const char** charset_get_all_name(void);
 const char* charset_get_name(enum MAAT_CHARSET charset);
 int lqueue_destroy_cb(void *data, long data_len, void *arg);
+
+//Caller is responsible to free the out_uncompressed_data buffer.
+int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz, unsigned char **out_uncompressed_data,	size_t *out_uncompressed_sz);
+
 int decrypt_open(const char* file_name, const char* key, const char* algorithm, unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz);
 int load_file_to_memory(const char* file_name, unsigned char**pp_out, size_t *out_sz);
 //do_encrypt: 1 for encryption, 0 for decryption.

test/test_maatframe.cpp

@@ -33,6 +33,8 @@ extern int my_scandir(const char *dir, struct dirent ***namelist,
 extern char* md5_file(const char* filename, char* md5string);
 extern int system_cmd_cp(const char* src_file,const char*dst_file);
 extern int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password);
+int system_cmd_gzip(const char* src_file, const char* dst_file);
+
 Maat_feather_t g_feather=NULL;
 void *g_logger=NULL;
 int g_iThreadNum=4;
@@ -117,6 +119,7 @@ const char* old_json="./json_update/old.json";
 const char* new_json="./json_update/new.json";
 const char* corrupted_json="./json_update/corrupted.json";
 const char* json_decrypt_key="himaat!";
+const char* tmp_gzipped_file_name="./json_update/tmp_gzipped_json.gz";
 
 class JSONUpdate : public testing::Test 
 {
@@ -124,10 +127,13 @@ class JSONUpdate : public testing::Test
 protected:
 	static void SetUpTestCase()
 	{	
+		int tmp=0;
+		system_cmd_gzip(old_json, tmp_gzipped_file_name);
+		system_cmd_encrypt(tmp_gzipped_file_name, watched_json, json_decrypt_key);
 		
-		system_cmd_encrypt(old_json, watched_json, json_decrypt_key);
-		
-		_shared_feather_j=Maat_feather(g_iThreadNum, table_info_path, g_logger);
+		_shared_feather_j=Maat_feather(g_iThreadNum, table_info_path, g_logger);	
+		tmp=1;
+		Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_JSON_IS_GZIPPED, &tmp, sizeof(tmp));
 		Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_DECRYPT_KEY, json_decrypt_key, strlen(json_decrypt_key)+1);
 		Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_JSON_FILE_PATH, watched_json, strlen(watched_json)+1);		
 		Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms));
@@ -152,11 +158,13 @@ TEST_F(JSONUpdate, OldCfg)
 }
 TEST_F(JSONUpdate, NewCfg)
 {
-	system_cmd_encrypt(corrupted_json, watched_json, json_decrypt_key);
+	system_cmd_gzip(corrupted_json, tmp_gzipped_file_name);
+	system_cmd_encrypt(tmp_gzipped_file_name, watched_json, json_decrypt_key);
 	sleep(2);
 	scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1);
 
-	system_cmd_encrypt(new_json, watched_json, json_decrypt_key);
+	system_cmd_gzip(new_json, tmp_gzipped_file_name);
+	system_cmd_encrypt(tmp_gzipped_file_name, watched_json, json_decrypt_key);
 	sleep(5);
 	scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 0);
 }