gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support parse encrypted json config

Browse Source
This commit is contained in:
liuwentan
2023-05-04 17:10:19 +08:00
parent f087a4382d
commit 33015d5aac
43 changed files with 543 additions and 332 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
include/maat.h
View File

@@ -99,7 +99,15 @@ int maat_options_set_logger(struct maat_options *opts, const char *log_path, enu
int maat_options_set_iris(struct maat_options *opts, const char *full_directory,
const char *increment_directory);
int maat_options_set_json_file(struct maat_options *opts, const char *json_filename);
/* Indicate whether the JSON file is compressed by gzip */
int maat_options_set_json_file_gzip_flag(struct maat_options *opts, int gzip_flag);
/* Specify the decryption key for the JSON file to be decrypted */
int maat_options_set_json_file_decrypt_key(struct maat_options *opts, const char *decrypt_key);
int maat_options_set_redis(struct maat_options *opts, const char *redis_ip,
uint16_t redis_port, int redis_db);

4
src/inc_internal/json2iris.h
View File

@@ -2,9 +2,9 @@
**********************************************************************************************
* File: json2iris.h
* Description: rule for transform json2iris
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Authors: Zheng chao <zhengchao@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

2
src/inc_internal/maat_bool_plugin.h
View File

@@ -4,7 +4,7 @@
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

51
src/inc_internal/maat_common.h
View File

@@ -1,51 +0,0 @@
/*
**********************************************************************************************
* File: maat_common.h
* Description: maat common entry
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_COMMON_H_
#define _MAAT_COMMON_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <stddef.h>
#include <limits.h>
#include "log/log.h"
#include "maat_rule.h"
struct maat_options {
char instance_name[NAME_MAX];
char foreign_cont_dir[NAME_MAX];
char log_path[PATH_MAX];
char stat_file[PATH_MAX];
size_t nr_worker_threads;
char *accept_tags;
int rule_effect_interval_ms;
int rule_update_checking_interval_ms;
int gc_timeout_ms;
int stat_on;
int perf_on;
int deferred_load_on;
int log_level;
enum data_source input_mode;
union {
struct source_iris_ctx iris_ctx;
struct source_json_ctx json_ctx;
struct source_redis_ctx redis_ctx;
};
};
#ifdef __cplusplus
}
#endif
#endif

4
src/inc_internal/maat_compile.h
View File

@@ -2,9 +2,9 @@
**********************************************************************************************
* File: maat_compile.h
* Description:
* Authors: Zheng Chao <zhengchao@geedgenetworks.com>
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

2
src/inc_internal/maat_config_monitor.h
View File

@@ -22,7 +22,7 @@ extern "C"
void config_monitor_traverse(long long version, const char *idx_dir,
void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *),
void (*finish_fn)(void *), void *u_param,
void (*finish_fn)(void *), void *u_param, const char *dec_key,
struct log_handle *logger);
int load_maat_json_file(struct maat *maat_instance, const char *json_filename,

2
src/inc_internal/maat_ex_data.h
View File

@@ -4,7 +4,7 @@
* Description: ex data
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

2
src/inc_internal/maat_expr.h
View File

@@ -4,7 +4,7 @@
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

2
src/inc_internal/maat_flag.h
View File

@@ -4,7 +4,7 @@
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

2
src/inc_internal/maat_fqdn_plugin.h
View File

@@ -4,7 +4,7 @@
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

2
src/inc_internal/maat_garbage_collection.h
View File

@@ -4,7 +4,7 @@
* Description: maat gc
* Authors: Zhengchao <zhengchao@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

2
src/inc_internal/maat_group.h
View File

@@ -4,7 +4,7 @@
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

2
src/inc_internal/maat_interval.h
View File

@@ -4,7 +4,7 @@
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

4
src/inc_internal/maat_kv.h
View File

@@ -1,10 +1,10 @@
/*
**********************************************************************************************
* File: maat_kv_map.h
* File: maat_kv.h
* Description: str2int map api
* Authors: Zheng chao <zhengchao@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

2
src/inc_internal/maat_redis_monitor.h
View File

@@ -4,7 +4,7 @@
* Description: maat redis monitor api
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-11-29
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

73
src/inc_internal/maat_rule.h
View File

@@ -171,6 +171,36 @@ struct rule_tag {
char *tag_val;
};
struct maat_options {
char instance_name[NAME_MAX];
char foreign_cont_dir[NAME_MAX];
char decrypt_algo[MAX_KEYWORDS_STR];
char decrypt_key[MAX_KEYWORDS_STR];
char log_path[PATH_MAX];
int log_level;
char stat_file[PATH_MAX];
size_t nr_worker_thread;
char *accept_tags;
int stat_on;
int perf_on;
int deferred_load_on;
int maat_json_is_gzipped;
int cumulative_update_off; //Default: cumulative update on
int gc_timeout_ms;
int rule_effect_interval_ms;
int rule_update_checking_interval_ms;
enum data_source input_mode;
union {
struct source_iris_ctx iris_ctx;
struct source_json_ctx json_ctx;
struct source_redis_ctx redis_ctx;
};
};
struct maat_stat {
char stat_file[PATH_MAX];
struct fieldstat_instance *fs_handle;
@@ -197,54 +227,27 @@ struct maat_stat {
};
struct maat {
char instance_name[NAME_MAX];
struct maat_runtime *maat_rt;
struct maat_runtime *creating_maat_rt;
struct table_manager *tbl_mgr;
enum data_source input_mode;
union {
struct source_iris_ctx iris_ctx;
struct source_json_ctx json_ctx;
struct source_redis_ctx mr_ctx;
};
struct log_handle *logger;
int deferred_load;
int is_running;
pthread_mutex_t background_update_mutex;
size_t nr_worker_thread;
struct maat_options opts;
long long maat_version;
long long last_full_version;
long long load_specific_version; //Default: Load the Latest. Only valid in redis mode, and maybe failed for too old
/* internal state */
long long new_version;
int is_running;
pthread_t cfg_mon_thread;
pthread_mutex_t background_update_mutex;
int rule_effect_interval_ms;
int rule_update_checking_interval_ms;
int gc_timeout_ms; //garbage collection timeout_ms;
int cumulative_update_off; //Default: cumulative update on
int stat_on;
int perf_on;
struct log_handle *logger;
struct maat_garbage_bin *garbage_bin;
int default_compile_table_id;
int g2g_table_id; //group2group table id
char decrypt_key[NAME_MAX];
char decrypt_algo[NAME_MAX];
int maat_json_is_gzipped;
long long load_specific_version; //Default: Load the Latest. Only valid in redis mode, and maybe failed for too old
char foreign_cont_dir[NAME_MAX];
/* internal state */
long long new_version;
/* statistics */
struct maat_stat *stat;
};

6
src/inc_internal/maat_utils.h
View File

@@ -4,7 +4,7 @@
* Description: maat utils entry
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
@@ -116,6 +116,10 @@ int system_cmd_mkdir(const char* path);
int system_cmd_rmdir(const char *dir);
int system_cmd_gzip(const char *src_file, const char *dst_file);
int system_cmd_encrypt(const char *src_file, const char *dst_file, const char *password);
#ifdef __cplusplus
}
#endif

2
src/inc_internal/maat_virtual.h
View File

@@ -4,7 +4,7 @@
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

4
src/inc_internal/rcu_hash.h
View File

@@ -1,10 +1,10 @@
/*
**********************************************************************************************
* File: rcu_hash.h
* Description: rcu hashtable
* Description: rcu hash table
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

4
src/json2iris.c
View File

@@ -1,8 +1,8 @@
/*
**********************************************************************************************
* File: json2iris.h
* File: json2iris.c
* Description: rule for transform json2iris
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Authors: Zheng Chao <zhengchao@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************

192
src/maat_api.c
View File

@@ -18,7 +18,6 @@
#include "json2iris.h"
#include "maat.h"
#include "maat_rule.h"
#include "maat_common.h"
#include "maat_kv.h"
#include "maat_command.h"
#include "maat_ex_data.h"
@@ -58,7 +57,7 @@ struct maat_options* maat_options_new(void)
{
struct maat_options *options = ALLOC(struct maat_options, 1);
options->nr_worker_threads = 1;
options->nr_worker_thread = 1;
options->deferred_load_on = 0;
options->rule_effect_interval_ms = 60 * 1000;
options->rule_update_checking_interval_ms = 1 * 1000;
@@ -75,10 +74,6 @@ void maat_options_free(struct maat_options *opts)
return;
}
if (opts->accept_tags != NULL) {
FREE(opts->accept_tags);
}
FREE(opts);
}
@@ -88,7 +83,7 @@ int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_th
return -1;
}
opts->nr_worker_threads = n_thread;
opts->nr_worker_thread = n_thread;
return 0;
}
@@ -206,6 +201,30 @@ int maat_options_set_json_file(struct maat_options *opts, const char *json_filen
return 0;
}
int maat_options_set_json_file_gzip_flag(struct maat_options *opts, int gzip_flag)
{
if (NULL == opts || (gzip_flag != 0 && gzip_flag != 1)) {
return -1;
}
opts->maat_json_is_gzipped = gzip_flag;
return 0;
}
int maat_options_set_json_file_decrypt_key(struct maat_options *opts, const char *decrypt_key)
{
if (NULL == opts || NULL == decrypt_key) {
return -1;
}
size_t str_len = MIN(sizeof(opts->decrypt_key), strlen(decrypt_key));
memcpy(opts->decrypt_key, decrypt_key, str_len);
return 0;
}
int maat_options_set_redis(struct maat_options *opts, const char *redis_ip,
uint16_t redis_port, int redis_db)
{
@@ -243,20 +262,20 @@ void maat_read_full_config(struct maat *maat_instance)
{
int ret = -1;
char err_str[NAME_MAX] = {0};
struct source_redis_ctx *mr_ctx = NULL;
struct source_redis_ctx *redis_ctx = NULL;
switch (maat_instance->input_mode) {
switch (maat_instance->opts.input_mode) {
case DATA_SOURCE_REDIS:
mr_ctx = &(maat_instance->mr_ctx);
redis_ctx = &(maat_instance->opts.redis_ctx);
log_info(maat_instance->logger, MODULE_MAAT_API,
"Maat initiate from Redis %s:%hu db%d",
mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip,
mr_ctx->redis_port,
mr_ctx->redis_db,
redis_ctx->redis_ip, redis_ctx->redis_port, redis_ctx->redis_db);
redis_ctx->read_ctx = maat_cmd_connect_redis(redis_ctx->redis_ip,
redis_ctx->redis_port,
redis_ctx->redis_db,
maat_instance->logger);
if (mr_ctx->read_ctx != NULL) {
redis_monitor_traverse(maat_instance->maat_version, mr_ctx,
if (redis_ctx->read_ctx != NULL) {
redis_monitor_traverse(maat_instance->maat_version, redis_ctx,
maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance);
}
@@ -264,38 +283,40 @@ void maat_read_full_config(struct maat *maat_instance)
if (NULL == maat_instance->creating_maat_rt) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"[%s:%d] At initiation: NO effective rule in redis %s:%hu db%d",
__FUNCTION__, __LINE__, mr_ctx->redis_ip, mr_ctx->redis_port,
mr_ctx->redis_db);
__FUNCTION__, __LINE__, redis_ctx->redis_ip, redis_ctx->redis_port,
redis_ctx->redis_db);
}
break;
case DATA_SOURCE_IRIS_FILE:
config_monitor_traverse(maat_instance->maat_version,
maat_instance->iris_ctx.full_idx_dir,
maat_instance->opts.iris_ctx.full_idx_dir,
maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance, maat_instance->logger);
maat_instance, maat_instance->opts.decrypt_key,
maat_instance->logger);
if (NULL == maat_instance->creating_maat_rt) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"[%s:%d] At initiation: NO effective rule in %s",
__FUNCTION__, __LINE__, maat_instance->iris_ctx.full_idx_dir);
__FUNCTION__, __LINE__, maat_instance->opts.iris_ctx.full_idx_dir);
}
break;
case DATA_SOURCE_JSON_FILE:
ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file,
ret = load_maat_json_file(maat_instance, maat_instance->opts.json_ctx.json_file,
err_str, sizeof(err_str));
if (ret < 0) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"[%s:%d] Maat re-initiate with JSON file %s failed: %s",
__FUNCTION__, __LINE__, maat_instance->json_ctx.json_file, err_str);
__FUNCTION__, __LINE__, maat_instance->opts.json_ctx.json_file, err_str);
}
config_monitor_traverse(maat_instance->maat_version,
maat_instance->json_ctx.iris_file,
maat_instance->opts.json_ctx.iris_file,
maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance, maat_instance->logger);
maat_instance, maat_instance->opts.decrypt_key,
maat_instance->logger);
if (NULL == maat_instance->creating_maat_rt) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"[%s:%d] At initiation: NO effective rule in %s",
__FUNCTION__, __LINE__, maat_instance->json_ctx.iris_file);
__FUNCTION__, __LINE__, maat_instance->opts.json_ctx.iris_file);
}
break;
default:
@@ -332,6 +353,11 @@ void _maat_free(struct maat *maat_instance)
maat_instance->stat = NULL;
}
if (maat_instance->opts.accept_tags != NULL) {
FREE(maat_instance->opts.accept_tags);
maat_instance->opts.accept_tags = NULL;
}
pthread_mutex_destroy(&(maat_instance->background_update_mutex));
FREE(maat_instance);
@@ -343,83 +369,59 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
return NULL;
}
int garbage_gc_timeout_s = 0;
struct maat *maat_instance = ALLOC(struct maat, 1);
if (strlen(opts->log_path) != 0) {
maat_instance->logger = log_handle_create(opts->log_path, opts->log_level);
maat_instance->opts = *opts;
if (strlen(maat_instance->opts.log_path) != 0) {
maat_instance->logger = log_handle_create(maat_instance->opts.log_path,
maat_instance->opts.log_level);
} else {
char log_path[1024] = {0};
if (strlen(maat_instance->instance_name) > 0) {
snprintf(log_path, sizeof(log_path), "%s.log",
maat_instance->instance_name);
if (strlen(maat_instance->opts.instance_name) > 0) {
snprintf(log_path, sizeof(log_path), "%s.log", maat_instance->opts.instance_name);
} else {
snprintf(log_path, sizeof(log_path), "maat.log");
}
maat_instance->logger = log_handle_create(log_path, opts->log_level);
maat_instance->logger = log_handle_create(log_path, maat_instance->opts.log_level);
}
if (0 == strlen(opts->foreign_cont_dir)) {
snprintf(maat_instance->foreign_cont_dir, sizeof(maat_instance->foreign_cont_dir),
if (0 == strlen(maat_instance->opts.foreign_cont_dir)) {
snprintf(maat_instance->opts.foreign_cont_dir, sizeof(maat_instance->opts.foreign_cont_dir),
"%s_files", table_info_path);
} else {
memcpy(maat_instance->foreign_cont_dir, opts->foreign_cont_dir, strlen(opts->foreign_cont_dir));
size_t len = strlen(maat_instance->foreign_cont_dir);
if (maat_instance->foreign_cont_dir[len - 1] == '/') {
maat_instance->foreign_cont_dir[len - 1] = '\0';
}
}
system_cmd_mkdir(maat_instance->foreign_cont_dir);
if (0 == strlen(opts->stat_file)) {
snprintf(opts->stat_file, sizeof(opts->stat_file), "maat.fs3");
}
maat_instance->input_mode = opts->input_mode;
system_cmd_mkdir(maat_instance->opts.foreign_cont_dir);
switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS:
memcpy(maat_instance->mr_ctx.redis_ip, opts->redis_ctx.redis_ip,
strlen(opts->redis_ctx.redis_ip));
maat_instance->mr_ctx.redis_port = opts->redis_ctx.redis_port;
maat_instance->mr_ctx.redis_db = opts->redis_ctx.redis_db;
break;
case DATA_SOURCE_IRIS_FILE:
memcpy(maat_instance->iris_ctx.full_idx_dir, opts->iris_ctx.full_idx_dir,
strlen(opts->iris_ctx.full_idx_dir));
memcpy(maat_instance->iris_ctx.inc_idx_dir, opts->iris_ctx.inc_idx_dir,
strlen(opts->iris_ctx.inc_idx_dir));
break;
case DATA_SOURCE_JSON_FILE:
memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file,
strlen(opts->json_ctx.json_file));
break;
default:
log_error(maat_instance->logger, MODULE_MAAT_API,
"[%s:%d] data source(%d) unsupported",
__FUNCTION__, __LINE__, maat_instance->input_mode);
goto failed;
if (0 == strlen(maat_instance->opts.stat_file)) {
snprintf(maat_instance->opts.stat_file, sizeof(maat_instance->opts.stat_file), "maat.fs3");
}
snprintf(maat_instance->opts.decrypt_algo, sizeof(maat_instance->opts.decrypt_algo), "aes-256-cbc");
maat_instance->is_running = 0;
maat_instance->maat_version = 0;
maat_instance->last_full_version = 0;
maat_instance->nr_worker_thread = opts->nr_worker_threads;
maat_instance->rule_effect_interval_ms = opts->rule_effect_interval_ms;
maat_instance->rule_update_checking_interval_ms = opts->rule_update_checking_interval_ms;
maat_instance->gc_timeout_ms = opts->gc_timeout_ms;
maat_instance->stat_on = opts->stat_on;
maat_instance->perf_on = opts->perf_on;
maat_instance->deferred_load = opts->deferred_load_on;
memcpy(maat_instance->foreign_cont_dir, opts->foreign_cont_dir, strlen(opts->foreign_cont_dir));
garbage_gc_timeout_s = (maat_instance->rule_effect_interval_ms / 1000) +
(maat_instance->gc_timeout_ms / 1000);
int garbage_gc_timeout_s = (maat_instance->opts.rule_effect_interval_ms / 1000) +
(maat_instance->opts.gc_timeout_ms / 1000);
if (maat_instance->opts.input_mode != DATA_SOURCE_IRIS_FILE &&
maat_instance->opts.input_mode != DATA_SOURCE_JSON_FILE &&
maat_instance->opts.input_mode != DATA_SOURCE_REDIS) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"[%s:%d] data source(%d) unsupported",
__FUNCTION__, __LINE__, maat_instance->opts.input_mode);
goto failed;
}
maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s);
maat_instance->stat = maat_stat_new(opts->stat_file, opts->nr_worker_threads, maat_instance->logger);
maat_instance->stat = maat_stat_new(maat_instance->opts.stat_file, maat_instance->opts.nr_worker_thread,
maat_instance->logger);
pthread_mutex_init(&(maat_instance->background_update_mutex), NULL);
maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags,
maat_instance->tbl_mgr = table_manager_create(table_info_path, maat_instance->opts.accept_tags,
maat_instance->garbage_bin, maat_instance->logger);
if (NULL == maat_instance->tbl_mgr) {
goto failed;
@@ -428,11 +430,11 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr);
maat_instance->g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr);
if (0 == maat_instance->deferred_load) {
if (0 == maat_instance->opts.deferred_load_on) {
maat_read_full_config(maat_instance);
}
if (1 == maat_instance->stat_on) {
if (1 == maat_instance->opts.stat_on) {
int ret = maat_stat_init(maat_instance->stat, maat_instance->tbl_mgr, maat_instance->garbage_bin);
if (ret < 0) {
log_error(maat_instance->logger, MODULE_MAAT_API,
@@ -1124,7 +1126,7 @@ int maat_scan_flag(struct maat *maat_instance, int table_id,
}
struct timespec start, end;
if (1 == maat_instance->perf_on) {
if (1 == maat_instance->opts.perf_on) {
clock_gettime(CLOCK_MONOTONIC, &start);
}
@@ -1179,7 +1181,7 @@ int maat_scan_flag(struct maat *maat_instance, int table_id,
void *flag_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_id);
assert(flag_rt != NULL);
if (1 == maat_instance->perf_on) {
if (1 == maat_instance->opts.perf_on) {
clock_gettime(CLOCK_MONOTONIC, &end);
flag_runtime_perf_stat(flag_rt, &start, &end, state->thread_id);
} else {
@@ -1208,7 +1210,7 @@ int maat_scan_integer(struct maat *maat_instance, int table_id,
}
struct timespec start, end;
if (1 == maat_instance->perf_on) {
if (1 == maat_instance->opts.perf_on) {
clock_gettime(CLOCK_MONOTONIC, &start);
}
@@ -1263,7 +1265,7 @@ int maat_scan_integer(struct maat *maat_instance, int table_id,
void *interval_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_id);
assert(interval_rt != NULL);
if (1 == maat_instance->perf_on) {
if (1 == maat_instance->opts.perf_on) {
clock_gettime(CLOCK_MONOTONIC, &end);
interval_runtime_perf_stat(interval_rt, &start, &end, state->thread_id);
} else {
@@ -1293,7 +1295,7 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id,
}
struct timespec start, end;
if (1 == maat_instance->perf_on) {
if (1 == maat_instance->opts.perf_on) {
clock_gettime(CLOCK_MONOTONIC, &start);
}
@@ -1348,7 +1350,7 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id,
void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_id);
assert(ip_rt != NULL);
if (1 == maat_instance->perf_on) {
if (1 == maat_instance->opts.perf_on) {
clock_gettime(CLOCK_MONOTONIC, &end);
ip_runtime_perf_stat(ip_rt, &start, &end, state->thread_id);
} else {
@@ -1378,7 +1380,7 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id,
}
struct timespec start, end;
if (1 == maat_instance->perf_on) {
if (1 == maat_instance->opts.perf_on) {
clock_gettime(CLOCK_MONOTONIC, &start);
}
@@ -1433,7 +1435,7 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id,
void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_id);
assert(ip_rt != NULL);
if (1 == maat_instance->perf_on) {
if (1 == maat_instance->opts.perf_on) {
clock_gettime(CLOCK_MONOTONIC, &end);
ip_runtime_perf_stat(ip_rt, &start, &end, state->thread_id);
} else {
@@ -1462,7 +1464,7 @@ int maat_scan_string(struct maat *maat_instance, int table_id, const char *data,
}
struct timespec start, end;
if (1 == maat_instance->perf_on) {
if (1 == maat_instance->opts.perf_on) {
clock_gettime(CLOCK_MONOTONIC, &start);
}
@@ -1517,7 +1519,7 @@ int maat_scan_string(struct maat *maat_instance, int table_id, const char *data,
void *expr_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_id);
assert(expr_rt != NULL);
if (1 == maat_instance->perf_on) {
if (1 == maat_instance->opts.perf_on) {
clock_gettime(CLOCK_MONOTONIC, &end);
expr_runtime_perf_stat(expr_rt, data_len, &start, &end, state->thread_id);
} else {
@@ -1596,7 +1598,7 @@ int maat_stream_scan(struct maat_stream *maat_stream, const char *data, int data
}
struct timespec start, end;
if (1 == maat_stream->ref_maat_instance->perf_on) {
if (1 == maat_stream->ref_maat_instance->opts.perf_on) {
clock_gettime(CLOCK_MONOTONIC, &start);
}
@@ -1636,7 +1638,7 @@ int maat_stream_scan(struct maat_stream *maat_stream, const char *data, int data
}
}
if (1 == maat_instance->perf_on) {
if (1 == maat_instance->opts.perf_on) {
clock_gettime(CLOCK_MONOTONIC, &end);
expr_runtime_perf_stat(expr_rt, data_len, &start, &end, state->thread_id);
} else {

30
src/maat_command.c
View File

@@ -1,10 +1,10 @@
/*
**********************************************************************************************
* File: maat_command.cpp
* File: maat_command.c
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
@@ -170,13 +170,14 @@ int redis_flushDB(redisContext *ctx, int db_index, struct log_handle *logger)
return redis_transaction_success;
}
static int connect_redis_for_write(struct source_redis_ctx *mr_ctx,
static int connect_redis_for_write(struct source_redis_ctx *redis_ctx,
struct log_handle *logger)
{
assert(mr_ctx->write_ctx == NULL);
mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port,
mr_ctx->redis_db, logger);
if (NULL == mr_ctx->write_ctx) {
assert(redis_ctx->write_ctx == NULL);
redis_ctx->write_ctx = maat_cmd_connect_redis(redis_ctx->redis_ip,
redis_ctx->redis_port,
redis_ctx->redis_db, logger);
if (NULL == redis_ctx->write_ctx) {
return -1;
} else {
return 0;
@@ -185,15 +186,14 @@ static int connect_redis_for_write(struct source_redis_ctx *mr_ctx,
redisContext *get_redis_ctx_for_write(struct maat *maat_instance)
{
if (NULL == maat_instance->mr_ctx.write_ctx) {
int ret = connect_redis_for_write(&(maat_instance->mr_ctx),
if (NULL == maat_instance->opts.redis_ctx.write_ctx) {
int ret = connect_redis_for_write(&(maat_instance->opts.redis_ctx),
maat_instance->logger);
if(ret!=0)
{
if (ret != 0) {
return NULL;
}
}
return maat_instance->mr_ctx.write_ctx;
return maat_instance->opts.redis_ctx.write_ctx;
}
int maat_cmd_flushDB(struct maat *maat_instance)
@@ -206,8 +206,8 @@ int maat_cmd_flushDB(struct maat *maat_instance)
}
do {
ret = redis_flushDB(maat_instance->mr_ctx.write_ctx,
maat_instance->mr_ctx.redis_db,
ret = redis_flushDB(maat_instance->opts.redis_ctx.write_ctx,
maat_instance->opts.redis_ctx.redis_db,
maat_instance->logger);
} while(0 == ret);
@@ -389,7 +389,7 @@ error_out:
int maat_cmd_set_file(struct maat *maat_instance, const char *key, const char *value,
size_t size, enum maat_operation op)
{
redisContext *ctx = maat_instance->mr_ctx.write_ctx;
redisContext *ctx = maat_instance->opts.redis_ctx.write_ctx;
if (NULL == ctx) {
log_error(maat_instance->logger, MODULE_MAAT_COMMAND,
"[%s:%d] failed: Redis is not connected.",

6
src/maat_compile.c
View File

@@ -1,10 +1,10 @@
/*
**********************************************************************************************
* File: maat_hierarchy.cpp
* File: maat_compile.c
* Description:
* Authors: Zheng Chao <zhengchao@geedgenetworks.com>
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

95
src/maat_config_monitor.c
View File

@@ -30,10 +30,11 @@ struct cm_table_info_t
char table_name[NAME_MAX];
char cfg_path[NAME_MAX];
int cfg_num;
char encryp_algorithm[NAME_MAX];
char encrypt_algo[NAME_MAX];
};
int cm_read_cfg_index_file(const char* path, struct cm_table_info_t* idx, int size)
int cm_read_cfg_index_file(const char* path, struct cm_table_info_t* idx, int size,
struct log_handle *logger)
{
int ret = 0;
int i = 0;
@@ -44,10 +45,9 @@ int cm_read_cfg_index_file(const char* path, struct cm_table_info_t* idx, int si
while (!feof(fp)) {
memset(line, 0, sizeof(line));
fgets(line, sizeof(line), fp);
ret=sscanf(line,"%s\t%d\t%s\t%s",idx[i].table_name
,&(idx[i].cfg_num)
,idx[i].cfg_path
,idx[i].encryp_algorithm);
ret = sscanf(line, "%s\t%d\t%s\t%s", idx[i].table_name, &(idx[i].cfg_num),
idx[i].cfg_path ,idx[i].encrypt_algo);
//jump over empty line
if (!(ret == 3 || ret == 4) || idx[i].cfg_num == 0) {
continue;
@@ -55,14 +55,15 @@ int cm_read_cfg_index_file(const char* path, struct cm_table_info_t* idx, int si
ret = stat(idx[i].cfg_path, &file_info);
if (ret != 0) {
//log_error
log_error(logger, MODULE_CONFIG_MONITOR, "%s of %s not exisit",
idx[i].cfg_path, path);
fclose(fp);
return -1;
}
i++;
if (i == size) {
//log_error
log_error(logger, MODULE_CONFIG_MONITOR, "Too much lines in %s", path);
break;
}
}
@@ -111,37 +112,57 @@ char *read_nxt_line_from_buff(const char *buff, size_t buff_size,
int cm_read_table_file(struct cm_table_info_t *index,
int (*update_fn)(const char *, const char *, void *),
void *u_param, struct log_handle *logger)
void *u_param, const char *dec_key, struct log_handle *logger)
{
int cfg_num = 0,i =0;
int ret = 0;
char line[MAX_CONFIG_LINE]={0};
char *ret_str=NULL;
char *table_file_buff=NULL;
size_t file_sz = 0;
size_t file_offset = 0;
char *file_buff = NULL;
char error_string[NAME_MAX];
ret = load_file_to_memory(index->cfg_path, (unsigned char **)&table_file_buff, &file_sz);
if (strlen(index->encrypt_algo) > 0) {
//JSON file has been encrypted
if (NULL == dec_key || 0 == strlen(dec_key)) {
log_error(logger, MODULE_CONFIG_MONITOR,
"update error, no key to decrypt %s.", index->cfg_path);
return -1;
}
ret = decrypt_open(index->cfg_path, dec_key, index->encrypt_algo,
(unsigned char**)&file_buff, &file_sz, error_string,
sizeof(error_string));
if (ret < 0) {
log_error(logger, MODULE_CONFIG_MONITOR,
"update error, decrypt %s failed: %s", index->cfg_path, error_string);
return -1;
}
} else {
// not encrypted
ret = load_file_to_memory(index->cfg_path, (unsigned char **)&file_buff, &file_sz);
if (ret < 0) {
log_error(logger, MODULE_CONFIG_MONITOR, "[%s:%d] open %s failed.",
__FUNCTION__, __LINE__, index->cfg_path);
return -1;
}
}
read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line));
size_t file_offset = 0;
char line[MAX_CONFIG_LINE] = {0};
read_nxt_line_from_buff(file_buff, file_sz, &file_offset, line, sizeof(line));
int cfg_num = 0;
sscanf(line, "%d\n", &cfg_num);
if(cfg_num != index->cfg_num) {
FREE(table_file_buff);
FREE(file_buff);
log_error(logger, MODULE_CONFIG_MONITOR, "[%s:%d] file %s config num not matched",
__FUNCTION__, __LINE__, index->cfg_path);
return -1;
}
for (i = 0; i < cfg_num; i++) {
for (int i = 0; i < cfg_num; i++) {
line[sizeof(line) - 1] = '\0';
ret_str = read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line));
char *ret_str = read_nxt_line_from_buff(file_buff, file_sz, &file_offset, line, sizeof(line));
if (ret_str == NULL) {
log_error(logger, MODULE_CONFIG_MONITOR, "[%s:%d] file %s line_num %d less than claimed %d",
__FUNCTION__, __LINE__, index->cfg_path, i, cfg_num);
@@ -160,7 +181,7 @@ int cm_read_table_file(struct cm_table_info_t *index,
}
}
FREE(table_file_buff);
FREE(file_buff);
return 0;
}
@@ -317,7 +338,7 @@ int get_new_idx_path(long long current_version, const char *file_dir,
void config_monitor_traverse(long long current_version, const char *idx_dir,
void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *),
void (*finish_fn)(void *), void *u_param,
void (*finish_fn)(void *), void *u_param, const char *dec_key,
struct log_handle *logger)
{
size_t i = 0;
@@ -332,7 +353,7 @@ void config_monitor_traverse(long long current_version, const char *idx_dir,
if (update_type != MAAT_UPDATE_TYPE_INVALID) {
for (i = 0; i < idx_path_num; i++) {
log_info(logger, MODULE_CONFIG_MONITOR, "load %s", idx_path_array[i]);
int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, MAX_TABLE_NUM);
int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, MAX_TABLE_NUM, logger);
if (table_num < 0) {
log_error(logger, MODULE_CONFIG_MONITOR,
"[%s:%d] load %s failed, abandon update",
@@ -348,7 +369,7 @@ void config_monitor_traverse(long long current_version, const char *idx_dir,
}
for (int j = 0; j < table_num; j++) {
cm_read_table_file(table_array + j, update_fn, u_param, logger);
cm_read_table_file(table_array + j, update_fn, u_param, dec_key, logger);
}
if (finish_fn != NULL) {
@@ -380,9 +401,9 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
"Maat initial with JSON file %s, formating...",
json_filename);
if (strlen(maat_instance->decrypt_key) && strlen(maat_instance->decrypt_algo)) {
ret = decrypt_open(json_filename, maat_instance->decrypt_key,
maat_instance->decrypt_algo,
if (strlen(maat_instance->opts.decrypt_key) && strlen(maat_instance->opts.decrypt_algo)) {
ret = decrypt_open(json_filename, maat_instance->opts.decrypt_key,
maat_instance->opts.decrypt_algo,
(unsigned char **)&decrypted_buff,
&decrypted_buff_sz,
err_str, err_str_sz);
@@ -397,7 +418,7 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
json_buff_sz=decrypted_buff_sz;
}
if (maat_instance->maat_json_is_gzipped) {
if (maat_instance->opts.maat_json_is_gzipped) {
ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff,
&uncompressed_buff_sz);
FREE(json_buff);
@@ -424,10 +445,10 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
}
ret = json2iris((const char*)json_buff, json_filename, NULL,
maat_instance->json_ctx.iris_file,
sizeof(maat_instance->json_ctx.iris_file),
strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL,
strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL,
maat_instance->opts.json_ctx.iris_file,
sizeof(maat_instance->opts.json_ctx.iris_file),
strlen(maat_instance->opts.decrypt_key) ? maat_instance->opts.decrypt_key : NULL,
strlen(maat_instance->opts.decrypt_algo) ? maat_instance->opts.decrypt_algo : NULL,
maat_instance->logger);
FREE(json_buff);
if (ret < 0) {
@@ -435,16 +456,16 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
}
ret = stat(json_filename, &fstat_buf);
maat_instance->json_ctx.last_md5_time = fstat_buf.st_ctim;
maat_instance->opts.json_ctx.last_md5_time = fstat_buf.st_ctim;
md5_file(maat_instance->json_ctx.json_file, maat_instance->json_ctx.effective_json_md5);
md5_file(maat_instance->opts.json_ctx.json_file, maat_instance->opts.json_ctx.effective_json_md5);
log_info(maat_instance->logger, MODULE_CONFIG_MONITOR,
"JSON file %s md5: %s, generate index file %s OK",
maat_instance->json_ctx.json_file,
maat_instance->json_ctx.effective_json_md5,
maat_instance->json_ctx.iris_file);
maat_instance->opts.json_ctx.json_file,
maat_instance->opts.json_ctx.effective_json_md5,
maat_instance->opts.json_ctx.iris_file);
maat_instance->input_mode = DATA_SOURCE_JSON_FILE;
maat_instance->opts.input_mode = DATA_SOURCE_JSON_FILE;
return 0;
}

4
src/maat_expr.c
View File

@@ -1,10 +1,10 @@
/*
**********************************************************************************************
* File: maat_expr.cpp
* File: maat_expr.c
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

4
src/maat_interval.c
View File

@@ -1,10 +1,10 @@
/*
**********************************************************************************************
* File: maat_interval.cpp
* File: maat_interval.c
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

4
src/maat_ip.c
View File

@@ -1,10 +1,10 @@
/*
**********************************************************************************************
* File: maat_ip.cpp
* File: maat_ip.c
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

4
src/maat_plugin.c
View File

@@ -1,10 +1,10 @@
/*
**********************************************************************************************
* File: maat_plugin.cpp
* File: maat_plugin.c
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

8
src/maat_redis_monitor.c
View File

@@ -1,9 +1,9 @@
/**********************************************************************************************
* File: maat_redis_monitor.cpp
* File: maat_redis_monitor.c
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-11-29
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
@@ -1401,7 +1401,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
maat_instance->load_specific_version,
&new_version, maat_instance->tbl_mgr,
&rule_list, &update_type,
maat_instance->cumulative_update_off,
maat_instance->opts.cumulative_update_off,
maat_instance->logger);
//redis communication error
if (rule_num < 0) {
@@ -1442,7 +1442,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
}
ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num,
maat_instance, maat_instance->foreign_cont_dir);
maat_instance, maat_instance->opts.foreign_cont_dir);
if (ret > 0) {
maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0,
maat_instance->logger);

71
src/maat_rule.c
View File

@@ -1,9 +1,9 @@
/**********************************************************************************************
* File: maat_rule.cpp
* File: maat_rule.c
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
@@ -240,7 +240,7 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in
struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1);
maat_rt->version = version;
int ret = table_manager_runtime_create(maat_instance->tbl_mgr, maat_instance->nr_worker_thread,
int ret = table_manager_runtime_create(maat_instance->tbl_mgr, maat_instance->opts.nr_worker_thread,
maat_instance->garbage_bin);
if (ret < 0) {
FREE(maat_rt);
@@ -252,7 +252,7 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in
maat_rt->sequence_map = maat_kv_store_new();
maat_rt->logger = maat_instance->logger;
maat_rt->ref_garbage_bin = maat_instance->garbage_bin;
maat_rt->ref_cnt = alignment_int64_array_alloc(maat_instance->nr_worker_thread);
maat_rt->ref_cnt = alignment_int64_array_alloc(maat_instance->opts.nr_worker_thread);
return maat_rt;
}
@@ -457,8 +457,8 @@ void *rule_monitor_loop(void *arg)
char maat_name[16] = {0};
struct maat *maat_instance = (struct maat *)arg;
if (strlen(maat_instance->instance_name) > 0) {
snprintf(maat_name, sizeof(maat_name), "MAAT_%s", maat_instance->instance_name);
if (strlen(maat_instance->opts.instance_name) > 0) {
snprintf(maat_name, sizeof(maat_name), "MAAT_%s", maat_instance->opts.instance_name);
} else {
snprintf(maat_name, sizeof(maat_name), "MAAT");
}
@@ -468,7 +468,7 @@ void *rule_monitor_loop(void *arg)
pthread_mutex_lock(&(maat_instance->background_update_mutex));
/* if deferred load on */
if (maat_instance->deferred_load != 0) {
if (maat_instance->opts.deferred_load_on != 0) {
log_info(maat_instance->logger, MODULE_MAAT_RULE,
"Deferred Loading ON, updating in %s:%d", __FUNCTION__, __LINE__);
maat_read_full_config(maat_instance);
@@ -480,42 +480,44 @@ void *rule_monitor_loop(void *arg)
struct stat attrib;
while (maat_instance->is_running) {
log_info(maat_instance->logger, MODULE_MAAT_RULE, "%s:%d.............", __FUNCTION__, __LINE__);
usleep(maat_instance->rule_update_checking_interval_ms * 1000);
usleep(maat_instance->opts.rule_update_checking_interval_ms * 1000);
if (0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
switch (maat_instance->input_mode) {
switch (maat_instance->opts.input_mode) {
case DATA_SOURCE_REDIS:
redis_monitor_traverse(maat_instance->maat_version,
&(maat_instance->mr_ctx),
&(maat_instance->opts.redis_ctx),
maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance);
break;
case DATA_SOURCE_IRIS_FILE:
config_monitor_traverse(maat_instance->maat_version,
maat_instance->iris_ctx.inc_idx_dir,
maat_instance->opts.iris_ctx.inc_idx_dir,
maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance, maat_instance->logger);
maat_instance, maat_instance->opts.decrypt_key,
maat_instance->logger);
break;
case DATA_SOURCE_JSON_FILE:
memset(md5_tmp, 0, sizeof(md5_tmp));
stat(maat_instance->json_ctx.json_file, &attrib);
if (memcmp(&attrib.st_ctim, &(maat_instance->json_ctx.last_md5_time), sizeof(attrib.st_ctim))) {
maat_instance->json_ctx.last_md5_time = attrib.st_ctim;
md5_file(maat_instance->json_ctx.json_file, md5_tmp);
if (0 != strcmp(md5_tmp, maat_instance->json_ctx.effective_json_md5)) {
ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file,
stat(maat_instance->opts.json_ctx.json_file, &attrib);
if (memcmp(&attrib.st_ctim, &(maat_instance->opts.json_ctx.last_md5_time), sizeof(attrib.st_ctim))) {
maat_instance->opts.json_ctx.last_md5_time = attrib.st_ctim;
md5_file(maat_instance->opts.json_ctx.json_file, md5_tmp);
if (0 != strcmp(md5_tmp, maat_instance->opts.json_ctx.effective_json_md5)) {
ret = load_maat_json_file(maat_instance, maat_instance->opts.json_ctx.json_file,
err_str, sizeof(err_str));
if (ret < 0) {
log_error(maat_instance->logger, MODULE_MAAT_RULE,
"[%s:%d] Maat re-initiate with JSON file %s (md5=%s)failed: %s\n",
__FUNCTION__, __LINE__, maat_instance->json_ctx.json_file,
__FUNCTION__, __LINE__, maat_instance->opts.json_ctx.json_file,
md5_tmp, err_str);
} else {
config_monitor_traverse(0, maat_instance->json_ctx.iris_file,
config_monitor_traverse(0, maat_instance->opts.json_ctx.iris_file,
maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance, maat_instance->logger);
maat_instance, maat_instance->opts.decrypt_key,
maat_instance->logger);
log_info(maat_instance->logger, MODULE_MAAT_RULE,
"Maat re-initiate with JSON file %s success, md5: %s\n",
maat_instance->json_ctx.json_file, md5_tmp);
maat_instance->opts.json_ctx.json_file, md5_tmp);
}
}
}
@@ -550,7 +552,7 @@ void *rule_monitor_loop(void *arg)
if (maat_instance->maat_rt != NULL) {
time_t time_window = time(NULL) - maat_instance->maat_rt->last_update_time;
if (time_window >= maat_instance->rule_effect_interval_ms / 1000) {
if (time_window >= maat_instance->opts.rule_effect_interval_ms / 1000) {
maat_runtime_commit(maat_instance->maat_rt, MAAT_UPDATE_TYPE_INC,
maat_instance->maat_rt->version, maat_instance->logger);
log_info(maat_instance->logger, MODULE_MAAT_RULE,
@@ -562,8 +564,8 @@ void *rule_monitor_loop(void *arg)
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
}
maat_garbage_collect_routine(maat_instance->garbage_bin);
if ((1 == maat_instance->stat_on) && (time(NULL) % 2 == 0)) {
maat_stat_output(maat_instance->stat, maat_instance->maat_version, maat_instance->perf_on);
if ((1 == maat_instance->opts.stat_on) && (time(NULL) % 2 == 0)) {
maat_stat_output(maat_instance->stat, maat_instance->maat_version, maat_instance->opts.perf_on);
}
}
@@ -576,18 +578,23 @@ void *rule_monitor_loop(void *arg)
maat_instance->stat = NULL;
}
if (maat_instance->input_mode == DATA_SOURCE_REDIS) {
if (maat_instance->mr_ctx.read_ctx != NULL) {
redisFree(maat_instance->mr_ctx.read_ctx);
maat_instance->mr_ctx.read_ctx = NULL;
if (maat_instance->opts.input_mode == DATA_SOURCE_REDIS) {
if (maat_instance->opts.redis_ctx.read_ctx != NULL) {
redisFree(maat_instance->opts.redis_ctx.read_ctx);
maat_instance->opts.redis_ctx.read_ctx = NULL;
}
if (maat_instance->mr_ctx.write_ctx != NULL) {
redisFree(maat_instance->mr_ctx.write_ctx);
maat_instance->mr_ctx.write_ctx = NULL;
if (maat_instance->opts.redis_ctx.write_ctx != NULL) {
redisFree(maat_instance->opts.redis_ctx.write_ctx);
maat_instance->opts.redis_ctx.write_ctx = NULL;
}
}
if (maat_instance->opts.accept_tags != NULL) {
FREE(maat_instance->opts.accept_tags);
maat_instance->opts.accept_tags = NULL;
}
log_handle_destroy(maat_instance->logger);
FREE(maat_instance);

2
src/maat_stat.c
View File

@@ -4,7 +4,7 @@
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

4
src/maat_table.c
View File

@@ -1,10 +1,10 @@
/*
**********************************************************************************************
* File: maat_table.cpp
* File: maat_table.c
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

25
src/maat_utils.c
View File

@@ -1,10 +1,10 @@
/*
**********************************************************************************************
* File: maat_utils.cpp
* File: maat_utils.c
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
@@ -237,6 +237,21 @@ int system_cmd_rmdir(const char *dir)
return system(cmd);
}
int system_cmd_gzip(const char *src_file, const char *dst_file)
{
char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
snprintf(cmd, sizeof(cmd), "gzip -9 < %s > %s", src_file, dst_file);
return system(cmd);
}
int system_cmd_encrypt(const char *src_file, const char *dst_file, const char *password)
{
char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
snprintf(cmd, sizeof(cmd), "openssl enc -e -aes-256-cbc -k %s -p -nosalt -in %s -out %s -md md5",
password, src_file, dst_file);
return system(cmd);
}
char *md5_file(const char *filename, char *md5string)
{
unsigned char md5[MD5_DIGEST_LENGTH] = {0};
@@ -322,6 +337,7 @@ int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_ou
out_buff_offset += out_blk_len;
EVP_CIPHER_CTX_free(ctx);
EVP_cleanup();
*out_sz = out_buff_offset;
return 0;
@@ -381,10 +397,10 @@ int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compresse
int z_result;
int ret = -1;
size_t buffer_sz = in_compressed_sz * 2;
*out_uncompressed_data = (unsigned char *)malloc(buffer_sz);
*out_uncompressed_data = ALLOC(unsigned char, buffer_sz);
do {
*out_uncompressed_sz=buffer_sz;
*out_uncompressed_sz = buffer_sz;
z_result = gzip_uncompress_one_try(in_compressed_data, in_compressed_sz,
out_uncompressed_data, out_uncompressed_sz);
switch (z_result) {
@@ -394,6 +410,7 @@ int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compresse
case Z_BUF_ERROR:
buffer_sz *= 2;
*out_uncompressed_data = (unsigned char *)realloc(*out_uncompressed_data, buffer_sz);
memset(*out_uncompressed_data, 0, buffer_sz);
break;
default:
ret = -1;

4
src/maat_virtual.c
View File

@@ -1,10 +1,10 @@
/*
**********************************************************************************************
* File: maat_virtual.cpp
* File: maat_virtual.c
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

4
src/rcu_hash.c
View File

@@ -1,10 +1,10 @@
/*
**********************************************************************************************
* File: maat_rhash.cpp
* File: rcu_hash.c
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/

1
test/CMakeLists.txt
View File

@@ -36,6 +36,7 @@ file(COPY maat_json.json DESTINATION ../tools/)
file(COPY ntcrule DESTINATION ./)
file(COPY testdata DESTINATION ./)
file(COPY test_streamfiles DESTINATION ./)
file(COPY json_update DESTINATION ./)
include(GoogleTest)
gtest_discover_tests(maat_framework_gtest)

32
test/json_update/corrupted.json Normal file
View File

@@ -0,0 +1,32 @@
{
"compile_table": "COMPILE",
"group_table": "GROUP",
"rules": [
{
"compile_id": 1
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "hello&world",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
]
}

32
test/json_update/new.json Normal file
View File

@@ -0,0 +1,32 @@
{
"compile_table": "COMPILE",
"group2compile_table": "GROUP2COMPILE",
"group2group_table": "GROUP2GROUP",
"rules": [
{
"compile_id": 2,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "MESA&Maat",
"expr_type": "and",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
]
}

33
test/json_update/old.json Normal file
View File

@@ -0,0 +1,33 @@
{
"compile_table": "COMPILE",
"group2compile_table": "GROUP2COMPILE",
"group2group_table": "GROUP2GROUP",
"rules": [
{
"compile_id": 1,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "hello&world",
"expr_type": "and",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
]
}

104
test/maat_framework_gtest.cpp
View File

@@ -125,7 +125,8 @@ int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
size_t total_line_cnt = 0;
char tmp_iris_full_idx_path[PATH_MAX] = {0};
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL,
&total_line_cnt, NULL, logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
s_rule->ref_ctx = c;
@@ -135,7 +136,8 @@ int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
}
absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL,
s_rule, NULL, logger);
s_rule->ref_ctx = NULL;
line_idx = 0;
absolute_expire_time = 0;
@@ -432,6 +434,104 @@ TEST_F(MaatIris, basic) {
}
#endif
const char* watched_json="./json_update/maat.json";
const char* old_json="./json_update/old.json";
const char* new_json="./json_update/new.json";
const char* corrupted_json="./json_update/corrupted.json";
const char* json_decrypt_key="himaat!";
const char* tmp_gzipped_file_name="./json_update/tmp_gzipped_json.gz";
class JsonUpdate : public testing::Test
{
protected:
static void SetUpTestCase() {
system_cmd_gzip(old_json, tmp_gzipped_file_name);
system_cmd_encrypt(tmp_gzipped_file_name, watched_json, json_decrypt_key);
int scan_interval_ms = 500;
logger = log_handle_create("./maat_framework_gtest.log", 0);
struct maat_options *opts = maat_options_new();
maat_options_set_instance_name(opts, "firewall");
maat_options_set_json_file(opts, watched_json);
maat_options_set_json_file_gzip_flag(opts, 1);
maat_options_set_json_file_decrypt_key(opts, json_decrypt_key);
maat_options_set_rule_update_checking_interval_ms(opts, scan_interval_ms);
_shared_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_GTEST,
"[%s:%d] create maat instance in JsonUpdate failed.",
__FUNCTION__, __LINE__);
assert(0);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_instance;
};
struct maat *JsonUpdate::_shared_maat_instance;
struct log_handle *JsonUpdate::logger;
void scan_with_old_or_new_cfg(struct maat *maat_instance, int is_old)
{
const char *hit_old_data = "Hello world! I'm eve.";
const char *hit_new_data = "Maat was borned in MESA.";
const char *table_name = "HTTP_URL";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat_state *state = maat_state_new(maat_instance, thread_id);
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
int ret = maat_scan_string(maat_instance, table_id, hit_old_data,
strlen(hit_old_data), results, ARRAY_SIZE,
&n_hit_result, state);
if (is_old) {
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_TRUE(results[0] == 1);
} else {
EXPECT_EQ(ret, MAAT_SCAN_OK);
}
maat_state_reset(state);
ret = maat_scan_string(maat_instance, table_id, hit_new_data,
strlen(hit_new_data), results, ARRAY_SIZE,
&n_hit_result, state);
if (!is_old) {
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 2);
} else {
EXPECT_EQ(ret, MAAT_SCAN_OK);
}
maat_state_free(state);
}
TEST_F(JsonUpdate, OldCfg) {
scan_with_old_or_new_cfg(JsonUpdate::_shared_maat_instance, 1);
}
TEST_F(JsonUpdate, NewCfg) {
system_cmd_gzip(corrupted_json, tmp_gzipped_file_name);
system_cmd_encrypt(tmp_gzipped_file_name, watched_json, json_decrypt_key);
sleep(2);
scan_with_old_or_new_cfg(JsonUpdate::_shared_maat_instance, 1);
system_cmd_gzip(new_json, tmp_gzipped_file_name);
int ret = system_cmd_encrypt(tmp_gzipped_file_name, watched_json, json_decrypt_key);
EXPECT_EQ(ret, 0);
sleep(5);
scan_with_old_or_new_cfg(JsonUpdate::_shared_maat_instance, 0);
}
class MaatFlagScan : public testing::Test
{

6
test/maat_framework_perf_gtest.cpp
View File

@@ -130,7 +130,8 @@ static int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
size_t total_line_cnt = 0;
char tmp_iris_full_idx_path[PATH_MAX] = {0};
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL,
&total_line_cnt, NULL, logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
long long server_time = maat_cmd_redis_server_time_s(c);
@@ -139,7 +140,8 @@ static int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
}
absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL,
s_rule, NULL, logger);
line_idx = 0;
absolute_expire_time = 0;

4
test/maat_input_mode_gtest.cpp
View File

@@ -189,7 +189,7 @@ TEST(redis_mode, maat_scan_string) {
char tmp_iris_full_idx_path[PATH_MAX] = {0};
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL,
&total_line_cnt, g_logger);
&total_line_cnt, NULL, g_logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
s_rule->ref_ctx = c;
@@ -198,7 +198,7 @@ TEST(redis_mode, maat_scan_string) {
absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL,
s_rule, g_logger);
s_rule, NULL, g_logger);
s_rule->ref_ctx = NULL;
int success_cnt = 0;
do {

8
tools/maat_redis_tool.cpp
View File

@@ -262,7 +262,7 @@ int write_config_to_redis(redisContext *c, char *json_filename, struct log_handl
}
size_t total_line_cnt = 0;
config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger);
config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, NULL, logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
s_rule->ref_ctx = c;
@@ -272,7 +272,7 @@ int write_config_to_redis(redisContext *c, char *json_filename, struct log_handl
}
s_rule->timeout = server_time + 300;
config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, logger);
config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, NULL, logger);
s_rule->ref_ctx = NULL;
line_idx = 0;
@@ -467,7 +467,7 @@ int main(int argc, char * argv[])
}
size_t total_line_cnt = 0;
config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger);
config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, NULL, logger);
log_error(logger, MODULE_REDIS_TOOL, "Serialize %s to %zu lines, write temp file to %s .",
json_file, total_line_cnt, tmp_iris_path);
@@ -484,7 +484,7 @@ int main(int argc, char * argv[])
s_rule->timeout = server_time + timeout;
}
log_info(logger, MODULE_REDIS_TOOL, "Timeout = %lld\n", s_rule->timeout);
config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, logger);
config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, NULL, logger);
s_rule->ref_ctx = NULL;
int success_cnt = 0;