gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[PATCH]delete useless code

Browse Source
This commit is contained in:
liuwentan
2024-04-03 16:47:30 +08:00
parent cbabcbd6b0
commit 7e25f48fdd
21 changed files with 3147 additions and 5266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
src/json2iris.c
View File

@@ -92,9 +92,9 @@ struct translate_command {
int default_int;
};
static struct iris_table *query_table_info(struct iris_description *p_iris,
const char *table_name,
enum table_type table_type)
static struct iris_table *
query_table_info(struct iris_description *p_iris, const char *table_name,
enum table_type table_type)
{
if (NULL == p_iris || NULL == table_name ||
table_type == TABLE_TYPE_INVALID) {
@@ -127,11 +127,12 @@ static void free_iris_table_info(void *p)
FREE(table);
}
static int set_iris_descriptor(const char *json_file, cJSON *json,
const char *encrypt_key, const char *encrypt_algo,
const char *compile_tn, const char *group2compile_tn,
const char* group2group_tn, redisContext *redis_write_ctx,
struct iris_description *iris_cfg)
static int
set_iris_descriptor(const char *json_file, cJSON *json,
const char *encrypt_key, const char *encrypt_algo,
const char *compile_tn, const char *group2compile_tn,
const char* group2group_tn, redisContext *redis_write_ctx,
struct iris_description *iris_cfg)
{
memset(iris_cfg, 0, sizeof(struct iris_description));
snprintf(iris_cfg->tmp_iris_dir, sizeof(iris_cfg->tmp_iris_dir),
@@ -251,9 +252,10 @@ static int create_tmp_dir(struct iris_description *p)
return 0;
}
static int write_plugin_line(cJSON *plug_table_json, int sequence,
struct iris_description *p_iris,
struct log_handle *logger)
static int
write_plugin_line(cJSON *plug_table_json, int sequence,
struct iris_description *p_iris,
struct log_handle *logger)
{
cJSON *item = cJSON_GetObjectItem(plug_table_json, "table_name");
if (NULL == item || item->type != cJSON_String) {
@@ -345,9 +347,10 @@ static int get_region_seq(struct iris_description *iris_cfg)
return sequence;
}
static int direct_write_rule(cJSON *json, struct maat_kv_store *str2int,
struct translate_command *cmd, int cmd_cnt,
struct iris_table *table, struct log_handle *logger)
static int
direct_write_rule(cJSON *json, struct maat_kv_store *str2int,
struct translate_command *cmd, int cmd_cnt,
struct iris_table *table, struct log_handle *logger)
{
int i = 0;
int ret = -1;
@@ -404,9 +407,11 @@ static int direct_write_rule(cJSON *json, struct maat_kv_store *str2int,
for (i = 0; i < cmd_cnt; i++) {
table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz,
cmd[i].json_value, strlen(cmd[i].json_value));
table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, "\t", 1);
table->write_pos += memcat(&(table->buff), table->write_pos,
&table->buff_sz, "\t", 1);
}
table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, "\n", 1);
table->write_pos += memcat(&(table->buff), table->write_pos,
&table->buff_sz, "\n", 1);
table->line_count++;
ret = 0;
error_out:
@@ -418,8 +423,9 @@ error_out:
return ret;
}
static int write_flag_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
static int
write_flag_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
{
struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0;
@@ -455,8 +461,9 @@ static int write_flag_line(cJSON *region_json, struct iris_description *p_iris,
json_cmd, cmd_cnt, table, logger);
}
static int write_expr_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
static int
write_expr_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
{
struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0;
@@ -503,8 +510,9 @@ static int write_expr_line(cJSON *region_json, struct iris_description *p_iris,
json_cmd, cmd_cnt, table, logger);
}
static int write_ip_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
static int
write_ip_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
{
struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0;
@@ -555,8 +563,9 @@ static int write_ip_line(cJSON *region_json, struct iris_description *p_iris,
json_cmd, cmd_cnt, table, logger);
}
static int write_interval_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
static int
write_interval_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
{
struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0;
@@ -592,8 +601,9 @@ static int write_interval_line(cJSON *region_json, struct iris_description *p_ir
json_cmd, cmd_cnt, table, logger);
}
static int write_region_rule(cJSON *region_json, int compile_id, int group_id,
struct iris_description *p_iris, struct log_handle *logger)
static int
write_region_rule(cJSON *region_json, int compile_id, int group_id,
struct iris_description *p_iris, struct log_handle *logger)
{
cJSON *item = cJSON_GetObjectItem(region_json, "table_name");
if (NULL == item || item->type != cJSON_String) {
@@ -662,11 +672,12 @@ static int write_region_rule(cJSON *region_json, int compile_id, int group_id,
return ret;
}
static int write_group2compile_line(int *group_ids, size_t n_group_id,
int compile_id, int group_not_flag,
int clause_index, const char *vtable,
struct iris_description *p_iris,
struct iris_table *g2c_table)
static int
write_group2compile_line(int *group_ids, size_t n_group_id,
int compile_id, int group_not_flag,
int clause_index, const char *vtable,
struct iris_description *p_iris,
struct iris_table *g2c_table)
{
char buff[4096] = {0};
struct iris_table *table = NULL;
@@ -689,23 +700,24 @@ static int write_group2compile_line(int *group_ids, size_t n_group_id,
strcat(group_id_str, tmp_str);
}
group_id_str[strlen(group_id_str) - 1] = '\0';
snprintf(buff, sizeof(buff), "%s\t%d\t%d\t%s\t%d\t1\n", group_id_str, compile_id,
group_not_flag, vtable, clause_index);
snprintf(buff, sizeof(buff), "%s\t%d\t%d\t%s\t%d\t1\n", group_id_str,
compile_id, group_not_flag, vtable, clause_index);
} else {
snprintf(buff, sizeof(buff), "%d\t%d\t%d\t%s\t%d\t1\n", group_ids[0], compile_id,
group_not_flag, vtable, clause_index);
snprintf(buff, sizeof(buff), "%d\t%d\t%d\t%s\t%d\t1\n", group_ids[0],
compile_id, group_not_flag, vtable, clause_index);
}
table->write_pos += memcat(&(table->buff), table->write_pos, &(table->buff_sz),
buff, strlen(buff));
table->write_pos += memcat(&(table->buff), table->write_pos,
&(table->buff_sz), buff, strlen(buff));
table->line_count++;
return 0;
}
static int write_group2group_line(int group_id, UT_array *incl_sub_group_ids,
UT_array *excl_sub_group_ids,
struct iris_description *p_iris)
static int
write_group2group_line(int group_id, UT_array *incl_sub_group_ids,
UT_array *excl_sub_group_ids,
struct iris_description *p_iris)
{
char buff[MAX_BUFF_LEN*4] = {0};
struct iris_table *table = p_iris->group2group_table;
@@ -765,10 +777,11 @@ static int write_group2group_line(int group_id, UT_array *incl_sub_group_ids,
return 0;
}
static int write_group_rule(cJSON *group_json, int parent_id,
int parent_type, int tracking_compile_id,
int Nth_group, struct iris_description *p_iris,
struct log_handle *logger)
static int
write_group_rule(cJSON *group_json, int parent_id,
int parent_type, int tracking_compile_id,
int Nth_group, struct iris_description *p_iris,
struct log_handle *logger)
{
int ret = 0;
int group_not_flag = 0;
@@ -839,11 +852,12 @@ static int write_group_rule(cJSON *group_json, int parent_id,
if (group_name_cnt > 0) {
int group_ids[group_name_cnt];
for (size_t i = 0; i < group_name_cnt; i++) {
struct group_info *group_info = group_info_read(p_iris->group_name_map,
group_name_array[i]);
struct group_info *group_info =
group_info_read(p_iris->group_name_map, group_name_array[i]);
if (NULL == group_info) {
log_fatal(logger, MODULE_JSON2IRIS, "[%s:%d] group_name:%s has no group_id",
__FUNCTION__, __LINE__, group_name_array[i]);
log_fatal(logger, MODULE_JSON2IRIS, "[%s:%d] group_name:%s"
" has no group_id", __FUNCTION__, __LINE__,
group_name_array[i]);
return -1;
}
group_ids[i] = group_info->group_id;
@@ -854,7 +868,8 @@ static int write_group_rule(cJSON *group_json, int parent_id,
virtual_table, p_iris, g2c_table);
} else {
struct group_info *group_info = group_info_read(p_iris->group_name_map, group_name);
struct group_info *group_info =
group_info_read(p_iris->group_name_map, group_name);
// exist group name, regions and sub groups will be ommit.
if (NULL == group_info) {
if (0 == strncasecmp(group_name, untitled_group_name,
@@ -864,8 +879,8 @@ static int write_group_rule(cJSON *group_json, int parent_id,
if (-1 == group_id) {
log_fatal(logger, MODULE_JSON2IRIS,
"[%s:%d] group_name:<%s> has no group_id", __FUNCTION__,
__LINE__, group_name);
"[%s:%d] group_name:<%s> has no group_id",
__FUNCTION__, __LINE__, group_name);
return -1;
}
@@ -924,13 +939,15 @@ static int write_group_rule(cJSON *group_json, int parent_id,
return 0;
}
static int write_compile_line(cJSON *compile, struct iris_description *p_iris,
struct log_handle *logger)
static int
write_compile_line(cJSON *compile, struct iris_description *p_iris,
struct log_handle *logger)
{
cJSON *item=cJSON_GetObjectItem(compile, "compile_id");
if (item->type != cJSON_Number) {
log_fatal(logger, MODULE_JSON2IRIS,
"[%s:%d] compile_id format not number", __FUNCTION__, __LINE__);
"[%s:%d] compile_id format not number",
__FUNCTION__, __LINE__);
return -1;
}
int compile_id = item->valueint;
@@ -1021,8 +1038,8 @@ static int write_compile_line(cJSON *compile, struct iris_description *p_iris,
table_info = query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE);
}
int ret = direct_write_rule(compile, p_iris->str2int_map, compile_cmd, cmd_cnt,
table_info, logger);
int ret = direct_write_rule(compile, p_iris->str2int_map, compile_cmd,
cmd_cnt, table_info, logger);
if (ret < 0) {
return -1;
}
@@ -1073,7 +1090,8 @@ static int write_index_file(struct iris_description *p_iris,
if (NULL == p_iris->idx_fp) {
log_fatal(logger, MODULE_JSON2IRIS,
"[%s:%d] index file %s fopen error %s",
__FUNCTION__, __LINE__, p_iris->index_path, strerror(errno));
__FUNCTION__, __LINE__, p_iris->index_path,
strerror(errno));
return -1;
}
@@ -1165,8 +1183,10 @@ static int write_group2group_rule(struct iris_description *p_iris,
struct group_info *group_info = NULL, *tmp_group_info = NULL;
HASH_ITER(hh, p_iris->group_name_map, group_info, tmp_group_info) {
ret = write_group2group_line(group_info->group_id, group_info->incl_sub_group_ids,
group_info->excl_sub_group_ids, p_iris);
ret = write_group2group_line(group_info->group_id,
group_info->incl_sub_group_ids,
group_info->excl_sub_group_ids,
p_iris);
if (ret < 0) {
log_fatal(logger, MODULE_JSON2IRIS,
"[%s:%d] write group2group line failed for super_group:%d",

109
src/maat_api.c
View File

@@ -90,7 +90,8 @@ void maat_options_free(struct maat_options *opts)
FREE(opts);
}
int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread)
int maat_options_set_caller_thread_number(struct maat_options *opts,
size_t n_thread)
{
if (NULL == opts) {
return -1;
@@ -100,7 +101,8 @@ int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_th
return 0;
}
int maat_options_set_accept_tags(struct maat_options *opts, const char *accept_tags)
int maat_options_set_accept_tags(struct maat_options *opts,
const char *accept_tags)
{
if (NULL == opts || NULL == accept_tags) {
return -1;
@@ -110,7 +112,8 @@ int maat_options_set_accept_tags(struct maat_options *opts, const char *accept_t
return 0;
}
int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms)
int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts,
int interval_ms)
{
if (NULL == opts || interval_ms < 0) {
return -1;
@@ -130,7 +133,8 @@ int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms)
return 0;
}
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name)
int maat_options_set_instance_name(struct maat_options *opts,
const char *instance_name)
{
if (NULL == opts || NULL == instance_name ||
strlen(instance_name) > MAX_INSTANCE_NAME_LEN) {
@@ -171,7 +175,8 @@ int maat_options_set_perf_on(struct maat_options *opts)
return 0;
}
int maat_options_set_foreign_cont_dir(struct maat_options *opts, const char *dir)
int maat_options_set_foreign_cont_dir(struct maat_options *opts,
const char *dir)
{
if (NULL == opts || NULL == dir || strlen(dir) >= NAME_MAX) {
return -1;
@@ -181,7 +186,8 @@ int maat_options_set_foreign_cont_dir(struct maat_options *opts, const char *dir
return 0;
}
int maat_options_set_iris(struct maat_options *opts, const char *full_directory,
int maat_options_set_iris(struct maat_options *opts,
const char *full_directory,
const char *increment_directory)
{
if (NULL == opts || strlen(full_directory) >= NAME_MAX ||
@@ -189,16 +195,20 @@ int maat_options_set_iris(struct maat_options *opts, const char *full_directory,
return -1;
}
memcpy(opts->iris_ctx.full_idx_dir, full_directory, strlen(full_directory));
memcpy(opts->iris_ctx.inc_idx_dir, increment_directory, strlen(increment_directory));
memcpy(opts->iris_ctx.full_idx_dir, full_directory,
strlen(full_directory));
memcpy(opts->iris_ctx.inc_idx_dir, increment_directory,
strlen(increment_directory));
opts->input_mode = DATA_SOURCE_IRIS_FILE;
return 0;
}
int maat_options_set_json_file(struct maat_options *opts, const char *json_filename)
int maat_options_set_json_file(struct maat_options *opts,
const char *json_filename)
{
strncpy(opts->json_ctx.json_file, json_filename, sizeof(opts->json_ctx.json_file));
strncpy(opts->json_ctx.json_file, json_filename,
sizeof(opts->json_ctx.json_file));
opts->input_mode = DATA_SOURCE_JSON_FILE;
return 0;
@@ -240,7 +250,8 @@ int maat_options_set_redis(struct maat_options *opts, const char *redis_ip,
return 0;
}
int maat_options_set_stat_file(struct maat_options *opts, const char *stat_filename)
int maat_options_set_stat_file(struct maat_options *opts,
const char *stat_filename)
{
size_t str_len = MIN(sizeof(opts->stat_file), strlen(stat_filename));
@@ -347,9 +358,10 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
} else {
char log_path[MAX_NAME_STR_LEN] = {0};
if (strlen(maat_inst->opts.inst_name) > 0) {
snprintf(log_path, sizeof(log_path), "%s.log", maat_inst->opts.inst_name);
snprintf(log_path, sizeof(log_path), "%s.log",
maat_inst->opts.inst_name);
} else {
snprintf(log_path, sizeof(log_path), "maat.log");
snprintf(log_path, sizeof(log_path), "./maat.log");
}
maat_inst->logger = log_handle_create(log_path, maat_inst->opts.log_level);
}
@@ -360,17 +372,20 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
}
if (0 == strlen(maat_inst->opts.foreign_cont_dir)) {
snprintf(maat_inst->opts.foreign_cont_dir, sizeof(maat_inst->opts.foreign_cont_dir),
snprintf(maat_inst->opts.foreign_cont_dir,
sizeof(maat_inst->opts.foreign_cont_dir),
"%s_files", table_info_path);
}
system_cmd_mkdir(maat_inst->opts.foreign_cont_dir);
if (0 == strlen(maat_inst->opts.stat_file)) {
snprintf(maat_inst->opts.stat_file, sizeof(maat_inst->opts.stat_file), "maat_stat.fs");
snprintf(maat_inst->opts.stat_file, sizeof(maat_inst->opts.stat_file),
"maat_stat.fs");
}
snprintf(maat_inst->opts.decrypt_algo, sizeof(maat_inst->opts.decrypt_algo), "aes-256-cbc");
snprintf(maat_inst->opts.decrypt_algo, sizeof(maat_inst->opts.decrypt_algo),
"aes-256-cbc");
maat_inst->is_running = 0;
maat_inst->maat_version = 0;
@@ -394,9 +409,10 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
pthread_mutex_init(&(maat_inst->background_update_mutex), NULL);
maat_inst->tbl_mgr = table_manager_create(table_info_path, maat_inst->opts.accept_tags,
maat_inst->opts.expr_engine, maat_inst->garbage_bin,
maat_inst->logger);
maat_inst->tbl_mgr =
table_manager_create(table_info_path, maat_inst->opts.accept_tags,
maat_inst->opts.expr_engine, maat_inst->garbage_bin,
maat_inst->logger);
if (NULL == maat_inst->tbl_mgr) {
goto failed;
}
@@ -406,7 +422,8 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
}
if (1 == maat_inst->opts.stat_on) {
int ret = maat_stat_init(maat_inst->stat, maat_inst->tbl_mgr, maat_inst->garbage_bin);
int ret = maat_stat_init(maat_inst->stat, maat_inst->tbl_mgr,
maat_inst->garbage_bin);
if (ret < 0) {
log_fatal(maat_inst->logger, MODULE_MAAT_API,
"[%s:%d] maat_stat_init failed.", __FUNCTION__, __LINE__);
@@ -414,7 +431,8 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
}
}
pthread_create(&(maat_inst->cfg_mon_thread), NULL, rule_monitor_loop, (void *)maat_inst);
pthread_create(&(maat_inst->cfg_mon_thread), NULL, rule_monitor_loop,
(void *)maat_inst);
return maat_inst;
@@ -488,7 +506,8 @@ int maat_get_table_id(struct maat *maat_inst, const char *table_name)
return table_manager_get_table_id(table_mgr, table_name);
}
const char *maat_get_table_schema_tag(struct maat *maat_inst, int table_id)
const char *
maat_get_table_schema_tag(struct maat *maat_inst, int table_id)
{
if (NULL == maat_inst || table_id < 0) {
return NULL;
@@ -498,18 +517,20 @@ const char *maat_get_table_schema_tag(struct maat *maat_inst, int table_id)
return table_manager_get_table_schema_tag(table_mgr, table_id);
}
static inline void maat_runtime_ref_inc(struct maat_runtime *maat_rt, int thread_id)
static inline void
maat_runtime_ref_inc(struct maat_runtime *maat_rt, int thread_id)
{
if (NULL == maat_rt) {
if (NULL == maat_rt || thread_id < 0) {
return;
}
alignment_int64_array_add(maat_rt->ref_cnt, thread_id, 1);
}
static inline void maat_runtime_ref_dec(struct maat_runtime *maat_rt, int thread_id)
static inline void
maat_runtime_ref_dec(struct maat_runtime *maat_rt, int thread_id)
{
if (NULL == maat_rt) {
if (NULL == maat_rt || thread_id < 0) {
return;
}
@@ -530,8 +551,9 @@ int maat_table_callback_register(struct maat *maat_inst, int table_id,
if (NULL == schema) {
pthread_mutex_unlock(&(maat_inst->background_update_mutex));
log_fatal(maat_inst->logger, MODULE_MAAT_API,
"[%s:%d] table(table_id:%d) schema is NULL, register callback failed",
__FUNCTION__, __LINE__, table_id);
"[%s:%d] table(table_id:%d) schema is NULL, "
"register callback failed", __FUNCTION__,
__LINE__, table_id);
return -1;
}
@@ -549,7 +571,8 @@ int maat_table_callback_register(struct maat *maat_inst, int table_id,
void *runtime = table_manager_get_runtime(maat_inst->tbl_mgr, table_id);
enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
enum table_type table_type =
table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
if (table_type != TABLE_TYPE_PLUGIN) {
pthread_mutex_unlock(&(maat_inst->background_update_mutex));
log_fatal(maat_inst->logger, MODULE_MAAT_API,
@@ -583,12 +606,13 @@ int maat_table_callback_register(struct maat *maat_inst, int table_id,
return 0;
}
static int generic_plugin_table_set_ex_schema(struct table_manager *tbl_mgr, int table_id,
maat_ex_new_func_t *new_func,
maat_ex_free_func_t *free_func,
maat_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger)
static int
generic_plugin_table_set_ex_schema(struct table_manager *tbl_mgr, int table_id,
maat_ex_new_func_t *new_func,
maat_ex_free_func_t *free_func,
maat_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger)
{
void *schema = table_manager_get_schema(tbl_mgr, table_id);
if (NULL == schema) {
@@ -1447,9 +1471,9 @@ int maat_scan_integer(struct maat *maat_inst, int table_id,
}
}
int maat_scan_ipv4_port(struct maat *maat_inst, int table_id, uint32_t ip_addr, int port,
long long *results, size_t n_result, size_t *n_hit_result,
struct maat_state *state)
int maat_scan_ipv4_port(struct maat *maat_inst, int table_id, uint32_t ip_addr,
int port, long long *results, size_t n_result,
size_t *n_hit_result, struct maat_state *state)
{
if ((NULL == maat_inst) || table_id < 0 || table_id >= MAX_TABLE_NUM ||
(NULL == results) || (0 == n_result) || (NULL == n_hit_result) ||
@@ -1532,9 +1556,9 @@ int maat_scan_ipv4_port(struct maat *maat_inst, int table_id, uint32_t ip_addr,
}
}
int maat_scan_ipv6_port(struct maat *maat_inst, int table_id, uint8_t *ip_addr, int port,
long long *results, size_t n_result, size_t *n_hit_result,
struct maat_state *state)
int maat_scan_ipv6_port(struct maat *maat_inst, int table_id, uint8_t *ip_addr,
int port, long long *results, size_t n_result,
size_t *n_hit_result, struct maat_state *state)
{
if ((NULL == maat_inst) || table_id < 0 || table_id >= MAX_TABLE_NUM ||
(NULL == ip_addr) || (NULL == results) || (0 == n_result) ||
@@ -1616,6 +1640,7 @@ int maat_scan_ipv6_port(struct maat *maat_inst, int table_id, uint8_t *ip_addr,
return MAAT_SCAN_OK;
}
}
#define PORT_IGNORED -1
inline int maat_scan_ipv6(struct maat *instance, int table_id, uint8_t *ip_addr,
long long *results, size_t n_result, size_t *n_hit_result,
@@ -1624,6 +1649,7 @@ inline int maat_scan_ipv6(struct maat *instance, int table_id, uint8_t *ip_addr,
return maat_scan_ipv6_port(instance, table_id, ip_addr, PORT_IGNORED,
results, n_result, n_hit_result, state);
}
inline int maat_scan_ipv4(struct maat *instance, int table_id, uint32_t ip_addr,
long long *results, size_t n_result, size_t *n_hit_result,
struct maat_state *state)
@@ -1631,6 +1657,7 @@ inline int maat_scan_ipv4(struct maat *instance, int table_id, uint32_t ip_addr,
return maat_scan_ipv4_port(instance, table_id, ip_addr, PORT_IGNORED,
results, n_result, n_hit_result, state);
}
int maat_scan_string(struct maat *maat_inst, int table_id,
const char *data, size_t data_len,
long long *results, size_t n_result,

288
src/maat_compile.c
View File

@@ -603,7 +603,9 @@ void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt)
return;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
struct compile_runtime *compile_rt =
(struct compile_runtime *)compile_runtime;
compile_rt->ref_maat_rt = maat_rt;
}
@@ -615,7 +617,9 @@ void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num,
return NULL;
}
struct group2compile_runtime *g2c_rt = ALLOC(struct group2compile_runtime, 1);
struct group2compile_runtime *g2c_rt =
ALLOC(struct group2compile_runtime, 1);
g2c_rt->tbl_not_clause_hash = NULL;
return g2c_rt;
@@ -623,7 +627,8 @@ void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num,
void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime)
{
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
struct group2compile_runtime *g2c_rt =
(struct group2compile_runtime *)g2c_runtime;
g2c_rt->ref_compile_rt = (struct compile_runtime *)compile_runtime;
}
@@ -634,7 +639,9 @@ void group2compile_runtime_free(void *g2c_runtime)
return;
}
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
struct group2compile_runtime *g2c_rt =
(struct group2compile_runtime *)g2c_runtime;
if (g2c_rt->tbl_not_clause_hash != NULL) {
struct table_clause *not_clause = NULL, *tmp_not_clause = NULL;
HASH_ITER(hh, g2c_rt->tbl_not_clause_hash, not_clause, tmp_not_clause) {
@@ -886,7 +893,8 @@ static void maat_compile_clause_remove_literal(struct maat_compile *compile,
}
static struct bool_matcher *
maat_compile_bool_matcher_new(struct compile_runtime *compile_rt, size_t *compile_cnt)
maat_compile_bool_matcher_new(struct compile_runtime *compile_rt,
size_t *compile_cnt)
{
if (NULL == compile_rt) {
return NULL;
@@ -912,7 +920,8 @@ maat_compile_bool_matcher_new(struct compile_runtime *compile_rt, size_t *compil
has_clause_num++;
if (0 == clause->clause_id) {
clause->clause_id = maat_runtime_get_sequence(compile_rt->ref_maat_rt, "clause_id");
clause->clause_id =
maat_runtime_get_sequence(compile_rt->ref_maat_rt, "clause_id");
}
}
assert(has_clause_num == iter_compile->actual_clause_num);
@@ -1063,19 +1072,24 @@ build_clause_id_kv_hash(struct compile_runtime *compile_rt, int not_flag)
struct clause_literal *tmp_cl = NULL;
for (size_t j = 0; j < utarray_len(clause->literals); j++) {
tmp_cl = (struct clause_literal *)utarray_eltptr(clause->literals, j);
for (size_t k = 0; k < tmp_cl->group_cnt; k++) {
struct clause_query_key key = {tmp_cl->group_ids[k], tmp_cl->vtable_id, clause->not_flag};
struct clause_query_key key =
{tmp_cl->group_ids[k], tmp_cl->vtable_id, clause->not_flag};
struct clause_id_kv *clause_id_kv = NULL;
HASH_FIND(hh, clause_id_kv_hash, &key, sizeof(struct clause_query_key), clause_id_kv);
HASH_FIND(hh, clause_id_kv_hash, &key, sizeof(struct clause_query_key),
clause_id_kv);
if (NULL == clause_id_kv) {
clause_id_kv = ALLOC(struct clause_id_kv, 1);
clause_id_kv->key = key;
utarray_new(clause_id_kv->clause_ids, &ut_clause_id_icd);
HASH_ADD_KEYPTR(hh, clause_id_kv_hash, &clause_id_kv->key, sizeof(clause_id_kv->key), clause_id_kv);
HASH_ADD_KEYPTR(hh, clause_id_kv_hash, &clause_id_kv->key,
sizeof(clause_id_kv->key), clause_id_kv);
}
if (utarray_find(clause_id_kv->clause_ids, &(clause->clause_id), compare_clause_id)) {
if (utarray_find(clause_id_kv->clause_ids, &(clause->clause_id),
compare_clause_id)) {
continue;
}
utarray_push_back(clause_id_kv->clause_ids, &(clause->clause_id));
@@ -1089,7 +1103,8 @@ build_clause_id_kv_hash(struct compile_runtime *compile_rt, int not_flag)
return clause_id_kv_hash;
}
static int maat_compile_has_clause(struct maat_compile *compile, long long clause_id)
static int
maat_compile_has_clause(struct maat_compile *compile, long long clause_id)
{
struct compile_clause *clause = NULL;
@@ -1107,8 +1122,9 @@ static int maat_compile_has_clause(struct maat_compile *compile, long long claus
return 0;
}
static size_t compile_state_if_new_hit_compile(struct compile_state *compile_state,
struct maat_compile *compile)
static size_t
compile_state_if_new_hit_compile(struct compile_state *compile_state,
struct maat_compile *compile)
{
size_t i = 0;
size_t r_in_c_cnt = 0;
@@ -1117,7 +1133,8 @@ static size_t compile_state_if_new_hit_compile(struct compile_state *compile_sta
long long new_hit_clause_id = 0;
if (0 == compile_state->this_scan_not_logic) {
for (i = 0; i < utarray_len(compile_state->this_scan_hit_clauses); i++) {
new_hit_clause_id = *(long long*)utarray_eltptr(compile_state->this_scan_hit_clauses, i);
new_hit_clause_id =
*(long long*)utarray_eltptr(compile_state->this_scan_hit_clauses, i);
ret = maat_compile_has_clause(compile, new_hit_clause_id);
if (ret) {
r_in_c_cnt++;
@@ -1125,7 +1142,8 @@ static size_t compile_state_if_new_hit_compile(struct compile_state *compile_sta
}
} else {
for (i = 0; i < utarray_len(compile_state->this_scan_hit_not_clauses); i++) {
new_hit_clause_id = *(long long*)utarray_eltptr(compile_state->this_scan_hit_not_clauses, i);
new_hit_clause_id =
*(long long*)utarray_eltptr(compile_state->this_scan_hit_not_clauses, i);
ret = maat_compile_has_clause(compile, new_hit_clause_id);
if (ret) {
r_in_c_cnt++;
@@ -1136,20 +1154,23 @@ static size_t compile_state_if_new_hit_compile(struct compile_state *compile_sta
return r_in_c_cnt;
}
static void compile_state_update_hit_compile_table_id(struct compile_state *compile_state,
long long compile_id, int table_id)
static void
compile_state_update_hit_compile_table_id(struct compile_state *compile_state,
long long compile_id, int table_id)
{
if (!utarray_find(compile_state->hit_compile_table_ids, &compile_id, compare_compile_id)) {
if (!utarray_find(compile_state->hit_compile_table_ids, &compile_id,
compare_compile_id)) {
struct compile2table_id compile_table_id = {compile_id, table_id};
utarray_push_back(compile_state->hit_compile_table_ids, &compile_table_id);
utarray_sort(compile_state->hit_compile_table_ids, compare_compile_id);
}
}
static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt,
struct compile_state *compile_state,
int thread_id, void **user_data_array,
size_t ud_array_size)
static size_t
maat_compile_bool_matcher_match(struct compile_runtime *compile_rt,
struct compile_state *compile_state,
int thread_id, void **user_data_array,
size_t ud_array_size)
{
size_t ud_result_cnt = 0;
struct maat_compile *compile = NULL;
@@ -1161,8 +1182,9 @@ static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt
compile_state->compile_rt_version = compile_rt->version;
}
if (NULL == compile_rt->bm || 0 == utarray_len(compile_state->all_hit_clauses)
|| compile_state->compile_rt_version != compile_rt->version) {
if (NULL == compile_rt->bm ||
0 == utarray_len(compile_state->all_hit_clauses) ||
compile_state->compile_rt_version != compile_rt->version) {
return 0;
}
@@ -1175,10 +1197,11 @@ static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt
}
#endif
int bool_match_ret = bool_matcher_match(compile_rt->bm,
(unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
utarray_len(compile_state->all_hit_clauses),
expr_match, MAX_HIT_COMPILE_NUM);
int bool_match_ret =
bool_matcher_match(compile_rt->bm,
(unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
utarray_len(compile_state->all_hit_clauses),
expr_match, MAX_HIT_COMPILE_NUM);
for (int i = 0; i < bool_match_ret && ud_result_cnt < ud_array_size; i++) {
compile = (struct maat_compile *)expr_match[i].user_tag;
assert(compile->magic_num == MAAT_COMPILE_MAGIC);
@@ -1187,7 +1210,8 @@ static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt
continue;
}
size_t n_new_hit_compile = compile_state_if_new_hit_compile(compile_state, compile);
size_t n_new_hit_compile =
compile_state_if_new_hit_compile(compile_state, compile);
if (compile->user_data != NULL && n_new_hit_compile > 0) {
user_data_array[ud_result_cnt] = compile->user_data;
@@ -1223,7 +1247,8 @@ maat_compile_clone(struct maat_compile *compile, int deep_copy)
new_compile->actual_clause_num = compile->actual_clause_num;
new_compile->declared_clause_num = compile->declared_clause_num;
if (1 == deep_copy && compile->user_data != NULL) {
new_compile->user_data = compile_item_clone((struct compile_item *)compile->user_data);
new_compile->user_data =
compile_item_clone((struct compile_item *)compile->user_data);
}
struct clause_literal *tmp_literal = NULL;
@@ -1233,7 +1258,8 @@ maat_compile_clone(struct maat_compile *compile, int deep_copy)
new_compile->clauses[i].not_flag = compile->clauses[i].not_flag;
utarray_new(new_compile->clauses[i].literals, &ut_clause_literal_icd);
for (size_t j = 0; j < utarray_len(compile->clauses[i].literals); j++) {
tmp_literal = (struct clause_literal *)utarray_eltptr(compile->clauses[i].literals, j);
tmp_literal =
(struct clause_literal *)utarray_eltptr(compile->clauses[i].literals, j);
utarray_push_back(new_compile->clauses[i].literals, tmp_literal);
}
}
@@ -1411,7 +1437,8 @@ struct compile_state *compile_state_new(void)
return compile_state;
}
static long long compile_state_hit_not_tbl_groups_free(struct compile_state *compile_state)
static long long
compile_state_hit_not_tbl_groups_free(struct compile_state *compile_state)
{
if (NULL == compile_state) {
return 0;
@@ -1420,7 +1447,8 @@ static long long compile_state_hit_not_tbl_groups_free(struct compile_state *com
long long free_bytes = 0;
struct table_group *tbl_group = NULL, *tmp_tbl_group = NULL;
HASH_ITER(hh, compile_state->hit_not_tbl_groups, tbl_group, tmp_tbl_group) {
free_bytes += (sizeof(tbl_group) + utarray_len(tbl_group->group_ids) * sizeof(long long));
free_bytes +=
(sizeof(tbl_group) + utarray_len(tbl_group->group_ids) * sizeof(long long));
HASH_DEL(compile_state->hit_not_tbl_groups, tbl_group);
if (tbl_group->group_ids != NULL) {
utarray_free(tbl_group->group_ids);
@@ -1474,49 +1502,57 @@ void compile_state_free(struct compile_state *compile_state,
}
if (compile_state->all_hit_clauses != NULL) {
free_bytes += utarray_size(compile_state->all_hit_clauses) * sizeof(long long);
free_bytes += utarray_size(compile_state->all_hit_clauses) *
sizeof(long long);
utarray_free(compile_state->all_hit_clauses);
compile_state->all_hit_clauses = NULL;
}
if (compile_state->this_scan_hit_clauses != NULL) {
free_bytes += utarray_size(compile_state->this_scan_hit_clauses) * sizeof(long long);
free_bytes += utarray_size(compile_state->this_scan_hit_clauses) *
sizeof(long long);
utarray_free(compile_state->this_scan_hit_clauses);
compile_state->this_scan_hit_clauses = NULL;
}
if (compile_state->this_scan_hit_not_clauses != NULL) {
free_bytes += utarray_size(compile_state->this_scan_hit_not_clauses) * sizeof(long long);
free_bytes += utarray_size(compile_state->this_scan_hit_not_clauses) *
sizeof(long long);
utarray_free(compile_state->this_scan_hit_not_clauses);
compile_state->this_scan_hit_not_clauses = NULL;
}
if (compile_state->exclude_not_clauses != NULL) {
free_bytes += utarray_size(compile_state->exclude_not_clauses) * sizeof(long long);
free_bytes += utarray_size(compile_state->exclude_not_clauses) *
sizeof(long long);
utarray_free(compile_state->exclude_not_clauses);
compile_state->exclude_not_clauses = NULL;
}
if (compile_state->direct_hit_groups != NULL) {
free_bytes += utarray_size(compile_state->direct_hit_groups) * sizeof(struct maat_hit_group);
free_bytes += utarray_size(compile_state->direct_hit_groups) *
sizeof(struct maat_hit_group);
utarray_free(compile_state->direct_hit_groups);
compile_state->direct_hit_groups = NULL;
}
if (compile_state->indirect_hit_groups != NULL) {
free_bytes += utarray_size(compile_state->indirect_hit_groups) * sizeof(struct maat_hit_group);
free_bytes += utarray_size(compile_state->indirect_hit_groups) *
sizeof(struct maat_hit_group);
utarray_free(compile_state->indirect_hit_groups);
compile_state->indirect_hit_groups = NULL;
}
if (compile_state->last_hit_groups != NULL) {
free_bytes += utarray_size(compile_state->last_hit_groups) * sizeof(struct maat_hit_group);
free_bytes += utarray_size(compile_state->last_hit_groups) *
sizeof(struct maat_hit_group);
utarray_free(compile_state->last_hit_groups);
compile_state->last_hit_groups = NULL;
}
if (compile_state->hit_compile_table_ids != NULL) {
free_bytes += utarray_size(compile_state->hit_compile_table_ids) * sizeof(struct compile2table_id);
free_bytes += utarray_size(compile_state->hit_compile_table_ids) *
sizeof(struct compile2table_id);
utarray_free(compile_state->hit_compile_table_ids);
compile_state->hit_compile_table_ids = NULL;
}
@@ -1530,9 +1566,10 @@ void compile_state_free(struct compile_state *compile_state,
thread_id, free_bytes);
}
static void compile_state_add_internal_hit_path(struct compile_state *compile_state,
long long item_id, long long group_id,
int vtable_id, int NOT_flag, int Nth_scan)
static void
compile_state_add_internal_hit_path(struct compile_state *compile_state,
long long item_id, long long group_id,
int vtable_id, int NOT_flag, int Nth_scan)
{
if (NULL == compile_state) {
return;
@@ -1580,9 +1617,10 @@ static int maat_compile_has_clause_query_key(struct maat_compile *compile,
return 0;
}
static size_t maat_compile_get_hit_clause_index(struct maat_compile *compile,
int vtable_id, long long hit_group_id,
int *clause_idx_array, size_t array_size)
static size_t
maat_compile_get_hit_clause_index(struct maat_compile *compile,
int vtable_id, long long hit_group_id,
int *clause_idx_array, size_t array_size)
{
size_t hit_clause_cnt = 0;
struct compile_clause *tmp_clause = NULL;
@@ -1613,8 +1651,9 @@ static size_t maat_compile_get_hit_clause_index(struct maat_compile *compile,
return hit_clause_cnt;
}
static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_paths,
size_t n_path, const struct maat_hit_path *find)
static int
maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_paths,
size_t n_path, const struct maat_hit_path *find)
{
for (size_t i = 0; i < n_path; i++) {
if (0 == memcmp(hit_paths + i, find, sizeof(*find))) {
@@ -1627,7 +1666,8 @@ static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_path
void populate_hit_path_with_compile(struct maat_hit_path *hit_path_array,
size_t array_idx, size_t n_hit_path,
size_t *n_new_hit_path, struct maat_compile *compile)
size_t *n_new_hit_path,
struct maat_compile *compile)
{
size_t i = 0;
size_t idx = array_idx;
@@ -1643,9 +1683,11 @@ void populate_hit_path_with_compile(struct maat_hit_path *hit_path_array,
if (hit_path_array[idx].compile_id < 0) {
hit_path_array[idx].compile_id = compile->compile_id;
// find out which clause in compile hit
n_clause_index = maat_compile_get_hit_clause_index(compile, hit_path_array[idx].vtable_id,
hit_path_array[idx].top_group_id, clause_index_array,
MAX_ITEMS_PER_BOOL_EXPR);
n_clause_index =
maat_compile_get_hit_clause_index(compile, hit_path_array[idx].vtable_id,
hit_path_array[idx].top_group_id,
clause_index_array,
MAX_ITEMS_PER_BOOL_EXPR);
hit_path_array[idx].clause_index = clause_index_array[0];
if (n_clause_index > 1) {
for (i = 1; i < n_clause_index; i++) {
@@ -1662,8 +1704,9 @@ void populate_hit_path_with_compile(struct maat_hit_path *hit_path_array,
if (!maat_compile_is_hit_path_existed(hit_path_array, n_hit_path + new_hit_path_cnt, &tmp_path)) {
hit_path_array[n_hit_path + new_hit_path_cnt] = tmp_path;
new_hit_path_cnt++;
n_clause_index = maat_compile_get_hit_clause_index(compile, tmp_path.vtable_id, tmp_path.top_group_id,
clause_index_array, MAX_ITEMS_PER_BOOL_EXPR);
n_clause_index =
maat_compile_get_hit_clause_index(compile, tmp_path.vtable_id, tmp_path.top_group_id,
clause_index_array, MAX_ITEMS_PER_BOOL_EXPR);
hit_path_array[n_hit_path + new_hit_path_cnt - 1].clause_index = clause_index_array[0];
if (n_clause_index > 1) {
for (i = 1; i < n_clause_index; i++) {
@@ -1696,9 +1739,10 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr
return 0;
}
int bool_match_ret = bool_matcher_match(compile_rt->bm,
(unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
utarray_len(compile_state->all_hit_clauses), expr_match, MAX_HIT_COMPILE_NUM);
int bool_match_ret =
bool_matcher_match(compile_rt->bm,
(unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
utarray_len(compile_state->all_hit_clauses), expr_match, MAX_HIT_COMPILE_NUM);
for (int idx = 0; idx < bool_match_ret; idx++) {
compile = (struct maat_compile *)expr_match[idx].user_tag;
@@ -1718,7 +1762,8 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr
key.vtable_id = hit_path_array[j].vtable_id;
key.not_flag = hit_path_array[j].NOT_flag;
if (maat_compile_has_clause_query_key(compile, &key)) {
populate_hit_path_with_compile(hit_path_array, j, n_hit_path, &n_new_hit_path, compile);
populate_hit_path_with_compile(hit_path_array, j, n_hit_path,
&n_new_hit_path, compile);
}
}
}
@@ -1726,9 +1771,10 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr
return (n_hit_path + n_new_hit_path);
}
static void compile_state_add_direct_hit_groups(struct compile_state *compile_state,
struct maat_item *hit_items,
size_t n_hit_items, int vtable_id)
static void
compile_state_add_direct_hit_groups(struct compile_state *compile_state,
struct maat_item *hit_items,
size_t n_hit_items, int vtable_id)
{
if (NULL == compile_state || NULL == hit_items) {
return;
@@ -1743,9 +1789,10 @@ static void compile_state_add_direct_hit_groups(struct compile_state *compile_st
}
}
static void compile_state_add_indirect_hit_groups(struct compile_state *compile_state,
long long *group_ids,
size_t n_group_ids, int vtable_id)
static void
compile_state_add_indirect_hit_groups(struct compile_state *compile_state,
long long *group_ids,
size_t n_group_ids, int vtable_id)
{
if (NULL == compile_state || NULL == group_ids) {
return;
@@ -1760,8 +1807,9 @@ static void compile_state_add_indirect_hit_groups(struct compile_state *compile_
}
}
static void compile_state_add_hit_clauses(struct compile_state *compile_state,
UT_array *clause_id_array)
static void
compile_state_add_hit_clauses(struct compile_state *compile_state,
UT_array *clause_id_array)
{
size_t i = 0;
long long *clause_id = NULL;
@@ -1787,12 +1835,14 @@ static void compile_state_add_hit_clauses(struct compile_state *compile_state,
}
}
static void compile_state_add_exclude_not_clauses(struct compile_state *compile_state,
UT_array *clause_id_array)
static void
compile_state_add_exclude_not_clauses(struct compile_state *compile_state,
UT_array *clause_id_array)
{
for (size_t i = 0; i < utarray_len(clause_id_array); i++) {
long long *clause_id = (long long *)utarray_eltptr(clause_id_array, i);
if (utarray_find(compile_state->exclude_not_clauses, clause_id, compare_clause_id)) {
if (utarray_find(compile_state->exclude_not_clauses, clause_id,
compare_clause_id)) {
continue;
}
utarray_push_back(compile_state->exclude_not_clauses, clause_id);
@@ -1800,8 +1850,9 @@ static void compile_state_add_exclude_not_clauses(struct compile_state *compile_
utarray_sort(compile_state->exclude_not_clauses, compare_clause_id);
}
static void compile_state_add_hit_not_clauses(struct compile_state *compile_state,
UT_array *clause_id_array)
static void
compile_state_add_hit_not_clauses(struct compile_state *compile_state,
UT_array *clause_id_array)
{
size_t i = 0;
long long *clause_id = NULL;
@@ -1832,9 +1883,10 @@ static void compile_state_add_hit_not_clauses(struct compile_state *compile_stat
}
}
static void compile_state_update_hit_clauses(struct compile_state *compile_state,
struct compile_runtime *compile_rt,
long long group_id, int vtable_id)
static void
compile_state_update_hit_clauses(struct compile_state *compile_state,
struct compile_runtime *compile_rt,
long long group_id, int vtable_id)
{
if (NULL == compile_state || NULL == compile_rt) {
return;
@@ -1855,10 +1907,11 @@ static void compile_state_update_hit_clauses(struct compile_state *compile_state
}
}
static void compile_state_cache_hit_not_groups(struct compile_state *compile_state,
struct compile_runtime *compile_rt,
long long *hit_group_ids,
size_t n_hit_group_id, int vtable_id)
static void
compile_state_cache_hit_not_groups(struct compile_state *compile_state,
struct compile_runtime *compile_rt,
long long *hit_group_ids,
size_t n_hit_group_id, int vtable_id)
{
if (NULL == compile_state || NULL == compile_rt) {
return;
@@ -1889,8 +1942,9 @@ static void compile_state_cache_hit_not_groups(struct compile_state *compile_sta
continue;
}
long long *tmp_group_id = bsearch(&(clause_id_kv->key.group_id), hit_group_ids,
n_hit_group_id, sizeof(long long), compare_group_id);
long long *tmp_group_id =
bsearch(&(clause_id_kv->key.group_id), hit_group_ids,
n_hit_group_id, sizeof(long long), compare_group_id);
if (tmp_group_id != NULL) {
continue;
}
@@ -1902,7 +1956,8 @@ static void compile_state_cache_hit_not_groups(struct compile_state *compile_sta
HASH_ADD_INT(compile_state->hit_not_tbl_groups, vtable_id, tbl_group);
}
if (!utarray_find(tbl_group->group_ids, &(clause_id_kv->key.group_id), compare_group_id)) {
if (!utarray_find(tbl_group->group_ids, &(clause_id_kv->key.group_id),
compare_group_id)) {
utarray_push_back(tbl_group->group_ids, &(clause_id_kv->key.group_id));
}
}
@@ -1926,10 +1981,11 @@ int compile_state_get_compile_table_id(struct compile_state *compile_state,
return tmp->table_id;
}
static int compile_runtime_add_compile(struct compile_runtime *compile_rt,
struct compile_schema *schema,
long long compile_id, const char *table_name,
const char *line, struct log_handle *logger)
static int
compile_runtime_add_compile(struct compile_runtime *compile_rt,
struct compile_schema *schema,
long long compile_id, const char *table_name,
const char *line, struct log_handle *logger)
{
struct maat_compile *compile = NULL;
struct compile_item *compile_item = compile_item_new(line, schema, table_name,
@@ -1941,8 +1997,8 @@ static int compile_runtime_add_compile(struct compile_runtime *compile_rt,
int table_id = table_manager_get_table_id(schema->ref_tbl_mgr, table_name);
if (table_id < 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d]table_name:%s has invalid table_id:%d, drop line:%s", __FUNCTION__,
__LINE__, table_name, table_id, line);
"[%s:%d]table_name:%s has invalid table_id:%d, drop line:%s",
__FUNCTION__, __LINE__, table_name, table_id, line);
return -1;
}
@@ -2173,8 +2229,8 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
}
int ret = -1;
struct group2compile_item *g2c_item = group2compile_item_new(line, schema, table_name,
compile_rt->logger);
struct group2compile_item *g2c_item =
group2compile_item_new(line, schema, table_name, compile_rt->logger);
if (NULL == g2c_item) {
g2c_rt->update_err_cnt++;
return -1;
@@ -2229,7 +2285,9 @@ long long group2compile_runtime_not_clause_count(void *g2c_runtime)
return 0;
}
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
struct group2compile_runtime *g2c_rt =
(struct group2compile_runtime *)g2c_runtime;
return g2c_rt->not_clause_cnt;
}
@@ -2239,7 +2297,9 @@ long long group2compile_runtime_rule_count(void *g2c_runtime)
return 0;
}
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
struct group2compile_runtime *g2c_rt =
(struct group2compile_runtime *)g2c_runtime;
return g2c_rt->rule_num;
}
@@ -2249,7 +2309,9 @@ long long group2compile_runtime_update_err_count(void *g2c_runtime)
return 0;
}
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
struct group2compile_runtime *g2c_rt =
(struct group2compile_runtime *)g2c_runtime;
return g2c_rt->update_err_cnt;
}
@@ -2260,7 +2322,8 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name,
return -1;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
struct compile_runtime *compile_rt =
(struct compile_runtime *)compile_runtime;
int updating_flag = rcu_hash_is_updating(compile_rt->cfg_hash);
if (0 == updating_flag) {
@@ -2341,7 +2404,9 @@ long long compile_runtime_rule_count(void *compile_runtime)
return 0;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
struct compile_runtime *compile_rt =
(struct compile_runtime *)compile_runtime;
return compile_rt->rule_num;
}
@@ -2351,7 +2416,9 @@ long long compile_runtime_update_err_count(void *compile_runtime)
return 0;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
struct compile_runtime *compile_rt =
(struct compile_runtime *)compile_runtime;
return compile_rt->update_err_cnt;
}
@@ -2392,10 +2459,11 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile
struct compile_item *compile_items[compile_ids_size];
// all hit clause_id -> compile_id
size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt, compile_state,
state->thread_id,
(void **)compile_items,
compile_ids_size);
size_t bool_match_ret =
maat_compile_bool_matcher_match(compile_rt, compile_state,
state->thread_id,
(void **)compile_items,
compile_ids_size);
if (bool_match_ret > 0) {
qsort(compile_items, bool_match_ret, sizeof(struct compile_item *),
compare_compile_item);
@@ -2528,7 +2596,8 @@ size_t compile_state_get_indirect_hit_groups(struct compile_state *compile_state
size_t i = 0;
struct maat_hit_group *hit_group = NULL;
for (i = 0; i < utarray_len(compile_state->indirect_hit_groups) && i < array_size; i++) {
hit_group = (struct maat_hit_group *)utarray_eltptr(compile_state->indirect_hit_groups, i);
hit_group =
(struct maat_hit_group *)utarray_eltptr(compile_state->indirect_hit_groups, i);
group_array[i].item_id = hit_group->item_id;
group_array[i].group_id = hit_group->group_id;
group_array[i].vtable_id = hit_group->vtable_id;
@@ -2551,7 +2620,8 @@ size_t compile_state_get_last_hit_groups(struct compile_state *compile_state,
size_t i = 0;
for (i = 0; i < utarray_len(compile_state->last_hit_groups) && i < array_size; i++) {
group_array[i] = *(struct maat_hit_group *)utarray_eltptr(compile_state->last_hit_groups, i);
group_array[i] =
*(struct maat_hit_group *)utarray_eltptr(compile_state->last_hit_groups, i);
}
return i;
@@ -2597,7 +2667,8 @@ size_t compile_state_get_internal_hit_paths(struct compile_state *compile_state,
struct internal_hit_path *internal_path = NULL;
for (int i = 0; i < utarray_len(compile_state->internal_hit_paths); i++) {
internal_path = (struct internal_hit_path *)utarray_eltptr(compile_state->internal_hit_paths, i);
internal_path =
(struct internal_hit_path *)utarray_eltptr(compile_state->internal_hit_paths, i);
/*
NOTE: maybe one item has been deleted, but it's item_id still exist in internal_hit_paths
*/
@@ -2605,8 +2676,9 @@ size_t compile_state_get_internal_hit_paths(struct compile_state *compile_state,
UT_array *valid_super_group_ids;
utarray_new(valid_super_group_ids, &ut_compile_group_id_icd);
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, &(internal_path->group_id), 1,
super_group_ids, MAX_HIT_GROUP_NUM);
size_t super_group_cnt =
group2group_runtime_get_super_groups(g2g_rt, &(internal_path->group_id), 1,
super_group_ids, MAX_HIT_GROUP_NUM);
for (size_t idx = 0; idx < super_group_cnt; idx++) {
utarray_push_back(valid_super_group_ids, &super_group_ids[idx]);
}
@@ -2623,7 +2695,8 @@ size_t compile_state_get_internal_hit_paths(struct compile_state *compile_state,
long long *p = NULL;
struct maat_hit_path tmp_path;
for (p = utarray_front(valid_super_group_ids); p != NULL && hit_path_cnt < array_size;
for (p = utarray_front(valid_super_group_ids);
p != NULL && hit_path_cnt < array_size;
p = utarray_next(valid_super_group_ids, p)) {
memset(&tmp_path, 0, sizeof(tmp_path));
tmp_path.Nth_scan = internal_path->Nth_scan;
@@ -2638,7 +2711,8 @@ size_t compile_state_get_internal_hit_paths(struct compile_state *compile_state,
/* check if internal_path is duplicated from hit_path_array[]
* element */
if (hit_path_cnt > 0) {
if (maat_compile_is_hit_path_existed(hit_path_array, hit_path_cnt, &tmp_path)) {
if (maat_compile_is_hit_path_existed(hit_path_array, hit_path_cnt,
&tmp_path)) {
continue;
}
}

2
src/maat_plugin.c
View File

@@ -39,7 +39,6 @@ struct plugin_runtime {
struct log_handle *logger;
long long rule_num;
long long acc_line_num;
long long update_err_cnt;
long long *scan_times;
};
@@ -372,7 +371,6 @@ static int plugin_runtime_update_row(struct plugin_runtime *plugin_rt,
ex_container_free(plugin_rt->ex_data_rt, ex_container);
return -1;
}
plugin_rt->acc_line_num++;
}
}

1
src/maat_rule.c
View File

@@ -528,7 +528,6 @@ void *rule_monitor_loop(void *arg)
pthread_mutex_unlock(&(maat_inst->background_update_mutex));
maat_garbage_collect_routine(maat_inst->garbage_bin);
maat_plugin_table_garbage_collect_routine(maat_inst->tbl_mgr);

2
test/CMakeLists.txt
View File

@@ -13,7 +13,7 @@ include_directories(${PROJECT_SOURCE_DIR}/scanner/bool_matcher)
add_executable(rcu_hash_gtest rcu_hash_gtest.cpp)
target_link_libraries(rcu_hash_gtest maat_frame_static gtest_static)
add_executable(maat_input_mode_gtest maat_input_mode_gtest.cpp)
add_executable(maat_input_mode_gtest maat_input_mode_gtest.cpp test_utils.cpp)
target_link_libraries(maat_input_mode_gtest maat_frame_static gtest_static)
add_executable(maat_framework_gtest maat_framework_gtest.cpp test_utils.cpp)

7
test/benchmark/CMakeLists.txt
View File

@@ -1,7 +1,4 @@
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
add_executable(benchmark_hs_gtest benchmark_hs_gtest.cpp)
target_link_libraries(benchmark_hs_gtest maat_frame_static gtest_static)
add_executable(benchmark_rs_gtest benchmark_rs_gtest.cpp)
target_link_libraries(benchmark_rs_gtest maat_frame_static gtest_static)
add_executable(benchmark_gtest benchmark_gtest.cpp ../test_utils.cpp)
target_link_libraries(benchmark_gtest maat_frame_static gtest_static)

3111
test/benchmark/benchmark_hs_gtest.cpp → test/benchmark/benchmark_gtest.cpp
View File

File diff suppressed because it is too large Load Diff

3638
test/benchmark/benchmark_rs_gtest.cpp
View File

File diff suppressed because it is too large Load Diff

154
test/expr_matcher_gtest.cpp
View File

@@ -41,7 +41,7 @@ enum expr_case_sensitive case_sensitive_str_to_enum(const char *str)
return case_sensitive;
}
int is_hexbin_str_to_int(const char *str)
static int is_hexbin_str_to_int(const char *str)
{
int ret = 0;
@@ -94,7 +94,8 @@ enum expr_pattern_type pattern_type_str_to_enum(const char *str)
return pattern_type;
}
int parse_config_file(const char *filename, struct expr_rule exprs[], size_t *n_expr)
static int
parse_config_file(const char *filename, struct expr_rule exprs[], size_t *n_expr)
{
unsigned char *json_buff = NULL;
size_t json_buff_size = 0;
@@ -147,17 +148,20 @@ int parse_config_file(const char *filename, struct expr_rule exprs[], size_t *n_
cJSON *item = cJSON_GetObjectItem(pat_item, "pattern_type");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].type = pattern_type_str_to_enum(item->valuestring);
exprs[i].patterns[j].type =
pattern_type_str_to_enum(item->valuestring);
}
item = cJSON_GetObjectItem(pat_item, "match_method");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].match_mode = match_method_to_match_mode(item->valuestring);
exprs[i].patterns[j].match_mode =
match_method_to_match_mode(item->valuestring);
}
item = cJSON_GetObjectItem(pat_item, "case_sensitive");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].case_sensitive = case_sensitive_str_to_enum(item->valuestring);
exprs[i].patterns[j].case_sensitive =
case_sensitive_str_to_enum(item->valuestring);
}
int is_hexbin = 0;
@@ -191,9 +195,11 @@ int parse_config_file(const char *filename, struct expr_rule exprs[], size_t *n_
if (item != NULL && item->type == cJSON_String) {
int key_left_offset = -1;
int key_right_offset = -1;
sscanf(item->valuestring, "%d~%d", &key_left_offset, &key_right_offset);
sscanf(item->valuestring, "%d~%d", &key_left_offset,
&key_right_offset);
if (key_left_offset < -1 || key_right_offset < -1) {
printf("Error: offset should not less than -1, left_offset:%d, right_offset:%d\n",
printf("Error: offset should not less than -1, "
"left_offset:%d, right_offset:%d\n",
key_left_offset, key_right_offset);
}
exprs[i].patterns[j].start_offset = key_left_offset;
@@ -219,7 +225,7 @@ next:
return ret;
}
void expr_array_free(struct expr_rule rules[], size_t n_rule)
static void expr_array_free(struct expr_rule rules[], size_t n_rule)
{
for (size_t i = 0; i < n_rule; i++) {
for (size_t j = 0; j < rules[i].n_patterns; j++) {
@@ -236,7 +242,8 @@ TEST(hs_expr_matcher_init, invalid_input_parameter)
struct expr_rule rules[64];
size_t n_rule = 0;
struct expr_matcher *matcher = expr_matcher_new(NULL, 0, EXPR_ENGINE_TYPE_HS, 1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(NULL, 0, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher == NULL);
matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
@@ -261,7 +268,8 @@ TEST(rs_expr_matcher_init, invalid_input_parameter)
struct expr_rule rules[64];
size_t n_rule = 0;
struct expr_matcher *matcher = expr_matcher_new(NULL, 0, EXPR_ENGINE_TYPE_RS, 1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(NULL, 0, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher == NULL);
matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
@@ -289,7 +297,8 @@ TEST(hs_expr_matcher_match, literal_sub_has_normal_offset)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -345,7 +354,8 @@ TEST(rs_expr_matcher_match, literal_sub_has_normal_offset)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -401,8 +411,8 @@ TEST(hs_expr_matcher_match, literal_sub_has_left_unlimit_offset)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -459,8 +469,8 @@ TEST(rs_expr_matcher_match, literal_sub_has_left_unlimit_offset)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -517,8 +527,8 @@ TEST(hs_expr_matcher_match, literal_sub_has_right_unlimit_offset)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -584,8 +594,8 @@ TEST(rs_expr_matcher_match, literal_sub_has_right_unlimit_offset)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -651,8 +661,8 @@ TEST(hs_expr_matcher_match, literal_sub_with_no_offset)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -709,8 +719,8 @@ TEST(rs_expr_matcher_match, literal_sub_with_no_offset)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -767,8 +777,8 @@ TEST(hs_expr_matcher_match, literal_exactly)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -815,8 +825,8 @@ TEST(rs_expr_matcher_match, literal_exactly)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -864,8 +874,8 @@ TEST(hs_expr_matcher_match, literal_prefix)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -924,8 +934,8 @@ TEST(rs_expr_matcher_match, literal_prefix)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -984,8 +994,8 @@ TEST(hs_expr_matcher_match, literal_suffix)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -1044,8 +1054,8 @@ TEST(rs_expr_matcher_match, literal_suffix)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -1104,8 +1114,8 @@ TEST(hs_expr_matcher_match, literal_sub_with_hex)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -1143,8 +1153,8 @@ TEST(rs_expr_matcher_match, literal_sub_with_hex)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -1182,8 +1192,8 @@ TEST(hs_expr_matcher_match, literal_with_chinese)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -1211,8 +1221,8 @@ TEST(rs_expr_matcher_match, literal_with_chinese)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -1240,8 +1250,8 @@ TEST(hs_expr_matcher_match, same_pattern_different_offset)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -1269,8 +1279,8 @@ TEST(rs_expr_matcher_match, same_pattern_different_offset)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -1298,8 +1308,8 @@ TEST(hs_expr_matcher_match, long_scan_data)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -1329,8 +1339,8 @@ TEST(rs_expr_matcher_match, long_scan_data)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -1377,20 +1387,22 @@ TEST(hs_expr_matcher_stream, basic)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
const char* scan_data1 = "A directed path in a directed graph is a finite";
const char *scan_data2 = " or infinite sequence of edges which joins a sequence of distinct vertices";
const char *scan_data2 = " or infinite sequence of edges which joins a "
"sequence of distinct vertices";
struct expr_scan_result result[64] = {0};
size_t n_hit_result = 0;
size_t n_hit_pattern = 0;
int thread_id = 0;
struct expr_matcher_stream *stream = expr_matcher_stream_open(matcher, thread_id);
struct expr_matcher_stream *stream =
expr_matcher_stream_open(matcher, thread_id);
EXPECT_TRUE(stream != NULL);
ret = expr_matcher_stream_match(stream, scan_data1, strlen(scan_data1), result,
@@ -1419,13 +1431,14 @@ TEST(rs_expr_matcher_stream, basic)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
const char* scan_data1 = "A directed path in a directed graph is a finite";
const char *scan_data2 = " or infinite sequence of edges which joins a sequence of distinct vertices";
const char *scan_data2 = " or infinite sequence of edges which joins a "
"sequence of distinct vertices";
struct expr_scan_result result[64] = {0};
size_t n_hit_result = 0;
@@ -1464,11 +1477,13 @@ TEST(hs_expr_matcher, regex_basic)
ret = expr_matcher_verify_regex_expression("[0-9]rain", g_logger);
EXPECT_EQ(ret, 1);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
const char *scan_data1 = "http://www.cyberessays.com/search_results.php?action=search&query=username,abckkk,1234567";
const char *scan_data1 = "http://www.cyberessays.com/search_results.php?"
"action=search&query=username,abckkk,1234567";
//const char *scan_data2 = "8rain";
struct expr_scan_result result[64] = {0};
size_t n_result = 0;
@@ -1495,12 +1510,13 @@ TEST(rs_expr_matcher, regex_basic)
ret = expr_matcher_verify_regex_expression("[0-9]rain", g_logger);
EXPECT_EQ(ret, 1);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
const char *scan_data1 = "http://www.cyberessays.com/search_results.php?action=search&query=username,abckkk,1234567";
const char *scan_data1 = "http://www.cyberessays.com/search_results.php?"
"action=search&query=username,abckkk,1234567";
struct expr_scan_result result[64] = {0};
size_t n_result = 0;
@@ -1525,7 +1541,8 @@ TEST(hs_expr_matcher, regex_unicode)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -1552,7 +1569,8 @@ TEST(rs_expr_matcher, regex_unicode)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);
@@ -1579,8 +1597,8 @@ TEST(hs_expr_matcher, hit_pattern_num)
int ret = parse_config_file("./expr_matcher.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS,
1, g_logger);
struct expr_matcher *matcher =
expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger);
EXPECT_TRUE(matcher != NULL);
expr_array_free(rules, n_rule);

3
test/group_nesting/group_nesting_gtest.cpp
View File

@@ -33,7 +33,8 @@ struct group2group_rule {
UT_icd ut_ex_group_id_icd = {sizeof(long long), NULL, NULL, NULL};
static int parse_config_file(const char *filename, struct group2group_rule *rules)
static int
parse_config_file(const char *filename, struct group2group_rule *rules)
{
unsigned char *json_buff = NULL;
size_t json_buff_size = 0;

2
test/ipport_plugin/CMakeLists.txt
View File

@@ -1,4 +1,4 @@
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
add_executable(ipport_plugin_gtest ipport_plugin_gtest.cpp)
add_executable(ipport_plugin_gtest ipport_plugin_gtest.cpp ../test_utils.cpp)
target_link_libraries(ipport_plugin_gtest maat_frame_static gtest_static)

140
test/ipport_plugin/ipport_plugin_gtest.cpp
View File

@@ -4,6 +4,7 @@
#include "maat_rule.h"
#include "maat_utils.h"
#include "log/log.h"
#include "../test_utils.h"
#include "maat_config_monitor.h"
#include "maat_redis_monitor.h"
@@ -33,133 +34,6 @@ protected:
struct log_handle *IPPortPluginTable::logger;
int count_line_num_cb(const char *table_name, const char *line, void *u_para)
{
(*((unsigned int *)u_para))++;
return 0;
}
int line_idx = 0;
long long absolute_expire_time=0;
int make_serial_rule(const char *table_name, const char *line, void *u_para)
{
struct serial_rule *s_rule=(struct serial_rule *)u_para;
redisContext *ctx = s_rule->ref_ctx;
char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1);
while(buff[strlen(line) - 1] == '\n' || buff[strlen(line) - 1] == '\t') {
buff[strlen(line) - 1] = '\0';
}
const char *redis_rule_key = "TEST_RULE_KEY";
redisReply *reply =
maat_wrap_redis_command(ctx, NULL, "INCRBY %s %d", redis_rule_key, 1);
if (reply->type == REDIS_REPLY_NIL) {
printf("incrby redis_rule_key:%s failed.", redis_rule_key);
return -1;
} else {
s_rule->rule_id = maat_read_redis_integer(reply);
freeReplyObject(reply);
reply = NULL;
}
maat_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, s_rule->rule_id,
table_name, buff, absolute_expire_time);
(s_rule + line_idx)->ref_ctx = ctx;
line_idx++;
FREE(buff);
return 0;
}
int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
struct log_handle *logger)
{
redisContext *c =
maat_connect_redis(redis_ip, redis_port, redis_db, logger);
if (NULL == c) {
return -1;
}
redisReply *reply = maat_wrap_redis_command(c, logger, "flushdb");
if (NULL == reply) {
return -1;
} else {
freeReplyObject(reply);
reply = NULL;
}
reply = maat_wrap_redis_command(c, logger, "SET MAAT_VERSION 1");
if (NULL == reply) {
return -1;
} else {
freeReplyObject(reply);
reply = NULL;
}
reply = maat_wrap_redis_command(c, logger, "SET MAAT_PRE_VER 1");
if (NULL == reply) {
return -1;
} else {
freeReplyObject(reply);
reply = NULL;
}
reply = maat_wrap_redis_command(c, logger, "SET SEQUENCE_REGION 1");
if (NULL == reply) {
return -1;
} else {
freeReplyObject(reply);
reply = NULL;
}
reply = maat_wrap_redis_command(c, logger, "SET SEQUENCE_GROUP 1");
if (NULL == reply) {
return -1;
} else {
freeReplyObject(reply);
reply = NULL;
}
size_t total_line_cnt = 0;
char iris_idx_path[PATH_MAX] = {0};
snprintf(iris_idx_path, sizeof(iris_idx_path), "test_data/index");
config_monitor_traverse(0, iris_idx_path, NULL, count_line_num_cb, NULL,
&total_line_cnt, NULL, logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
s_rule->ref_ctx = c;
long long server_time = maat_redis_server_time_s(c);
if (server_time < 0) {
return -1;
}
absolute_expire_time = server_time + 300;
config_monitor_traverse(0, iris_idx_path, NULL, make_serial_rule, NULL,
s_rule, NULL, logger);
s_rule->ref_ctx = NULL;
line_idx = 0;
absolute_expire_time = 0;
int success_cnt = 0;
do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt,
server_time, logger);
} while (success_cnt < 0);
EXPECT_EQ(success_cnt, (int)total_line_cnt);
for (size_t i = 0; i < total_line_cnt; i++) {
maat_clear_rule_cache(s_rule + i);
}
FREE(s_rule);
redisFree(c);
return 0;
}
struct ipport_plugin_ud {
long long rule_id;
char *buffer;
@@ -365,9 +239,13 @@ TEST_F(IPPortPluginTable, WITHOUT_SAME_IP) {
char redis_ip[32] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
char iris_idx_path[PATH_MAX] = {0};
struct log_handle *logger = IPPortPluginTable::logger;
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
snprintf(iris_idx_path, sizeof(iris_idx_path), "test_data/index");
int ret = write_iris_to_redis(iris_idx_path, redis_ip, redis_port,
redis_db, logger);
assert(ret == 0);
struct maat_options *opts = maat_options_new();
@@ -434,9 +312,13 @@ TEST_F(IPPortPluginTable, WITH_256SAME_IP) {
char redis_ip[32] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
char iris_idx_path[PATH_MAX] = {0};
struct log_handle *logger = IPPortPluginTable::logger;
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
snprintf(iris_idx_path, sizeof(iris_idx_path), "test_data/index");
int ret = write_iris_to_redis(iris_idx_path, redis_ip, redis_port,
redis_db, logger);
assert(ret == 0);
struct maat_options *opts = maat_options_new();

35
test/maat_ex_data_gtest.cpp
View File

@@ -20,8 +20,9 @@ struct user_info {
int id;
};
void ex_data_new_cb(const char *table_name, int table_id, const char *key, const char *table_line,
void **ad, long argl, void *argp)
static void
ex_data_new_cb(const char *table_name, int table_id, const char *key,
const char *table_line, void **ad, long argl, void *argp)
{
int *counter = (int *)argp;
struct user_info *u = ALLOC(struct user_info, 1);
@@ -33,7 +34,7 @@ void ex_data_new_cb(const char *table_name, int table_id, const char *key, const
(*counter)++;
}
void ex_data_free_cb(int table_id, void **ad, long argl, void *argp)
static void ex_data_free_cb(int table_id, void **ad, long argl, void *argp)
{
struct user_info *u = (struct user_info *)(*ad);
@@ -42,7 +43,8 @@ void ex_data_free_cb(int table_id, void **ad, long argl, void *argp)
*ad = NULL;
}
void ex_data_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
static void
ex_data_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
{
struct user_info *u = (struct user_info *)(*from);
*to = u;
@@ -70,7 +72,8 @@ TEST(EXDataRuntime, Update) {
const char *row1 = "101\t192.168.0.1\tmahuateng";
const char *key1 = "192.168.0.1";
size_t key1_len = strlen(key1);
void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, table_name, row1, key1, key1_len);
void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, table_name, row1,
key1, key1_len);
EXPECT_EQ(ex_data_counter, 1);
struct ex_container *ex_container = ex_container_new(ex_data, NULL);
@@ -130,27 +133,7 @@ int main(int argc, char ** argv)
::testing::InitGoogleTest(&argc, argv);
g_logger = log_handle_create("./maat_ex_data_gtest.log", 0);
char json_iris_path[NAME_MAX] = {0};
char tmp_iris_path[PATH_MAX] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", g_json_filename);
if ((access(json_iris_path, F_OK)) == 0) {
system_cmd_rmdir(json_iris_path);
}
if (access(json_iris_path, F_OK) < 0) {
char *json_buff = NULL;
size_t json_buff_sz = 0;
int ret = load_file_to_memory(g_json_filename, (unsigned char**)&json_buff, &json_buff_sz);
EXPECT_NE(ret, -1);
ret = json2iris(json_buff, g_json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path),
NULL, NULL, g_logger);
FREE(json_buff);
EXPECT_NE(ret, -1);
}
struct maat_options *opts = maat_options_new();
char json_path[PATH_MAX] = {0};
snprintf(json_path, sizeof(json_path), "./%s", g_json_filename);

13
test/maat_framework_gtest.cpp
View File

@@ -58,19 +58,6 @@ int del_command(struct maat *maat_inst, int compile_id)
compile_id, "null", 1, 0);
}
static void random_keyword_generate(char *keyword_buf, size_t sz)
{
#define MIN_KEYWORD_LEN 4
size_t i = 0;
size_t len = random() % (sz - 1 - MIN_KEYWORD_LEN) + MIN_KEYWORD_LEN;
for (i = 0; i < len; i++) {
keyword_buf[i] = 'a' + random() % ('z' - 'a');
}
keyword_buf[i] = '\0';
}
const char *watched_json = "./json_update/maat.json";
const char *old_json = "./json_update/old.json";
const char *new_json = "./json_update/new.json";

85
test/maat_framework_perf_gtest.cpp
View File

@@ -30,19 +30,6 @@ struct thread_param {
struct log_handle *logger;
};
void random_keyword_generate(char *keyword_buf, size_t sz)
{
#define MIN_KEYWORD_LEN 4
size_t i = 0;
size_t len = random() % (sz - 1 - MIN_KEYWORD_LEN) + MIN_KEYWORD_LEN;
for (i = 0; i < len; i++) {
keyword_buf[i] = 'a' + random() % ('z' - 'a');
}
keyword_buf[i] = '\0';
}
static void
test_add_expr_command(struct maat *maat_inst, const char *table_name,
const char *keywords)
@@ -1644,78 +1631,6 @@ TEST_F(MaatPerfFileScan, IPPlugin) {
}
}
class MaatTSGFqdnScan : public testing::Test
{
protected:
static void SetUpTestCase() {
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
const char *tsg_table_info = "./tsg_table_info.conf";
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_iris_to_redis("./tsgrule", redis_ip, redis_port,
redis_db, logger);
if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.",
__FUNCTION__, __LINE__);
}
struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
LOG_LEVEL_INFO);
_shared_maat_inst = maat_new(opts, tsg_table_info);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatTSGFqdnScan failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_inst);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_inst;
};
struct maat *MaatTSGFqdnScan::_shared_maat_inst;
struct log_handle *MaatTSGFqdnScan::logger;
#if 0
TEST_F(MaatTSGFqdnScan, mem_used) {
char keyword_buf[128];
const char *table_name1 = "TSG_OBJ_FQDN";
const char *table_name2= "TSG_IP_ADDR";
struct maat *maat_inst = MaatTSGFqdnScan::_shared_maat_inst;
struct log_handle *logger = MaatTSGFqdnScan::logger;
for (size_t i = 0; i < 100; i++) {
random_keyword_generate(keyword_buf, sizeof(keyword_buf));
int ret = expr_table_set_line(maat_inst, table_name1, MAAT_OP_ADD,
600000000+i, 600000000+i, keyword_buf,
"null", 1, 0, 0, 0);
EXPECT_EQ(ret, 1);
ret = ip_table_set_line(maat_inst, table_name2, MAAT_OP_ADD,
600000000+i, 600000000+i, IPv4, "100.64.1.1",
"100.64.1.2", 0);
EXPECT_EQ(ret, 1);
log_info(logger, MODULE_FRAMEWORK_PERF_GTEST, "MaatTSGFqdnScan update one line");
sleep(150);
}
}
#endif
class MaatPerfIPPortPluginScan : public testing::Test
{
protected:

224
test/maat_input_mode_gtest.cpp
View File

@@ -2,6 +2,7 @@
#include "maat_utils.h"
#include "maat_rule.h"
#include "json2iris.h"
#include "test_utils.h"
#include "maat_table.h"
#include "maat_config_monitor.h"
#include "maat_redis_monitor.h"
@@ -15,29 +16,26 @@ const char *g_json_filename = "maat_json.json";
struct log_handle *g_logger = NULL;
TEST(json_mode, maat_scan_string) {
char tmp_iris_path[PATH_MAX] = {0};
char iris_idx_path[PATH_MAX] = {0};
char json_iris_path[PATH_MAX] = {0};
snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", g_json_filename);
char json_path[PATH_MAX] = {0};
long long results[5] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
char scan_data[128] = "string1, string2, string3, string4, "
"string5, string6, string7, string8";
const char *table_name = "KEYWORDS_TABLE";
if (access(json_iris_path, F_OK) < 0) {
char *json_buff = NULL;
size_t json_buff_sz = 0;
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp",
g_json_filename);
snprintf(iris_idx_path, sizeof(iris_idx_path), "%s/index", json_iris_path);
snprintf(json_path, sizeof(json_path), "./%s", g_json_filename);
int ret = load_file_to_memory(g_json_filename,
(unsigned char**)&json_buff,
&json_buff_sz);
EXPECT_NE(ret, -1);
ret = json2iris(json_buff, g_json_filename, NULL, tmp_iris_path,
sizeof(tmp_iris_path), NULL, NULL, g_logger);
FREE(json_buff);
EXPECT_NE(ret, -1);
}
int ret = write_json_to_iris(g_json_filename, iris_idx_path,
sizeof(iris_idx_path), g_logger);
assert(ret == 0);
struct maat_options *opts = maat_options_new();
char json_path[PATH_MAX] = {0};
snprintf(json_path, sizeof(json_path), "./%s", g_json_filename);
maat_options_set_json_file(opts, json_path);
maat_options_set_stat_on(opts);
maat_options_set_perf_on(opts);
@@ -45,188 +43,76 @@ TEST(json_mode, maat_scan_string) {
struct maat *maat_inst = maat_new(opts, g_table_info_path);
EXPECT_TRUE(maat_inst != NULL);
const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_get_table_id(maat_inst, table_name);
char scan_data[128] =
"string1, string2, string3, string4, string5, string6, string7, string8";
long long results[5] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, sizeof(results), &n_hit_result, state);
ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, sizeof(results), &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 182);
maat_options_free(opts);
maat_state_free(state);
state = NULL;
maat_free(maat_inst);
}
TEST(iris_mode, maat_scan_string) {
char tmp_iris_path[512] = {0};
char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", g_json_filename);
char iris_idx_path[PATH_MAX] = {0};
long long results[5] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
const char *table_name = "KEYWORDS_TABLE";
char scan_data[128] = "string1, string2, string3, string4, "
"string5, string6, string7, string8";
if (access(json_iris_path, F_OK) < 0) {
char *json_buff = NULL;
size_t json_buff_sz = 0;
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp",
g_json_filename);
snprintf(iris_idx_path, sizeof(iris_idx_path), "%s/index", json_iris_path);
int ret = load_file_to_memory(g_json_filename,
(unsigned char**)&json_buff,
&json_buff_sz);
EXPECT_NE(ret, -1);
ret = json2iris(json_buff, g_json_filename, NULL, tmp_iris_path,
sizeof(tmp_iris_path), NULL, NULL, g_logger);
FREE(json_buff);
EXPECT_NE(ret, -1);
}
char tmp_iris_full_idx_path[PATH_MAX] = {0};
char tmp_iris_inc_idx_path[PATH_MAX] = {0};
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path),
"%s/index", json_iris_path);
snprintf(tmp_iris_inc_idx_path, sizeof(tmp_iris_inc_idx_path),
"%s/index", json_iris_path);
int ret = write_json_to_iris(g_json_filename, iris_idx_path,
sizeof(iris_idx_path), g_logger);
assert(ret == 0);
struct maat_options *opts = maat_options_new();
maat_options_set_stat_on(opts);
maat_options_set_perf_on(opts);
maat_options_set_iris(opts, tmp_iris_full_idx_path, tmp_iris_inc_idx_path);
maat_options_set_iris(opts, iris_idx_path, iris_idx_path);
maat_options_set_logger(opts, "./maat_input_mode_gtest.log", LOG_LEVEL_INFO);
struct maat *maat_inst = maat_new(opts, g_table_info_path);
EXPECT_TRUE(maat_inst != NULL);
const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_get_table_id(maat_inst, table_name);
char scan_data[128] =
"string1, string2, string3, string4, string5, string6, string7, string8";
long long results[5] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, sizeof(results), &n_hit_result, state);
ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, sizeof(results), &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 182);
maat_options_free(opts);
maat_state_free(state);
state = NULL;
maat_free(maat_inst);
}
int count_line_num_cb(const char *table_name, const char *line, void *u_para)
{
(*((unsigned int *)u_para))++;
return 0;
}
int line_idx = 0;
long long absolute_expire_time=0;
int make_serial_rule(const char *table_name, const char *line, void *u_para)
{
struct serial_rule *s_rule=(struct serial_rule *)u_para;
redisContext *ctx = s_rule->ref_ctx;
char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1);
while(buff[strlen(line) - 1] == '\n' || buff[strlen(line) - 1] == '\t') {
buff[strlen(line) - 1] = '\0';
}
const char *redis_rule_key = "TEST_RULE_KEY";
redisReply *reply =
maat_wrap_redis_command(ctx, NULL, "INCRBY %s %d", redis_rule_key, 1);
if (reply->type == REDIS_REPLY_NIL) {
printf("incrby redis_rule_key:%s failed.", redis_rule_key);
return -1;
} else {
s_rule->rule_id = maat_read_redis_integer(reply);
freeReplyObject(reply);
reply = NULL;
}
maat_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, s_rule->rule_id,
table_name, buff, absolute_expire_time);
(s_rule + line_idx)->ref_ctx = ctx;
line_idx++;
FREE(buff);
return 0;
}
TEST(redis_mode, maat_scan_string) {
char json_iris_path[512] = {0};
char redis_ip[64] = "127.0.0.1";
char redis_ip[32] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
const char *table_name = "KEYWORDS_TABLE";
long long results[5] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
char scan_data[128] = "string1, string2, string3, string4, "
"string5, string6, string7, string8";
snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", g_json_filename);
redisContext *c =
maat_connect_redis(redis_ip, redis_port, redis_db, g_logger);
EXPECT_TRUE(c != NULL);
redisReply *reply = maat_wrap_redis_command(c, NULL, "flushdb");
EXPECT_TRUE(reply != NULL);
freeReplyObject(reply);
reply = NULL;
if (access(json_iris_path, F_OK) < 0) {
char tmp_iris_path[128] = {0};
char *json_buff = NULL;
size_t json_buff_sz = 0;
int ret = load_file_to_memory(g_json_filename,
(unsigned char **)&json_buff,
&json_buff_sz);
EXPECT_NE(ret, -1);
ret = json2iris(json_buff, g_json_filename, c, tmp_iris_path,
sizeof(tmp_iris_path), NULL, NULL, g_logger);
FREE(json_buff);
EXPECT_NE(ret, -1);
}
size_t total_line_cnt = 0;
char tmp_iris_full_idx_path[PATH_MAX] = {0};
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path),
"%s/index", json_iris_path);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb,
NULL, &total_line_cnt, NULL, g_logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
s_rule->ref_ctx = c;
long long server_time = maat_redis_server_time_s(c);
EXPECT_NE(server_time, -1);
absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule,
NULL, s_rule, NULL, g_logger);
s_rule->ref_ctx = NULL;
int success_cnt = 0;
do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt,
server_time, g_logger);
} while (success_cnt < 0);
EXPECT_EQ(success_cnt, (int)total_line_cnt);
for (size_t i = 0; i < total_line_cnt; i++) {
maat_clear_rule_cache(s_rule + i);
}
FREE(s_rule);
redisFree(c);
int ret = write_json_to_redis(g_json_filename, redis_ip, redis_port,
redis_db, g_logger);
assert(ret == 0);
struct maat_options *opts = maat_options_new();
maat_options_set_stat_on(opts);
@@ -236,23 +122,19 @@ TEST(redis_mode, maat_scan_string) {
LOG_LEVEL_INFO);
struct maat *maat_inst = maat_new(opts, g_table_info_path);
const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_get_table_id(maat_inst, table_name);
char scan_data[128] =
"string1, string2, string3, string4, string5, string6, string7, string8";
long long results[5] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
assert(maat_inst != NULL);
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, sizeof(results), &n_hit_result, state);
int table_id = maat_get_table_id(maat_inst, table_name);
ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, sizeof(results), &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 182);
maat_options_free(opts);
maat_state_free(state);
state = NULL;
maat_free(maat_inst);
}

87
test/test_utils.cpp
View File

@@ -9,28 +9,34 @@
#include <assert.h>
#include <unistd.h>
int count_line_num_cb(const char *table_name, const char *line, void *u_para)
int line_idx = 0;
long long absolute_expire_time = 0;
static int
count_line_num_cb(const char *table_name, const char *line, void *u_para)
{
(*((unsigned int *)u_para))++;
return 0;
}
int line_idx = 0;
long long absolute_expire_time = 0;
int make_serial_rule(const char *table_name, const char *line, void *u_para)
static int
make_serial_rule(const char *table_name, const char *line, void *u_para)
{
struct serial_rule *s_rule=(struct serial_rule *)u_para;
redisContext *ctx = s_rule->ref_ctx;
char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1);
while (buff[strlen(line) - 1] == '\n' || buff[strlen(line) - 1] == '\t') {
while (buff[strlen(line) - 1] == '\n' ||
buff[strlen(line) - 1] == '\t') {
buff[strlen(line) - 1] = '\0';
}
const char *redis_rule_key = "TEST_RULE_KEY";
redisReply *reply = maat_wrap_redis_command(ctx, NULL, "INCRBY %s %d",
redis_rule_key, 1);
redisReply *reply =
maat_wrap_redis_command(ctx, NULL, "INCRBY %s %d", redis_rule_key, 1);
if (reply->type == REDIS_REPLY_NIL) {
printf("incrby redis_rule_key:%s failed.", redis_rule_key);
return -1;
@@ -49,29 +55,32 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
return 0;
}
int write_json_to_iris(const char* json_fn, char *iris_path, size_t path_sz, struct log_handle *logger)
int write_json_to_iris(const char* json_fn, char *iris_path, size_t path_sz,
struct log_handle *logger)
{
char *json_buff = NULL;
size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_fn, (unsigned char **)&json_buff,
&json_buff_sz);
&json_buff_sz);
if (ret < 0) {
return -1;
}
ret = json2iris(json_buff, json_fn, NULL, iris_path,
path_sz, NULL, NULL, logger);
ret = json2iris(json_buff, json_fn, NULL, iris_path, path_sz, NULL, NULL, logger);
FREE(json_buff);
if (ret < 0) {
return -1;
}
return 0;
}
int write_iris_to_redis(const char* iris_path, char *redis_ip, int redis_port, int redis_db,
struct log_handle *logger)
int write_iris_to_redis(const char *iris_path, char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger)
{
redisContext *c = maat_connect_redis(redis_ip, redis_port, redis_db, logger);
if (NULL == c) {
@@ -119,12 +128,21 @@ int write_iris_to_redis(const char* iris_path, char *redis_ip, int redis_port, i
redisFree(c);
return 0;
}
int write_json_to_redis(const char* json_filename, char *redis_ip, int redis_port, int redis_db,
struct log_handle *logger)
int write_json_to_redis(const char *json_filename, char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger)
{
char iris_path[512] = {0};
write_json_to_iris(json_filename, iris_path, sizeof(iris_path), logger);
write_iris_to_redis(iris_path, redis_ip, redis_port, redis_db, logger);
int ret = write_json_to_iris(json_filename, iris_path, sizeof(iris_path), logger);
if (ret < 0) {
return -1;
}
ret = write_iris_to_redis(iris_path, redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
return -1;
}
return 0;
}
@@ -147,7 +165,8 @@ int compile_table_set_line(struct maat *maat_inst, const char *table_name,
return maat_cmd_set_line(maat_inst, &line_rule);
}
#define TO_GROUP2X_KEY(group_id, parent_id, clause_index) (((unsigned long)group_id<<32|parent_id) + clause_index)
#define TO_GROUP2X_KEY(group_id, parent_id, clause_index) \
(((unsigned long)group_id<<32|parent_id) + clause_index)
int group2compile_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long group_id,
@@ -197,9 +216,10 @@ int expr_table_set_line(struct maat *maat_inst, const char *table_name,
return 0;
}
enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr,
table_id);
assert(table_type == TABLE_TYPE_EXPR || table_type == TABLE_TYPE_EXPR_PLUS);
enum table_type table_type =
table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
assert(table_type == TABLE_TYPE_EXPR ||
table_type == TABLE_TYPE_EXPR_PLUS);
if (table_type == TABLE_TYPE_EXPR_PLUS) {
sprintf(table_line, "%lld\t%lld\t%s\t%s\t%d\t%d\t%d\t%d",
@@ -231,9 +251,10 @@ int interval_table_set_line(struct maat *maat_inst, const char *table_name,
return 0;
}
enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr,
table_id);
assert(table_type == TABLE_TYPE_INTERVAL || table_type == TABLE_TYPE_INTERVAL_PLUS);
enum table_type table_type =
table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
assert(table_type == TABLE_TYPE_INTERVAL ||
table_type == TABLE_TYPE_INTERVAL_PLUS);
if (table_type == TABLE_TYPE_INTERVAL_PLUS) {
sprintf(table_line, "%lld\t%lld\t%s\t%u\t%u\t%d",
@@ -254,8 +275,8 @@ int interval_table_set_line(struct maat *maat_inst, const char *table_name,
int ip_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long item_id,
long long group_id, enum IP_TYPE type, const char *ip1,
const char *ip2, int expire_after)
long long group_id, enum IP_TYPE type,
const char *ip1, const char *ip2, int expire_after)
{
char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_inst, table_name);
@@ -300,4 +321,16 @@ int flag_table_set_line(struct maat *maat_inst, const char *table_name,
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_inst, &line_rule);
}
}
void random_keyword_generate(char *keyword_buf, size_t sz)
{
#define MIN_KEYWORD_LEN 4
size_t i = 0, len = 0;
len = random() % (sz - 1 - MIN_KEYWORD_LEN) + MIN_KEYWORD_LEN;
for (i = 0; i < len; i++) {
keyword_buf[i] = 'a' + random() % ('z' - 'a');
}
keyword_buf[i] = '\0';
}

38
test/test_utils.h
View File

@@ -1,37 +1,53 @@
#pragma once
#include "maat_utils.h"
#include "maat_command.h"
#include "ip_matcher.h"
int write_json_to_redis(const char* json_filename, char *redis_ip, int redis_port, int redis_db,
struct log_handle *logger);
int write_iris_to_redis(const char* iris_path, char *redis_ip, int redis_port, int redis_db,
struct log_handle *logger);
#include "ip_matcher/ip_matcher.h"
int write_json_to_redis(const char *json_filename, char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger);
int write_iris_to_redis(const char *iris_path, char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger);
int write_json_to_iris(const char* json_fn, char *iris_path, size_t path_sz,
struct log_handle *logger);
int compile_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long compile_id,
const char *user_region, int clause_num,
int expire_after);
int group2compile_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long group_id,
long long compile_id, int not_flag,
const char *vtable_name, int clause_index,
int expire_after);
int group2group_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long group_id,
long long sub_group_id, int expire_after);
int expr_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long item_id,
long long group_id, const char *keywords,
const char *district, int expr_type,
int match_method, int is_hexbin, int expire_after);
int interval_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long item_id, long long group_id,
unsigned int low_boundary, unsigned int up_boundary,
const char *district, int expire_after);
enum maat_operation op, long long item_id,
long long group_id, unsigned int low_boundary,
unsigned int up_boundary, const char *district,
int expire_after);
int ip_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long item_id,
long long group_id, enum IP_TYPE type, const char *ip1,
const char *ip2, int expire_after);
int flag_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long item_id,
long long group_id, long long flag,
long long flag_mask, int expire_after);
enum maat_operation op, long long item_id,
long long group_id, long long flag,
long long flag_mask, int expire_after);
void random_keyword_generate(char *keyword_buf, size_t sz);

3
tools/CMakeLists.txt
View File

@@ -2,8 +2,9 @@ include_directories(/opt/MESA/include/MESA/)
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
include_directories(${PROJECT_SOURCE_DIR}/deps)
include_directories(${PROJECT_SOURCE_DIR}/scanner)
include_directories(${PROJECT_SOURCE_DIR}/test)
add_executable(maat_redis_tool maat_redis_tool.cpp)
add_executable(maat_redis_tool maat_redis_tool.cpp ../test/test_utils.cpp)
target_link_libraries(maat_redis_tool maat_frame_static)
install(TARGETS maat_redis_tool DESTINATION /usr/local/bin/ COMPONENT TOOLS)

333
tools/maat_redis_tool.cpp
View File

@@ -12,14 +12,19 @@
#include "maat_config_monitor.h"
#include "maat_redis_monitor.h"
#include "json2iris.h"
#include "../test/test_utils.h"
#include "hiredis/hiredis.h"
#define MODULE_REDIS_TOOL module_name_str("maat.redis_tool")
#define WORK_MODE_DUMP 0
#define WORK_MODE_UPLOAD 1
#define WORK_MODE_TEST_TRANS 2
const char *redis_dump_dir = "./redis_dump";
const char *default_table_info = "./table_info.conf";
void maat_tool_print_usage(void)
static void maat_tool_print_usage(void)
{
printf("maat_redis_tool manipulate rules from redis.\n");
printf("Usage:\n");
@@ -30,7 +35,6 @@ void maat_tool_print_usage(void)
printf("\t-k, try to execute the 'keys EFFECTIVE_RULE:*' transaction, and give a specific reason if an error occurs\n");
printf("\t-u [json_file], flush redis and upload all rules to redis, confirm the risk before proceeding\n");
printf("\t-v [version], dump specific [version] from redis, dump latest version as default.\n");
printf("\t-j [payload.json], add or delete rules as maat json. Must have field compile_table field, and plugin table's valid flag must be in the last column.\n");
printf("\t-t [timeout], timeout config after t seconds, default is 0 which means never timeout.\n");
printf("example: ./maat_redis_tool -h 127.0.0.1 -p 6379 -d %s\n",redis_dump_dir);
printf(" ./maat_redis_tool -h 127.0.0.1 -p 6379 -k\n");
@@ -50,9 +54,10 @@ static int compare_serial_rule(const void *a, const void *b)
return ret;
}
int set_file_rulenum(const char *path, int rule_num)
static int set_file_rulenum(const char *path, int rule_num)
{
FILE* fp=NULL;
if (0 == rule_num) {
fp = fopen(path, "w");
} else {
@@ -60,7 +65,8 @@ int set_file_rulenum(const char *path, int rule_num)
}
if (NULL == fp) {
fprintf(stderr, "fopen %s failed %s at set rule num\n", path, strerror(errno));
fprintf(stderr, "fopen %s failed %s at set rule num\n",
path, strerror(errno));
return -1;
}
@@ -70,7 +76,8 @@ int set_file_rulenum(const char *path, int rule_num)
return 0;
}
void read_rule_from_redis(redisContext *c, const char *output_path, struct log_handle *logger)
static void read_rule_from_redis(redisContext *c, const char *output_path,
struct log_handle *logger)
{
int i = 0;
int ret = 0;
@@ -85,7 +92,8 @@ void read_rule_from_redis(redisContext *c, const char *output_path, struct log_h
FILE *index_fp = NULL;
struct serial_rule *rule_list = NULL;
int rule_num = maat_get_rm_key_list(c, 0, &version, NULL, &rule_list, &update_type, logger);
int rule_num = maat_get_rm_key_list(c, 0, &version, NULL, &rule_list,
&update_type, logger);
if (0 == rule_num) {
printf("No Effective Rules.\n");
return;
@@ -110,13 +118,15 @@ void read_rule_from_redis(redisContext *c, const char *output_path, struct log_h
printf("Sorting.\n");
qsort(rule_list, rule_num, sizeof(struct serial_rule), compare_serial_rule);
if ((access(output_path, F_OK)) <0) {
if ((mkdir(output_path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) < 0) {
printf("mkdir %s error\n", output_path);
}
}
snprintf(foreign_files_dir, sizeof(foreign_files_dir), "%s/foreign_files/",output_path);
snprintf(foreign_files_dir, sizeof(foreign_files_dir),
"%s/foreign_files/", output_path);
if ((access(foreign_files_dir, F_OK)) <0) {
if((mkdir(foreign_files_dir, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) < 0) {
@@ -124,13 +134,16 @@ void read_rule_from_redis(redisContext *c, const char *output_path, struct log_h
}
}
ret = maat_get_foreign_keys_by_prefix(c, rule_list, rule_num, foreign_files_dir, logger);
ret = maat_get_foreign_keys_by_prefix(c, rule_list, rule_num,
foreign_files_dir, logger);
if (ret > 0) {
printf("%d lines has foreign content.\n", ret);
maat_get_foreign_conts(c, rule_list, rule_num, 1, NULL);
}
snprintf(index_path,sizeof(index_path), "%s/full_config_index.%020lld", output_path, version);
snprintf(index_path,sizeof(index_path), "%s/full_config_index.%020lld",
output_path, version);
index_fp = fopen(index_path, "w");
if (NULL == index_fp) {
printf("Open %s failed.\n", index_path);
@@ -142,7 +155,8 @@ void read_rule_from_redis(redisContext *c, const char *output_path, struct log_h
maat_rewrite_table_line_with_foreign(rule_list+i);
}
if (NULL == cur_table || 0 != strcmp(cur_table,rule_list[i].table_name)) {
if (NULL == cur_table ||
0 != strcmp(cur_table,rule_list[i].table_name)) {
if (table_fp != NULL) {
fprintf(index_fp, "%s\t%d\t%s\n", cur_table, line_count, table_path);
fclose(table_fp);
@@ -152,8 +166,8 @@ void read_rule_from_redis(redisContext *c, const char *output_path, struct log_h
printf("Written table %s\n",table_path);
}
snprintf(table_path, sizeof(table_path), "%s/%s.%020lld", output_path,
rule_list[i].table_name, version);
snprintf(table_path, sizeof(table_path), "%s/%s.%020lld",
output_path, rule_list[i].table_name, version);
set_file_rulenum(table_path, 0);
table_fp = fopen(table_path, "a");
@@ -170,7 +184,8 @@ void read_rule_from_redis(redisContext *c, const char *output_path, struct log_h
rule_list[i].table_line[line_len - 1] = '\0';
}
fprintf(table_fp, "%s\tkey=%lld\n", rule_list[i].table_line, rule_list[i].rule_id);
fprintf(table_fp, "%s\tkey=%lld\n", rule_list[i].table_line,
rule_list[i].rule_id);
line_count++;
}
@@ -189,10 +204,6 @@ clean_up:
FREE(rule_list);
if (c != NULL) {
redisFree(c);
}
if (index_fp != NULL) {
fclose(index_fp);
}
@@ -202,195 +213,7 @@ clean_up:
}
}
int count_line_num_cb(const char *table_name, const char *line, void *u_para)
{
(*((unsigned int *)u_para))++;
return 0;
}
int line_idx = 0;
int make_serial_rule(const char *table_name, const char *line, void *u_para)
{
struct serial_rule *s_rule=(struct serial_rule *)u_para;
redisContext *ctx = s_rule->ref_ctx;
long long absolute_expire_time = s_rule->timeout;
char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1);
while(buff[strlen(line) - 1] == '\n' || buff[strlen(line) - 1] == '\t') {
buff[strlen(line) - 1] = '\0';
}
const char *redis_rule_key = "TEST_RULE_KEY";
redisReply *reply =
maat_wrap_redis_command(ctx, NULL, "INCRBY %s %d", redis_rule_key, 1);
if (reply->type == REDIS_REPLY_NIL) {
printf("incrby redis_rule_key:%s failed.", redis_rule_key);
return -1;
} else {
s_rule->rule_id = maat_read_redis_integer(reply);
freeReplyObject(reply);
reply = NULL;
}
maat_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, s_rule->rule_id,
table_name, buff, absolute_expire_time);
(s_rule + line_idx)->ref_ctx = ctx;
line_idx++;
FREE(buff);
return 0;
}
int write_config_to_redis(redisContext *c, char *json_filename,
struct log_handle *logger)
{
char tmp_iris_path[128] = {0};
char *json_buff = NULL;
size_t json_buff_sz = 0;
snprintf(tmp_iris_path, sizeof(tmp_iris_path),
"%s_iris_tmp", json_filename);
int ret = load_file_to_memory(json_filename,
(unsigned char **)&json_buff,
&json_buff_sz);
if (ret < 0) {
return -1;
}
ret = json2iris(json_buff, json_filename, c, tmp_iris_path,
sizeof(tmp_iris_path), NULL, NULL, logger);
FREE(json_buff);
if (ret < 0) {
return -1;
}
size_t total_line_cnt = 0;
config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL,
&total_line_cnt, NULL, logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
s_rule->ref_ctx = c;
long long server_time = maat_redis_server_time_s(c);
if (server_time < 0) {
return -1;
}
s_rule->timeout = server_time + 300;
config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL,
s_rule, NULL, logger);
s_rule->ref_ctx = NULL;
line_idx = 0;
int success_cnt = 0;
do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt,
server_time, logger);
} while (success_cnt < 0);
assert(success_cnt == (int)total_line_cnt);
for (size_t i = 0; i < total_line_cnt; i++) {
maat_clear_rule_cache(s_rule + i);
}
FREE(s_rule);
return 0;
}
int rollback_redis_version(redisContext *c, struct log_handle *logger)
{
redisReply *reply =
maat_wrap_redis_command(c, logger, "SET MAAT_VERSION 0");
if (NULL == reply) {
log_fatal(logger, MODULE_REDIS_TOOL,
"[%s:%d] set MAAT_VERSION failed, "
"Redis Communication error: %s",
__FUNCTION__, __LINE__, c->errstr);
return -1;
}
freeReplyObject(reply);
reply = NULL;
return 0;
}
int clear_config_in_redis(redisContext *c, struct log_handle *logger)
{
long long redis_version = 0;
redisReply *reply = maat_wrap_redis_command(c, logger, "GET MAAT_VERSION");
if (reply != NULL) {
if (reply->type == REDIS_REPLY_NIL ||
reply->type == REDIS_REPLY_ERROR) {
log_fatal(logger, MODULE_REDIS_TOOL,
"[%s:%d] GET MAAT_VERSION failed, maybe Redis is busy",
__FUNCTION__, __LINE__);
freeReplyObject(reply);
reply = NULL;
return -1;
}
} else {
log_fatal(logger, MODULE_REDIS_TOOL,
"[%s:%d] GET MAAT_VERSION failed with NULL reply, error: %s",
__FUNCTION__, __LINE__, c->errstr);
return -1;
}
redis_version = maat_read_redis_integer(reply);
if (redis_version < 0) {
if (reply->type == REDIS_REPLY_ERROR) {
log_fatal(logger, MODULE_REDIS_TOOL,
"[%s:%d] Redis Communication error: %s",
__FUNCTION__, __LINE__, reply->str);
}
freeReplyObject(reply);
reply = NULL;
return -1;
}
freeReplyObject(reply);
reply = NULL;
reply = maat_wrap_redis_command(c, logger, "MULTI");
freeReplyObject(reply);
reply = NULL;
int append_cmd_cnt = 0;
redisAppendCommand(c, "FLUSHDB");
append_cmd_cnt++;
redisAppendCommand(c, "SET MAAT_VERSION %lld", redis_version);
append_cmd_cnt++;
redisAppendCommand(c, "SET MAAT_PRE_VER %lld", redis_version);
append_cmd_cnt++;
redisAppendCommand(c, "SET %s 1", mr_region_id_var);
append_cmd_cnt++;
redisAppendCommand(c, "SET %s 1", mr_group_id_var);
append_cmd_cnt++;
redisAppendCommand(c, "EXEC");
append_cmd_cnt++;
int redis_transaction_success = 1;
for (int i = 0; i < append_cmd_cnt; i++) {
int ret = maat_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_OK) {
if (reply->type == REDIS_REPLY_NIL) {
redis_transaction_success = 0;
}
freeReplyObject(reply);
reply = NULL;
}
}
if (0 == redis_transaction_success) {
return -1;
}
return 0;
}
int exec_keys_transaction(redisContext *c, struct log_handle *logger)
static int exec_keys_transaction(redisContext *c, struct log_handle *logger)
{
printf("Ready to execute [keys EFFECTIVE_RULE:*] transaction...\n");
struct timespec start, end;
@@ -454,20 +277,14 @@ int exec_keys_transaction(redisContext *c, struct log_handle *logger)
return 0;
}
#define WORK_MODE_DUMP 0
#define WORK_MODE_JSON 1
#define WORK_MODE_UPLOAD 2
#define WORK_MODE_TEST_TRANS 3
int main(int argc, char * argv[])
{
int oc = 0;
int opt = 0;
int mode = 0;
char redis_ip[64] = {0};
int redis_port = 6379;
int redis_db = 0;
char dump_dir[128] = {0};
char json_file[128] = {0};
char upload_file[128] = {0};
char log_path[128] = "./maat_redis_tool.log";
int timeout = 0;
@@ -476,8 +293,8 @@ int main(int argc, char * argv[])
strncpy(dump_dir, redis_dump_dir, sizeof(dump_dir));
struct log_handle *logger = log_handle_create(log_path, 0);
while ((oc = getopt(argc,argv,"h:p:n:d:f:j:t:u:k")) != -1) {
switch (oc) {
while ((opt = getopt(argc,argv,"h:p:n:d:f:t:u:k")) != -1) {
switch (opt) {
case 'h':
strncpy(redis_ip, optarg, sizeof(redis_ip));
break;
@@ -494,10 +311,6 @@ int main(int argc, char * argv[])
dump_dir[strlen(dump_dir)-1] = '\0';
}
break;
case 'j':
strncpy(json_file, optarg, sizeof(json_file));
mode = WORK_MODE_JSON;
break;
case 'k':
mode = WORK_MODE_TEST_TRANS;
break;
@@ -516,96 +329,28 @@ int main(int argc, char * argv[])
}
}
if (mode == WORK_MODE_UPLOAD) {
return write_json_to_redis(upload_file, redis_ip, redis_port,
redis_db, logger);
}
redisContext *c = maat_connect_redis(redis_ip, redis_port, redis_db, logger);
if (NULL == c) {
return -1;
}
int ret = 0;
char tmp_iris_path[128] = {0};
if (mode == WORK_MODE_DUMP) {
log_info(logger, MODULE_REDIS_TOOL, "Reading key list from %s:%d db%d.",
redis_ip, redis_port, redis_db);
read_rule_from_redis(c, dump_dir, logger);
} else if(mode == WORK_MODE_JSON) {
char *json_buff = NULL;
size_t json_buff_sz = 0;
ret = load_file_to_memory(json_file, (unsigned char**)&json_buff, &json_buff_sz);
if (ret < 0) {
log_fatal(logger, MODULE_REDIS_TOOL, "open file:%s failed.", json_file);
return -1;
}
ret = json2iris(json_buff, json_file, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, logger);
FREE(json_buff);
if (ret < 0) {
log_fatal(logger, MODULE_REDIS_TOOL, "Invalid json format.");
return -1;
}
size_t total_line_cnt = 0;
config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, NULL, logger);
log_fatal(logger, MODULE_REDIS_TOOL, "Serialize %s to %zu lines, write temp file to %s .",
json_file, total_line_cnt, tmp_iris_path);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
s_rule->ref_ctx = c;
long long server_time = maat_redis_server_time_s(c);
if (!server_time) {
log_fatal(logger, MODULE_REDIS_TOOL, "Get Redis Time failed.");
FREE(s_rule);
return -1;
}
if (timeout > 0) {
s_rule->timeout = server_time + timeout;
}
log_info(logger, MODULE_REDIS_TOOL, "Timeout = %lld\n", s_rule->timeout);
config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, NULL, logger);
s_rule->ref_ctx = NULL;
int success_cnt = 0;
do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, logger);
} while(success_cnt < 0);
if (success_cnt != (int)total_line_cnt) {
log_fatal(logger, MODULE_REDIS_TOOL, "Only Add %d of %zu, rule id maybe conflicts.",
success_cnt, total_line_cnt);
}
for (size_t i = 0; i < total_line_cnt; i++) {
maat_clear_rule_cache(s_rule+i);
}
FREE(s_rule);
redisFree(c);
} else if (mode == WORK_MODE_UPLOAD) {
ret = clear_config_in_redis(c, logger);
if (ret < 0) {
log_fatal(logger, MODULE_REDIS_TOOL, "clear config in redis failed.");
return -1;
}
ret = write_config_to_redis(c, upload_file, logger);
if (ret < 0) {
log_fatal(logger, MODULE_REDIS_TOOL, "write config to redis failed.");
return -1;
}
ret = rollback_redis_version(c, logger);
if (ret < 0) {
log_fatal(logger, MODULE_REDIS_TOOL, "rollback redis version failed.");
return -1;
}
redisFree(c);
} else if (mode == WORK_MODE_TEST_TRANS) {
ret = exec_keys_transaction(c, logger);
if (ret < 0) {
printf("[ERROR]: execute 'KEYS EFFECTIVE_RULE:*' failed.");
return -1;
}
redisFree(c);
}
return 0;
redisFree(c);
return ret;
}