gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add input mode unit-test

Browse Source
This commit is contained in:
liuwentan
2022-12-05 23:21:18 +08:00
parent ea4c1ba4c3
commit 6d18cf0f36
24 changed files with 519 additions and 347 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
include/maat/maat.h
View File

@@ -64,8 +64,8 @@ int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts,
int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms); int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, size_t name_len); int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, size_t name_len);
int maat_options_set_deferred_load_on(struct maat_options *opts); int maat_options_set_deferred_load_on(struct maat_options *opts);
int maat_options_set_iris_full_dir(struct maat_options *opts, const char *full_dir); int maat_options_set_iris_full_index_dir(struct maat_options *opts, const char *full_idx_dir);
int maat_options_set_iris_inc_dir(struct maat_options *opts, const char *inc_dir); int maat_options_set_iris_inc_index_dir(struct maat_options *opts, const char *inc_idx_dir);
int maat_options_set_json_file(struct maat_options *opts, const char *json_filename); int maat_options_set_json_file(struct maat_options *opts, const char *json_filename);
int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip); int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip);
int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port); int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port);

40
scanner/adapter_hs.cpp
View File

@@ -76,14 +76,14 @@ int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_thr
for (size_t i = 1; i < nr_worker_threads; i++) { for (size_t i = 1; i < nr_worker_threads; i++) {
hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]); hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
fprintf(stderr, "Unable to clone scratch prototype"); fprintf(stderr, "Unable to clone scratch prototype\n");
hs_free_database(database); hs_free_database(database);
return -1; return -1;
} }
err = hs_scratch_size(hs_rt->scratchs[i], &hs_rt->scratch_size); err = hs_scratch_size(hs_rt->scratchs[i], &hs_rt->scratch_size);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
fprintf(stderr, "Unable to query scratch size"); fprintf(stderr, "Unable to query scratch size\n");
hs_free_database(database); hs_free_database(database);
return -1; return -1;
} }
@@ -115,7 +115,7 @@ int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
scan_mode, NULL, &hs_rt->literal_db, &compile_err); scan_mode, NULL, &hs_rt->literal_db, &compile_err);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
if (compile_err) { if (compile_err) {
fprintf(stderr, "%s compile error: %s", __func__, compile_err->message); fprintf(stderr, "%s compile error: %s\n", __func__, compile_err->message);
} }
hs_free_compile_error(compile_err); hs_free_compile_error(compile_err);
@@ -129,7 +129,7 @@ int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
scan_mode, NULL, &hs_rt->regex_db, &compile_err); scan_mode, NULL, &hs_rt->regex_db, &compile_err);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
if (compile_err) { if (compile_err) {
fprintf(stderr, "%s compile error: %s", __func__, compile_err->message); fprintf(stderr, "%s compile error: %s\n", __func__, compile_err->message);
} }
hs_free_compile_error(compile_err); hs_free_compile_error(compile_err);
goto error; goto error;
@@ -170,23 +170,23 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patt
if (hs_cd->patterns != NULL) { if (hs_cd->patterns != NULL) {
for (size_t i = 0; i < n_patterns; i++) { for (size_t i = 0; i < n_patterns; i++) {
free(hs_cd->patterns[i]); FREE(hs_cd->patterns[i]);
} }
free(hs_cd->patterns); FREE(hs_cd->patterns);
free(hs_cd->pattern_lens); FREE(hs_cd->pattern_lens);
free(hs_cd->ids); FREE(hs_cd->ids);
free(hs_cd->flags); FREE(hs_cd->flags);
} }
free(hs_cd); FREE(hs_cd);
} }
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array) struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array)
{ {
if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) || if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) ||
0 == nr_worker_threads || NULL == expr_array || 0 == n_expr_array) { 0 == nr_worker_threads || NULL == expr_array || 0 == n_expr_array) {
fprintf(stderr, "%s input parameters illegal!", __func__); fprintf(stderr, "%s input parameters illegal!\n", __func__);
return NULL; return NULL;
} }
@@ -285,13 +285,13 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
/* create bool matcher */ /* create bool matcher */
hs_instance->hs_rt->bm = bool_matcher_new(exprs, n_expr_array, &mem_size); hs_instance->hs_rt->bm = bool_matcher_new(exprs, n_expr_array, &mem_size);
if (hs_instance->hs_rt->bm != NULL) { if (hs_instance->hs_rt->bm != NULL) {
fprintf(stdout, "Adapter_hs module: build bool matcher of %zu expressions with %zu bytes memory.", fprintf(stdout, "Adapter_hs module: build bool matcher of %zu expressions with %zu bytes memory\n",
n_expr_array, mem_size); n_expr_array, mem_size);
} else { } else {
fprintf(stderr, "Adapter_hs module: build bool matcher failed."); fprintf(stderr, "Adapter_hs module: build bool matcher failed\n");
goto error; goto error;
} }
free(exprs); FREE(exprs);
/* build hs database */ /* build hs database */
ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, scan_mode); ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, scan_mode);
@@ -348,16 +348,16 @@ void adapter_hs_destroy(struct adapter_hs *hs_instance)
} }
} }
} }
free(hs_instance->hs_rt->scratchs); FREE(hs_instance->hs_rt->scratchs);
if (hs_instance->hs_rt->bm != NULL) { if (hs_instance->hs_rt->bm != NULL) {
bool_matcher_free(hs_instance->hs_rt->bm); bool_matcher_free(hs_instance->hs_rt->bm);
} }
free(hs_instance->hs_rt); FREE(hs_instance->hs_rt);
} }
free(hs_instance); FREE(hs_instance);
} }
static inline int compare_pattern_id(const void* a, const void* b) static inline int compare_pattern_id(const void* a, const void* b)
@@ -443,7 +443,7 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d
} }
*n_results = bool_matcher_ret; *n_results = bool_matcher_ret;
free(bool_matcher_results); FREE(bool_matcher_results);
utarray_free(pattern_id_set); utarray_free(pattern_id_set);
return 0; return 0;
@@ -519,7 +519,7 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
} }
*n_results = bool_matcher_ret; *n_results = bool_matcher_ret;
free(bool_matcher_results); FREE(bool_matcher_results);
return 0; return 0;
} }
@@ -534,5 +534,5 @@ void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
/* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */ /* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */
hs_stream->hs_rt = NULL; hs_stream->hs_rt = NULL;
free(hs_stream); FREE(hs_stream);
} }

2
scanner/adapter_hs_gtest.cpp
View File

@@ -51,7 +51,7 @@ void expr_array_free(and_expr_t expr_array[], size_t n_expr_array)
{ {
for (size_t i = 0; i < n_expr_array; i++) { for (size_t i = 0; i < n_expr_array; i++) {
for (size_t j = 0; j < expr_array[i].n_patterns; j++) { for (size_t j = 0; j < expr_array[i].n_patterns; j++) {
free(expr_array[i].patterns[j].pat); FREE(expr_array[i].patterns[j].pat);
} }
} }
} }

9
src/inc_internal/maat_rule.h
View File

@@ -50,8 +50,8 @@ enum data_source {
}; };
struct source_iris_ctx { struct source_iris_ctx {
char inc_dir[NAME_MAX]; char inc_idx_dir[NAME_MAX];
char full_dir[NAME_MAX]; char full_idx_dir[NAME_MAX];
}; };
struct source_json_ctx struct source_json_ctx
@@ -83,7 +83,6 @@ struct foreign_key {
struct serial_rule { struct serial_rule {
enum maat_operation op;//0: delete, 1: add. enum maat_operation op;//0: delete, 1: add.
unsigned long rule_id; unsigned long rule_id;
int label_id;
long long timeout; // absolute unix time. long long timeout; // absolute unix time.
char table_name[NAME_MAX]; char table_name[NAME_MAX];
char *table_line; char *table_line;
@@ -174,9 +173,9 @@ int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type,
const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len); const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len);
int maat_cmd_exec_serial_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, long long server_time); int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, long long server_time);
void maat_cmd_empty_serial_rule(struct serial_rule *s_rule); void maat_cmd_clear_rule_cache(struct serial_rule *s_rule);
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version,
long long *new_version, struct table_schema_manager* table_schema_mgr, long long *new_version, struct table_schema_manager* table_schema_mgr,

2
src/inc_internal/maat_utils.h
View File

@@ -60,6 +60,8 @@ size_t memcat(void **dest, size_t offset, size_t *n_dest, const void *src, size_
/* system cmd wrapper */ /* system cmd wrapper */
int system_cmd_mkdir(const char* path); int system_cmd_mkdir(const char* path);
int system_cmd_rmdir(const char *dir);
#ifdef __cpluscplus #ifdef __cpluscplus
} }
#endif #endif

38
src/json2iris.cpp
View File

@@ -223,14 +223,14 @@ int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_descript
{ {
cJSON *item = cJSON_GetObjectItem(plug_table_json, "table_name"); cJSON *item = cJSON_GetObjectItem(plug_table_json, "table_name");
if (NULL == item || item->type != cJSON_String) { if (NULL == item || item->type != cJSON_String) {
fprintf(stderr, "The %d plugin_table's table_name not defined or format error.", sequence); fprintf(stderr, "The %d plugin_table's table_name not defined or format error\n", sequence);
return -1; return -1;
} }
const char *table_name = item->valuestring; const char *table_name = item->valuestring;
cJSON *table_content = cJSON_GetObjectItem(plug_table_json, "table_content"); cJSON *table_content = cJSON_GetObjectItem(plug_table_json, "table_content");
if (NULL == table_content || table_content->type != cJSON_Array) { if (NULL == table_content || table_content->type != cJSON_Array) {
fprintf(stderr, "%d plugin_table's table_content not defined or format error.", sequence); fprintf(stderr, "%d plugin_table's table_content not defined or format error\n", sequence);
return -1; return -1;
} }
int line_cnt = cJSON_GetArraySize(table_content); int line_cnt = cJSON_GetArraySize(table_content);
@@ -242,7 +242,7 @@ int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_descript
for (int i = 0; i < line_cnt; i++) { for (int i = 0; i < line_cnt; i++) {
each_line = cJSON_GetArrayItem(table_content, i); each_line = cJSON_GetArrayItem(table_content, i);
if (NULL == each_line || each_line->type != cJSON_String) { if (NULL == each_line || each_line->type != cJSON_String) {
fprintf(stderr, "plugin_table %s's line %d format error.", table_info->table_name, i+1); fprintf(stderr, "plugin_table %s's line %d format error\n", table_info->table_name, i+1);
continue; continue;
} }
@@ -331,7 +331,7 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla
} }
if (NULL == item || item->type != cmd[i].json_type) { if (NULL == item || item->type != cmd[i].json_type) {
fprintf(stderr, "%s not defined or wrong format.", cmd[i].json_string); fprintf(stderr, "%s not defined or wrong format\n", cmd[i].json_string);
ret = -1; ret = -1;
goto error_out; goto error_out;
} }
@@ -341,7 +341,7 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla
char *p = item->valuestring; char *p = item->valuestring;
ret = maat_kv_read(str2int, p, &int_value); ret = maat_kv_read(str2int, p, &int_value);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "%s's value %s is not valid format.", cmd[i].json_string, p); fprintf(stderr, "%s's value %s is not valid format\n", cmd[i].json_string, p);
FREE(p); FREE(p);
ret = -1; ret = -1;
goto error_out; goto error_out;
@@ -725,14 +725,14 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i
{ {
cJSON *item = cJSON_GetObjectItem(region_json, "table_name"); cJSON *item = cJSON_GetObjectItem(region_json, "table_name");
if (NULL == item || item->type != cJSON_String) { if (NULL == item || item->type != cJSON_String) {
fprintf(stderr, "compile rule %d's table_name not defined or format error.", compile_id); fprintf(stderr, "compile rule %d's table_name not defined or format error\n", compile_id);
return -1; return -1;
} }
const char *table_name = item->valuestring; const char *table_name = item->valuestring;
item = cJSON_GetObjectItem(region_json, "table_type"); item = cJSON_GetObjectItem(region_json, "table_type");
if (NULL == item || item->type != cJSON_String) { if (NULL == item || item->type != cJSON_String) {
fprintf(stderr, "compile rule %d's table name %s's table_type not defined or format error.", fprintf(stderr, "compile rule %d's table name %s's table_type not defined or format error\n",
compile_id, table_name); compile_id, table_name);
return -1; return -1;
} }
@@ -741,14 +741,14 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i
enum table_type table_type = TABLE_TYPE_EXPR; enum table_type table_type = TABLE_TYPE_EXPR;
int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type)); int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type));
if (ret != 1) { if (ret != 1) {
fprintf(stderr, "compile rule %d table name %s's table_type %s invalid.", fprintf(stderr, "compile rule %d table name %s's table_type %s invalid\n",
compile_id, table_name, table_type_str); compile_id, table_name, table_type_str);
return -1; return -1;
} }
cJSON *table_content = cJSON_GetObjectItem(region_json, "table_content"); cJSON *table_content = cJSON_GetObjectItem(region_json, "table_content");
if (NULL == table_content || table_content->type != cJSON_Object) { if (NULL == table_content || table_content->type != cJSON_Object) {
fprintf(stderr, "compile rule %d table name %s's table_content not defined or format error.", fprintf(stderr, "compile rule %d table name %s's table_content not defined or format error\n",
compile_id, table_name); compile_id, table_name);
return -1; return -1;
} }
@@ -865,7 +865,7 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
cJSON_ArrayForEach(region_rule, region_json) { cJSON_ArrayForEach(region_rule, region_json) {
ret = write_region_rule(region_rule, tracking_compile_id, group_info->group_id, p_iris); ret = write_region_rule(region_rule, tracking_compile_id, group_info->group_id, p_iris);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "compile rule %d write region error.", tracking_compile_id); fprintf(stderr, "compile rule %d write region error\n", tracking_compile_id);
return -1; return -1;
} }
} }
@@ -885,7 +885,7 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
} }
if (NULL == region_json && NULL == sub_groups) { if (NULL == region_json && NULL == sub_groups) {
fprintf(stdout, "A group of compile rule %d has neither regions, sub groups, nor refered another exisited group.", fprintf(stdout, "A group of compile rule %d has neither regions, sub groups, nor refered another exisited group\n",
tracking_compile_id); tracking_compile_id);
} }
} }
@@ -897,7 +897,7 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
} }
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "%s rule %d write group error.", str_parent_type[parent_type], parent_id); fprintf(stderr, "%s rule %d write group error\n", str_parent_type[parent_type], parent_id);
return -1; return -1;
} }
@@ -908,7 +908,7 @@ int write_compile_line(cJSON *compile, struct iris_description *p_iris)
{ {
cJSON *item=cJSON_GetObjectItem(compile, "compile_id"); cJSON *item=cJSON_GetObjectItem(compile, "compile_id");
if (item->type != cJSON_Number) { if (item->type != cJSON_Number) {
fprintf(stderr, "compile_id format not number."); fprintf(stderr, "compile_id format not number\n");
return -1; return -1;
} }
int compile_id = item->valueint; int compile_id = item->valueint;
@@ -1045,7 +1045,7 @@ int write_index_file(struct iris_description *p_iris)
{ {
p_iris->idx_fp = fopen(p_iris->index_path, "w"); p_iris->idx_fp = fopen(p_iris->index_path, "w");
if (NULL == p_iris->idx_fp) { if (NULL == p_iris->idx_fp) {
fprintf(stderr, "index file %s fopen error %s.",p_iris->index_path, strerror(errno)); fprintf(stderr, "index file %s fopen error %s\n",p_iris->index_path, strerror(errno));
return -1; return -1;
} }
@@ -1111,19 +1111,19 @@ int write_iris(cJSON *json, struct iris_description *p_iris)
cJSON_ArrayForEach(compile_obj, compile_array) { cJSON_ArrayForEach(compile_obj, compile_array) {
int compile_id = write_compile_line(compile_obj,p_iris); int compile_id = write_compile_line(compile_obj,p_iris);
if (compile_id < 0) { if (compile_id < 0) {
fprintf(stderr, "In %d compile rule.", i); fprintf(stderr, "In %d compile rule\n", i);
return -1; return -1;
} }
group_array = cJSON_GetObjectItem(compile_obj, "groups"); group_array = cJSON_GetObjectItem(compile_obj, "groups");
if (NULL == group_array) { if (NULL == group_array) {
fprintf(stderr, "compile rule %d have no group.",compile_id); fprintf(stderr, "compile rule %d have no group\n",compile_id);
return -1; return -1;
} }
int group_cnt = cJSON_GetArraySize(group_array); int group_cnt = cJSON_GetArraySize(group_array);
if (group_cnt <= 0) { if (group_cnt <= 0) {
fprintf(stderr, "compile rule %d have no groups.", compile_id); fprintf(stderr, "compile rule %d have no groups\n", compile_id);
return -1; return -1;
} }
@@ -1158,7 +1158,7 @@ int json2iris(const char *json_buff, const char *json_filename, const char *comp
cJSON *json = cJSON_Parse(json_buff); cJSON *json = cJSON_Parse(json_buff);
if (!json) { if (!json) {
fprintf(stderr, "Error before: %-200.200s", cJSON_GetErrorPtr()); fprintf(stderr, "Error before: %-200.200s\n", cJSON_GetErrorPtr());
goto error_out; goto error_out;
} }
@@ -1186,7 +1186,7 @@ int json2iris(const char *json_buff, const char *json_filename, const char *comp
ret = create_tmp_dir(&iris_cfg); ret = create_tmp_dir(&iris_cfg);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "create tmp folder %s error", iris_cfg.tmp_iris_dir); fprintf(stderr, "create tmp folder %s error\n", iris_cfg.tmp_iris_dir);
goto error_out; goto error_out;
} }

47
src/maat_api.cpp
View File

@@ -72,25 +72,25 @@ int maat_options_set_deferred_load_on(struct maat_options *opts)
return 0; return 0;
} }
int maat_options_set_iris_full_dir(struct maat_options *opts, const char *full_dir) int maat_options_set_iris_full_index_dir(struct maat_options *opts, const char *full_idx_dir)
{ {
if (strlen(full_dir) >= NAME_MAX) { if (strlen(full_idx_dir) >= NAME_MAX) {
return -1; return -1;
} }
memcpy(opts->iris_ctx.full_dir, full_dir, strlen(full_dir)); memcpy(opts->iris_ctx.full_idx_dir, full_idx_dir, strlen(full_idx_dir));
opts->input_mode = DATA_SOURCE_IRIS_FILE; opts->input_mode = DATA_SOURCE_IRIS_FILE;
return 0; return 0;
} }
int maat_options_set_iris_inc_dir(struct maat_options *opts, const char *inc_dir) int maat_options_set_iris_inc_index_dir(struct maat_options *opts, const char *inc_idx_dir)
{ {
if (strlen(inc_dir) >= NAME_MAX) { if (strlen(inc_idx_dir) >= NAME_MAX) {
return -1; return -1;
} }
memcpy(opts->iris_ctx.inc_dir, inc_dir, strlen(inc_dir)); memcpy(opts->iris_ctx.inc_idx_dir, inc_idx_dir, strlen(inc_idx_dir));
opts->input_mode = DATA_SOURCE_IRIS_FILE; opts->input_mode = DATA_SOURCE_IRIS_FILE;
return 0; return 0;
@@ -135,7 +135,7 @@ void maat_read_full_config(struct maat *maat_instance)
switch (maat_instance->input_mode) { switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS: case DATA_SOURCE_REDIS:
mr_ctx = &(maat_instance->mr_ctx); mr_ctx = &(maat_instance->mr_ctx);
fprintf(stdout, "Maat initiate from Redis %s:%hu db%d.", fprintf(stdout, "Maat initiate from Redis %s:%hu db%d\n",
mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
if (mr_ctx->read_ctx != NULL) { if (mr_ctx->read_ctx != NULL) {
@@ -145,24 +145,24 @@ void maat_read_full_config(struct maat *maat_instance)
} }
if (NULL == maat_instance->creating_maat_rt) { if (NULL == maat_instance->creating_maat_rt) {
fprintf(stderr, "At initiation: NO effective rule in %s.", fprintf(stderr, "At initiation: NO effective rule in redis %s:%hu db%d\n",
maat_instance->iris_ctx.full_dir); mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
} }
break; break;
case DATA_SOURCE_IRIS_FILE: case DATA_SOURCE_IRIS_FILE:
config_monitor_traverse(maat_instance->maat_version, config_monitor_traverse(maat_instance->maat_version,
maat_instance->iris_ctx.full_dir, maat_instance->iris_ctx.full_idx_dir,
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance); maat_instance);
if (NULL == maat_instance->creating_maat_rt) { if (NULL == maat_instance->creating_maat_rt) {
fprintf(stderr, "At initiation: NO effective rule in %s.", fprintf(stderr, "At initiation: NO effective rule in %s\n",
maat_instance->iris_ctx.full_dir); maat_instance->iris_ctx.full_idx_dir);
} }
break; break;
case DATA_SOURCE_JSON_FILE: case DATA_SOURCE_JSON_FILE:
ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str)); ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str));
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "Maat re-initiate with JSON file %s failed: %s", maat_instance->json_ctx.json_file, err_str); fprintf(stderr, "Maat re-initiate with JSON file %s failed: %s\n", maat_instance->json_ctx.json_file, err_str);
return; return;
} }
@@ -171,7 +171,7 @@ void maat_read_full_config(struct maat *maat_instance)
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance); maat_instance);
if (NULL == maat_instance->creating_maat_rt) { if (NULL == maat_instance->creating_maat_rt) {
fprintf(stderr, "At initiation: NO effective rule in %s.", fprintf(stderr, "At initiation: NO effective rule in %s\n",
maat_instance->json_ctx.iris_file); maat_instance->json_ctx.iris_file);
} }
break; break;
@@ -210,8 +210,8 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
maat_instance->mr_ctx.redis_db = opts->redis_ctx.redis_db; maat_instance->mr_ctx.redis_db = opts->redis_ctx.redis_db;
break; break;
case DATA_SOURCE_IRIS_FILE: case DATA_SOURCE_IRIS_FILE:
memcpy(maat_instance->iris_ctx.full_dir, opts->iris_ctx.full_dir, strlen(opts->iris_ctx.full_dir)); memcpy(maat_instance->iris_ctx.full_idx_dir, opts->iris_ctx.full_idx_dir, strlen(opts->iris_ctx.full_idx_dir));
memcpy(maat_instance->iris_ctx.inc_dir, opts->iris_ctx.inc_dir, strlen(opts->iris_ctx.inc_dir)); memcpy(maat_instance->iris_ctx.inc_idx_dir, opts->iris_ctx.inc_idx_dir, strlen(opts->iris_ctx.inc_idx_dir));
break; break;
case DATA_SOURCE_JSON_FILE: case DATA_SOURCE_JSON_FILE:
memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file, strlen(opts->json_ctx.json_file)); memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file, strlen(opts->json_ctx.json_file));
@@ -237,21 +237,24 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
maat_read_full_config(maat_instance); maat_read_full_config(maat_instance);
} }
pthread_create(&(maat_instance->cfg_mon_thread), NULL, rule_monitor_loop, (void*)maat_instance); pthread_create(&(maat_instance->cfg_mon_thread), NULL, rule_monitor_loop, (void *)maat_instance);
return maat_instance; return maat_instance;
failed: failed:
free(maat_instance); FREE(maat_instance);
return NULL; return NULL;
} }
void maat_free(struct maat *maat_instance) void maat_free(struct maat *maat_instance)
{ {
if (NULL == maat_instance) {
void* ret = NULL;
pthread_join(maat_instance->cfg_mon_thread, &ret);
return; return;
}
void *ret = NULL;
maat_instance->is_running = 0;
pthread_join(maat_instance->cfg_mon_thread, &ret);
} }
int maat_table_get_id(struct maat *maat_instance, const char *table_name) int maat_table_get_id(struct maat *maat_instance, const char *table_name)

12
src/maat_command.cpp
View File

@@ -59,7 +59,7 @@ redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int r
redisContext *c = redisConnectWithTimeout(redis_ip, redis_port, connect_timeout); redisContext *c = redisConnectWithTimeout(redis_ip, redis_port, connect_timeout);
if (NULL == c || c->err) { if (NULL == c || c->err) {
fprintf(stderr, "Unable to connect redis server %s:%d db%d, error: %s", fprintf(stderr, "Unable to connect redis server %s:%d db%d, error: %s\n",
redis_ip, redis_port, redis_db, c == NULL ? "Unknown" : c->errstr); redis_ip, redis_port, redis_db, c == NULL ? "Unknown" : c->errstr);
if (c != NULL) { if (c != NULL) {
@@ -91,7 +91,7 @@ void save_serial_rule(void *data, void *user)
array->array[i].op = MAAT_OP_ADD; array->array[i].op = MAAT_OP_ADD;
} }
void maat_cmd_empty_serial_rule(struct serial_rule *s_rule) void maat_cmd_clear_rule_cache(struct serial_rule *s_rule)
{ {
if (s_rule->table_line != NULL) { if (s_rule->table_line != NULL) {
FREE(s_rule->table_line); FREE(s_rule->table_line);
@@ -302,7 +302,7 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, line_rule->table_name); int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, line_rule->table_name);
if (table_id < 0) { if (table_id < 0) {
fprintf(stderr, "Command set line id %d failed: unknown table %s.", line_rule->rule_id, line_rule->table_name); fprintf(stderr, "Command set line id %d failed: unknown table %s\n", line_rule->rule_id, line_rule->table_name);
FREE(s_rule); FREE(s_rule);
return -1; return -1;
} }
@@ -315,7 +315,7 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
int valid_flag_column = table_schema_get_valid_flag_column(table_schema); int valid_flag_column = table_schema_get_valid_flag_column(table_schema);
if (valid_flag_column < 0) { if (valid_flag_column < 0) {
fprintf(stderr, "Command set line id %d failed: table %s is not a plugin or ip_plugin table.", fprintf(stderr, "Command set line id %d failed: table %s is not a plugin or ip_plugin table\n",
line_rule->rule_id, line_rule->table_name); line_rule->rule_id, line_rule->table_name);
FREE(s_rule); FREE(s_rule);
return -1; return -1;
@@ -331,7 +331,7 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id, line_rule->table_name, maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id, line_rule->table_name,
line_rule->table_line, absolute_expire_time); line_rule->table_line, absolute_expire_time);
int success_cnt = maat_cmd_exec_serial_rule(write_ctx, s_rule, 1, server_time); int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time);
if (success_cnt != 1) { if (success_cnt != 1) {
ret = -1; ret = -1;
goto error_out; goto error_out;
@@ -341,7 +341,7 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
maat_instance->line_cmd_acc_num += success_cnt; maat_instance->line_cmd_acc_num += success_cnt;
error_out: error_out:
maat_cmd_empty_serial_rule(s_rule); maat_cmd_clear_rule_cache(s_rule);
FREE(s_rule); FREE(s_rule);
return ret; return ret;

36
src/maat_config_monitor.cpp
View File

@@ -154,7 +154,7 @@ int cm_read_table_file(struct cm_table_info_t* index,
} }
} }
free(table_file_buff); FREE(table_file_buff);
return 0; return 0;
} }
@@ -241,14 +241,14 @@ int get_new_idx_path(long long current_version, const char *file_dir, char ***id
} }
if (strlen(namelist[i]->d_name) > 42) { if (strlen(namelist[i]->d_name) > 42) {
fprintf(stderr, "config file %s filename too long,should like full_config_index.00000000000000000001", fprintf(stderr, "config file %s filename too long, should like full_config_index.000000000001\n",
namelist[i]->d_name); namelist[i]->d_name);
continue; continue;
} }
int ret = sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld", update_str, &config_seq); int ret = sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld", update_str, &config_seq);
if (ret != 2) { if (ret != 2) {
fprintf(stderr, "config file %s filename error,should like full_config_index.00000000000000000001", fprintf(stderr, "config file %s filename error, should like full_config_index.000000000001\n",
namelist[i]->d_name); namelist[i]->d_name);
continue; continue;
} }
@@ -267,7 +267,7 @@ int get_new_idx_path(long long current_version, const char *file_dir, char ***id
} }
} }
} else { } else {
fprintf(stderr, "config file %s,not full or inc config", namelist[i]->d_name); fprintf(stderr, "config file %s, not full or inc config\n", namelist[i]->d_name);
} }
} }
@@ -294,11 +294,11 @@ int get_new_idx_path(long long current_version, const char *file_dir, char ***id
update_type = CM_UPDATE_TYPE_NONE; update_type = CM_UPDATE_TYPE_NONE;
} }
free(inc_file_idx); FREE(inc_file_idx);
for (i = 0; i < n; i++) { for (i = 0; i < n; i++) {
free(namelist[i]); FREE(namelist[i]);
} }
free(namelist); FREE(namelist);
return update_type; return update_type;
} }
@@ -322,7 +322,7 @@ void config_monitor_traverse(long long current_version, const char *idx_dir,
for (i = 0; i < idx_path_num; i++) { for (i = 0; i < idx_path_num; i++) {
int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, CM_MAX_TABLE_NUM); int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, CM_MAX_TABLE_NUM);
if (table_num < 0) { if (table_num < 0) {
fprintf(stderr, "load %s failed, abandon udpate.", idx_path_array[i]); fprintf(stderr, "load %s failed, abandon update\n", idx_path_array[i]);
break; break;
} }
@@ -344,10 +344,10 @@ void config_monitor_traverse(long long current_version, const char *idx_dir,
} }
for (i = 0; i < idx_path_num; i++) { for (i = 0; i < idx_path_num; i++) {
free(idx_path_array[i]); FREE(idx_path_array[i]);
} }
free(idx_path_array); FREE(idx_path_array);
} }
int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz) int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz)
@@ -361,13 +361,13 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, c
size_t decrypted_buff_sz = 0; size_t decrypted_buff_sz = 0;
size_t uncompressed_buff_sz = 0; size_t uncompressed_buff_sz = 0;
fprintf(stdout, "Maat initial with JSON file %s, formating..", json_filename); fprintf(stdout, "Maat initial with JSON file %s, formating...\n", json_filename);
if (strlen(maat_instance->decrypt_key) && strlen(maat_instance->decrypt_algo)) { if (strlen(maat_instance->decrypt_key) && strlen(maat_instance->decrypt_algo)) {
ret = decrypt_open(json_filename, maat_instance->decrypt_key, maat_instance->decrypt_algo, ret = decrypt_open(json_filename, maat_instance->decrypt_key, maat_instance->decrypt_algo,
(unsigned char **)&decrypted_buff, &decrypted_buff_sz, err_str, err_str_sz); (unsigned char **)&decrypted_buff, &decrypted_buff_sz, err_str, err_str_sz);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "Decrypt Maat JSON file %s failed.", json_filename); fprintf(stderr, "Decrypt Maat JSON file %s failed\n", json_filename);
return -1; return -1;
} }
@@ -377,9 +377,9 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, c
if (maat_instance->maat_json_is_gzipped) { if (maat_instance->maat_json_is_gzipped) {
ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz); ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz);
free(json_buff); FREE(json_buff);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "Uncompress Maat JSON file %s failed.", json_filename); fprintf(stderr, "Uncompress Maat JSON file %s failed\n", json_filename);
return -1; return -1;
} }
@@ -391,7 +391,7 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, c
if (NULL == json_buff) { if (NULL == json_buff) {
ret = load_file_to_memory(json_filename, &json_buff, &json_buff_sz); ret = load_file_to_memory(json_filename, &json_buff, &json_buff_sz);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "Read Maat JSON file %s failed.", json_filename); fprintf(stderr, "Read Maat JSON file %s failed\n", json_filename);
return -1; return -1;
} }
} }
@@ -404,7 +404,7 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, c
strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL, strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL,
strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL); strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL);
free(json_buff); FREE(json_buff);
json_buff = NULL; json_buff = NULL;
if (ret < 0) { if (ret < 0) {
return -1; return -1;
@@ -414,11 +414,11 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, c
strncpy(maat_instance->json_ctx.json_file, json_filename, sizeof(maat_instance->json_ctx.json_file)); strncpy(maat_instance->json_ctx.json_file, json_filename, sizeof(maat_instance->json_ctx.json_file));
} }
ret=stat(json_filename, &fstat_buf); ret = stat(json_filename, &fstat_buf);
maat_instance->json_ctx.last_md5_time = fstat_buf.st_ctim; maat_instance->json_ctx.last_md5_time = fstat_buf.st_ctim;
md5_file(maat_instance->json_ctx.json_file, maat_instance->json_ctx.effective_json_md5); md5_file(maat_instance->json_ctx.json_file, maat_instance->json_ctx.effective_json_md5);
fprintf(stdout, "JSON file %s md5: %s, generate index file %s OK.", fprintf(stdout, "JSON file %s md5: %s, generate index file %s OK\n",
maat_instance->json_ctx.json_file, maat_instance->json_ctx.json_file,
maat_instance->json_ctx.effective_json_md5, maat_instance->json_ctx.effective_json_md5,
maat_instance->json_ctx.iris_file); maat_instance->json_ctx.iris_file);

6
src/maat_ex_data.cpp
View File

@@ -31,7 +31,7 @@ struct ex_data_runtime {
void cache_row_free(void *p) void cache_row_free(void *p)
{ {
free(*(char **)p); FREE(*(char **)p);
} }
UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free}; UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
@@ -63,7 +63,7 @@ void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt)
rcu_hash_free(ex_data_rt->htable); rcu_hash_free(ex_data_rt->htable);
} }
free(ex_data_rt); FREE(ex_data_rt);
} }
void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt) void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt)
@@ -147,7 +147,7 @@ void ex_data_runtime_del_ex_data(struct ex_data_runtime *ex_data_rt, const char
{ {
void *tmp_data = rcu_hash_find(ex_data_rt->htable, key, key_len); void *tmp_data = rcu_hash_find(ex_data_rt->htable, key, key_len);
if (NULL == tmp_data) { if (NULL == tmp_data) {
fprintf(stderr, "ex data del error: no such key:%s", key); fprintf(stderr, "ex data del error: no such key:%s\n", key);
return; return;
} }

8
src/maat_garbage_collection.cpp
View File

@@ -48,11 +48,11 @@ void maat_garbage_bin_free(struct maat_garbage_bin* bin)
while (p = TAILQ_FIRST(&bin->garbage_q)) { while (p = TAILQ_FIRST(&bin->garbage_q)) {
p->garbage_free(p->garbage); p->garbage_free(p->garbage);
TAILQ_REMOVE(&bin->garbage_q, p, entries); TAILQ_REMOVE(&bin->garbage_q, p, entries);
free(p); FREE(p);
bin->bag_cnt--; bin->bag_cnt--;
} }
free(bin); FREE(bin);
} }
size_t maat_garbage_bin_get_size(struct maat_garbage_bin* bin) size_t maat_garbage_bin_get_size(struct maat_garbage_bin* bin)
@@ -82,7 +82,7 @@ void maat_garbage_collect_routine(struct maat_garbage_bin* bin)
if ((now - p->create_time) > p->timeout || p->timeout == 0) { if ((now - p->create_time) > p->timeout || p->timeout == 0) {
p->garbage_free(p->garbage); p->garbage_free(p->garbage);
TAILQ_REMOVE(&bin->garbage_q, p, entries); TAILQ_REMOVE(&bin->garbage_q, p, entries);
free(p); FREE(p);
n_clollected++; n_clollected++;
} }
n_bag++; n_bag++;
@@ -98,7 +98,7 @@ void maat_garbage_collect_by_force(struct maat_garbage_bin* bin)
while (p = TAILQ_FIRST(&bin->garbage_q)) { while (p = TAILQ_FIRST(&bin->garbage_q)) {
p->garbage_free(p->garbage); p->garbage_free(p->garbage);
TAILQ_REMOVE(&bin->garbage_q, p, entries); TAILQ_REMOVE(&bin->garbage_q, p, entries);
free(p); FREE(p);
bin->bag_cnt--; bin->bag_cnt--;
} }
} }

6
src/maat_kv.cpp
View File

@@ -51,10 +51,10 @@ struct maat_kv_pair* maat_kv_pair_new(const char* key, size_t keylen, int value)
void maat_kv_pair_free(struct maat_kv_pair* kv) void maat_kv_pair_free(struct maat_kv_pair* kv)
{ {
free(kv->key); FREE(kv->key);
kv->key = NULL; kv->key = NULL;
free(kv); FREE(kv);
} }
struct maat_kv_store* maat_kv_store_new(void) struct maat_kv_store* maat_kv_store_new(void)
@@ -77,7 +77,7 @@ void maat_kv_store_free(struct maat_kv_store* store)
maat_kv_pair_free(kv); maat_kv_pair_free(kv);
} }
free(store); FREE(store);
} }
int maat_kv_register_unNull(struct maat_kv_store* store, const char* key, size_t keylen, int value) int maat_kv_register_unNull(struct maat_kv_store* store, const char* key, size_t keylen, int value)

134
src/maat_redis_monitor.cpp
View File

@@ -55,7 +55,7 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
for (int i = 0; i < n_foreign; i++) { for (int i = 0; i < n_foreign; i++) {
const char *p_foreign = maat_cmd_find_Nth_column(p_rule->table_line, foreign_columns[i], &foreign_key_size); const char *p_foreign = maat_cmd_find_Nth_column(p_rule->table_line, foreign_columns[i], &foreign_key_size);
if (NULL == p_foreign) { if (NULL == p_foreign) {
fprintf(stderr, "Get %s,%lu foreign keys failed: No %dth column.", fprintf(stderr, "Get %s,%lu foreign keys failed: No %dth column\n",
p_rule->table_name, p_rule->rule_id, foreign_columns[i]); p_rule->table_name, p_rule->rule_id, foreign_columns[i]);
continue; continue;
} }
@@ -66,7 +66,7 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
} }
if (0 != strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) { if (0 != strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) {
fprintf(stderr, "Get %s,%lu foreign key failed: Invalid source prefix %s.", fprintf(stderr, "Get %s,%lu foreign key failed: Invalid source prefix %s\n",
p_rule->table_name, p_rule->rule_id, p_foreign); p_rule->table_name, p_rule->rule_id, p_foreign);
continue; continue;
} }
@@ -76,7 +76,7 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
p_foreign += strlen(foreign_source_prefix); p_foreign += strlen(foreign_source_prefix);
if (0 != strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) { if (0 != strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) {
fprintf(stdout, "%s, %lu foreign key prefix %s is not recommended.", fprintf(stdout, "%s, %lu foreign key prefix %s is not recommended\n",
p_rule->table_name, p_rule->rule_id, p_foreign); p_rule->table_name, p_rule->rule_id, p_foreign);
} }
@@ -181,7 +181,7 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
for (i = 0; i < rule_num; i++) { for (i = 0; i < rule_num; i++) {
ret = maat_cmd_wrap_redis_get_reply(c, &reply); ret = maat_cmd_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) { if (ret == REDIS_ERR) {
fprintf(stderr, "Redis GET %s:%s,%lu failed, redis server error.", mr_key_prefix[rule_list[i].op], fprintf(stderr, "Redis GET %s:%s,%lu failed, redis server error\n", mr_key_prefix[rule_list[i].op],
rule_list[i].table_name, rule_list[i].rule_id); rule_list[i].table_name, rule_list[i].rule_id);
error_happened = 1; error_happened = 1;
break; break;
@@ -194,7 +194,7 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
retry_ids[failed_cnt] = i; retry_ids[failed_cnt] = i;
failed_cnt++; failed_cnt++;
} else { } else {
fprintf(stderr, "Redis GET %s:%s,%lu failed",mr_key_prefix[rule_list[i].op], fprintf(stderr, "Redis GET %s:%s,%lu failed\n",mr_key_prefix[rule_list[i].op],
rule_list[i].table_name, rule_list[i].rule_id); rule_list[i].table_name, rule_list[i].rule_id);
error_happened = 1; error_happened = 1;
} }
@@ -222,8 +222,8 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
idx = retry_ids[i]; idx = retry_ids[i];
ret = maat_cmd_wrap_redis_get_reply(c, &reply); ret = maat_cmd_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) { if (ret == REDIS_ERR) {
fprintf(stderr, "redis command %s failed, redis server error.", redis_cmd); fprintf(stderr, "redis command %s failed, redis server error\n", redis_cmd);
free(retry_ids); FREE(retry_ids);
return -1; return -1;
} }
@@ -231,10 +231,10 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
rule_list[idx].table_line = maat_strdup(reply->str); rule_list[idx].table_line = maat_strdup(reply->str);
} else if(reply->type==REDIS_REPLY_ERROR) { } else if(reply->type==REDIS_REPLY_ERROR) {
//Deal with Redis response: "Loading Redis is loading the database in memory" //Deal with Redis response: "Loading Redis is loading the database in memory"
fprintf(stderr, "redis command %s error, reply type=%d, error str=%s", redis_cmd, reply->type, reply->str); fprintf(stderr, "redis command %s error, reply type=%d, error str=%s\n", redis_cmd, reply->type, reply->str);
} else { } else {
//Handle type "nil" //Handle type "nil"
fprintf(stderr, "redis command %s failed, reply type=%d", redis_cmd, reply->type); fprintf(stderr, "redis command %s failed, reply type=%d\n", redis_cmd, reply->type);
} }
freeReplyObject(reply); freeReplyObject(reply);
@@ -284,7 +284,7 @@ int get_inc_key_list(long long instance_version, long long target_version,
redisReply *reply = (redisReply *)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld", mr_status_sset, redisReply *reply = (redisReply *)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld", mr_status_sset,
instance_version,target_version); instance_version,target_version);
if (NULL == reply) { if (NULL == reply) {
fprintf(stderr, "GET %s failed with a NULL reply, error: %s.", mr_status_sset, c->errstr); fprintf(stderr, "GET %s failed with a NULL reply, error: %s\n", mr_status_sset, c->errstr);
return -1; return -1;
} }
@@ -299,7 +299,7 @@ int get_inc_key_list(long long instance_version, long long target_version,
redisReply *tmp_reply= maat_cmd_wrap_redis_command(c, "ZSCORE %s %s", mr_status_sset, reply->element[0]->str); redisReply *tmp_reply= maat_cmd_wrap_redis_command(c, "ZSCORE %s %s", mr_status_sset, reply->element[0]->str);
if (tmp_reply->type != REDIS_REPLY_STRING) { if (tmp_reply->type != REDIS_REPLY_STRING) {
fprintf(stderr, "ZSCORE %s %s failed Version: %lld->%lld", mr_status_sset, fprintf(stderr, "ZSCORE %s %s failed Version: %lld->%lld\n", mr_status_sset,
reply->element[0]->str, instance_version, target_version); reply->element[0]->str, instance_version, target_version);
freeReplyObject(tmp_reply); freeReplyObject(tmp_reply);
tmp_reply = NULL; tmp_reply = NULL;
@@ -317,7 +317,7 @@ int get_inc_key_list(long long instance_version, long long target_version,
} }
if (nearest_rule_version != instance_version + 1) { if (nearest_rule_version != instance_version + 1) {
fprintf(stdout, "Noncontinuous VERSION Redis: %lld MAAT: %lld.", fprintf(stdout, "Noncontinuous VERSION Redis: %lld MAAT: %lld\n",
nearest_rule_version, instance_version); nearest_rule_version, instance_version);
} }
@@ -326,12 +326,12 @@ int get_inc_key_list(long long instance_version, long long target_version,
char op_str[4] = {0}; char op_str[4] = {0};
struct serial_rule *s_rule = ALLOC(struct serial_rule, reply->elements); struct serial_rule *s_rule = ALLOC(struct serial_rule, reply->elements);
for (i = 0, j = 0; i < reply->elements; i++) { for (i = 0, j = 0; i < (int)reply->elements; i++) {
assert(reply->element[i]->type == REDIS_REPLY_STRING); assert(reply->element[i]->type == REDIS_REPLY_STRING);
int ret = sscanf(reply->element[i]->str, "%[^,],%[^,],%lu", op_str, int ret = sscanf(reply->element[i]->str, "%[^,],%[^,],%lu", op_str,
s_rule[j].table_name, &(s_rule[j].rule_id)); s_rule[j].table_name, &(s_rule[j].rule_id));
if (ret != 3 || s_rule[i].rule_id < 0) { if (ret != 3 || s_rule[i].rule_id < 0) {
fprintf(stderr, "Invalid Redis Key: %s", reply->element[i]->str); fprintf(stderr, "Invalid Redis Key: %s\n", reply->element[i]->str);
continue; continue;
} }
@@ -340,7 +340,7 @@ int get_inc_key_list(long long instance_version, long long target_version,
} else if(strncmp(op_str, "DEL", strlen("DEL")) == 0) { } else if(strncmp(op_str, "DEL", strlen("DEL")) == 0) {
s_rule[j].op = MAAT_OP_DEL; s_rule[j].op = MAAT_OP_DEL;
} else { } else {
fprintf(stderr, "Invalid Redis Key: %s", reply->element[i]->str); fprintf(stderr, "Invalid Redis Key: %s\n", reply->element[i]->str);
continue; continue;
} }
j++; j++;
@@ -377,7 +377,6 @@ struct serial_rule *serial_rule_clone(const struct serial_rule *s_rule)
new_rule->op = s_rule->op; new_rule->op = s_rule->op;
new_rule->rule_id = s_rule->rule_id; new_rule->rule_id = s_rule->rule_id;
new_rule->label_id = s_rule->label_id;
new_rule->timeout = s_rule->timeout; new_rule->timeout = s_rule->timeout;
memcpy(new_rule->table_name, s_rule->table_name, strlen(s_rule->table_name)); memcpy(new_rule->table_name, s_rule->table_name, strlen(s_rule->table_name));
new_rule->n_foreign = s_rule->n_foreign; new_rule->n_foreign = s_rule->n_foreign;
@@ -468,20 +467,20 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
redisReply *reply = (redisReply *)redisCommand(c, "GET MAAT_VERSION"); redisReply *reply = (redisReply *)redisCommand(c, "GET MAAT_VERSION");
if (reply != NULL) { if (reply != NULL) {
if (reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) { if (reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) {
fprintf(stderr, "GET MAAT_VERSION failed, maybe Redis is busy."); fprintf(stderr, "GET MAAT_VERSION failed, maybe Redis is busy\n");
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
return -1; return -1;
} }
} else { } else {
fprintf(stderr, "GET MAAT_VERSION failed with NULL reply, error: %s.", c->errstr); fprintf(stderr, "GET MAAT_VERSION failed with NULL reply, error: %s\n", c->errstr);
return -1; return -1;
} }
long long redis_version = maat_cmd_read_redis_integer(reply); long long redis_version = maat_cmd_read_redis_integer(reply);
if (redis_version < 0) { if (redis_version < 0) {
if (reply->type == REDIS_REPLY_ERROR) { if (reply->type == REDIS_REPLY_ERROR) {
fprintf(stderr, "Redis Communication error: %s.", reply->str); fprintf(stderr, "Redis Communication error: %s\n", reply->str);
} }
return -1; return -1;
} }
@@ -498,7 +497,7 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
} }
if (redis_version < instance_version) { if (redis_version < instance_version) {
fprintf(stderr, "VERSION roll back MAAT: %lld -> Redis: %lld.", instance_version, redis_version); fprintf(stderr, "VERSION roll back MAAT: %lld -> Redis: %lld\n", instance_version, redis_version);
goto FULL_UPDATE; goto FULL_UPDATE;
} }
@@ -522,12 +521,12 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
} while (0 == rule_num && target_version <= redis_version && 1 == cumulative_off); } while (0 == rule_num && target_version <= redis_version && 1 == cumulative_off);
if (0 == rule_num) { if (0 == rule_num) {
fprintf(stdout, "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s", mr_status_sset, fprintf(stdout, "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s\n", mr_status_sset,
instance_version, target_version-1, cumulative_off == 1 ? "OFF" : "ON"); instance_version, target_version-1, cumulative_off == 1 ? "OFF" : "ON");
return 0; return 0;
} }
fprintf(stdout, "Inc Update from instance_version %lld to %lld (%d entries).", fprintf(stdout, "Inc Update from instance_version %lld to %lld (%d entries)\n",
instance_version, target_version, rule_num); instance_version, target_version, rule_num);
*list = s_rule_array; *list = s_rule_array;
@@ -536,7 +535,7 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
return rule_num; return rule_num;
FULL_UPDATE: FULL_UPDATE:
fprintf(stdout, "Initiate full udpate from instance_version %lld to %lld.", instance_version, fprintf(stdout, "Initiate full update from instance_version %lld to %lld\n", instance_version,
desired_version == 0 ? redis_version : desired_version); desired_version == 0 ? redis_version : desired_version);
size_t append_cmd_cnt = 0; size_t append_cmd_cnt = 0;
int ret = redisAppendCommand(c, "MULTI"); int ret = redisAppendCommand(c, "MULTI");
@@ -558,12 +557,12 @@ FULL_UPDATE:
reply = maat_cmd_wrap_redis_command(c, "EXEC"); reply = maat_cmd_wrap_redis_command(c, "EXEC");
if (NULL == reply) { if (NULL == reply) {
fprintf(stderr, "Redis Communication error: %s.", c->errstr); fprintf(stderr, "Redis Communication error: %s\n", c->errstr);
return -1; return -1;
} }
if (reply->type != REDIS_REPLY_ARRAY) { if (reply->type != REDIS_REPLY_ARRAY) {
fprintf(stderr, "Invalid Redis Key List type %d", reply->type); fprintf(stderr, "Invalid Redis Key List type %d\n", reply->type);
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
return -1; return -1;
@@ -572,7 +571,7 @@ FULL_UPDATE:
*new_version = maat_cmd_read_redis_integer(reply->element[0]); *new_version = maat_cmd_read_redis_integer(reply->element[0]);
redisReply *sub_reply = reply->element[1]; redisReply *sub_reply = reply->element[1];
if (sub_reply->type != REDIS_REPLY_ARRAY) { if (sub_reply->type != REDIS_REPLY_ARRAY) {
fprintf(stderr, "Invalid Redis Key List type %d", sub_reply->type); fprintf(stderr, "Invalid Redis Key List type %d\n", sub_reply->type);
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
return -1; return -1;
@@ -582,7 +581,7 @@ FULL_UPDATE:
s_rule_array = ALLOC(struct serial_rule, sub_reply->elements); s_rule_array = ALLOC(struct serial_rule, sub_reply->elements);
for (i = 0, full_idx = 0; i < sub_reply->elements; i++) { for (i = 0, full_idx = 0; i < sub_reply->elements; i++) {
if (sub_reply->element[i]->type != REDIS_REPLY_STRING) { if (sub_reply->element[i]->type != REDIS_REPLY_STRING) {
fprintf(stderr, "Invalid Redis Key Type: %d", sub_reply->element[i]->type); fprintf(stderr, "Invalid Redis Key Type: %d\n", sub_reply->element[i]->type);
continue; continue;
} }
@@ -592,7 +591,7 @@ FULL_UPDATE:
s_rule_array[full_idx].op = MAAT_OP_ADD; s_rule_array[full_idx].op = MAAT_OP_ADD;
if (ret != 2 || s_rule_array[full_idx].rule_id < 0 || strlen(s_rule_array[full_idx].table_name) == 0) { if (ret != 2 || s_rule_array[full_idx].rule_id < 0 || strlen(s_rule_array[full_idx].table_name) == 0) {
fprintf(stderr, "Invalid Redis Key Format: %s", sub_reply->element[i]->str); fprintf(stderr, "Invalid Redis Key Format: %s\n", sub_reply->element[i]->str);
continue; continue;
} }
@@ -614,9 +613,11 @@ FULL_UPDATE:
struct serial_rule *changed_rule_array = NULL; struct serial_rule *changed_rule_array = NULL;
int changed_rule_num = get_inc_key_list(desired_version, redis_version, c, &changed_rule_array); int changed_rule_num = get_inc_key_list(desired_version, redis_version, c, &changed_rule_array);
if (changed_rule_num < 0) { if (changed_rule_num < 0) {
fprintf(stderr, "Recover history version %lld faild where as redis version is %lld.", desired_version, redis_version); fprintf(stderr, "Recover history version %lld faild where as redis version is %lld\n",
desired_version, redis_version);
} else if(0 == changed_rule_num) { } else if(0 == changed_rule_num) {
fprintf(stderr, "Nothing to recover from history version %lld to redis version is %lld.", desired_version, redis_version); fprintf(stderr, "Nothing to recover from history version %lld to redis version is %lld\n",
desired_version, redis_version);
} else { } else {
struct serial_rule *history_rule_array = NULL; struct serial_rule *history_rule_array = NULL;
ret = recovery_history_version(s_rule_array, full_idx, changed_rule_array, changed_rule_num, &history_rule_array); ret = recovery_history_version(s_rule_array, full_idx, changed_rule_array, changed_rule_num, &history_rule_array);
@@ -625,7 +626,7 @@ FULL_UPDATE:
s_rule_array = history_rule_array; s_rule_array = history_rule_array;
rule_num = ret; rule_num = ret;
*new_version = desired_version; *new_version = desired_version;
fprintf(stdout, "Successfully recovered from history version %lld to redis version is %lld.", fprintf(stdout, "Successfully recovered from history version %lld to redis version is %lld\n",
desired_version, redis_version); desired_version, redis_version);
} }
} }
@@ -634,7 +635,7 @@ FULL_UPDATE:
*list = s_rule_array; *list = s_rule_array;
*update_type = CM_UPDATE_TYPE_FULL; *update_type = CM_UPDATE_TYPE_FULL;
fprintf(stdout, "Full update %d keys of version %lld.", rule_num, *new_version); fprintf(stdout, "Full update %d keys of version %lld\n", rule_num, *new_version);
return rule_num ; return rule_num ;
} }
@@ -662,7 +663,7 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule
ret = remove(rule_list[i].f_keys[j].filename); ret = remove(rule_list[i].f_keys[j].filename);
if (ret == -1) { if (ret == -1) {
fprintf(stderr, "Foreign content file %s remove failed.", fprintf(stderr, "Foreign content file %s remove failed\n",
rule_list[i].f_keys[j].filename); rule_list[i].f_keys[j].filename);
} }
} }
@@ -693,7 +694,7 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule
for (i = 0; i < key_num; i++) { for (i = 0; i < key_num; i++) {
ret = maat_cmd_wrap_redis_get_reply(c, &reply); ret = maat_cmd_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) { if (ret == REDIS_ERR) {
fprintf(stderr, "Get %s,%lu foreign key %s content failed, redis server error.", fprintf(stderr, "Get %s,%lu foreign key %s content failed, redis server error\n",
rule_list[track[i].rule_idx].table_name, rule_list[track[i].rule_idx].table_name,
rule_list[track[i].rule_idx].rule_id, rule_list[track[i].rule_idx].rule_id,
rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key); rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key);
@@ -701,7 +702,7 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule
} }
if (reply->type != REDIS_REPLY_STRING) { if (reply->type != REDIS_REPLY_STRING) {
fprintf(stderr, "Get %s,%lu foreign key %s content failed.", fprintf(stderr, "Get %s,%lu foreign key %s content failed\n",
rule_list[track[i].rule_idx].table_name, rule_list[track[i].rule_idx].table_name,
rule_list[track[i].rule_idx].rule_id, rule_list[track[i].rule_idx].rule_id,
rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key); rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key);
@@ -710,7 +711,7 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule
s_rule = rule_list+track[i].rule_idx; s_rule = rule_list+track[i].rule_idx;
FILE *fp = fopen(s_rule->f_keys[track[i].foreign_idx].filename, "w"); FILE *fp = fopen(s_rule->f_keys[track[i].foreign_idx].filename, "w");
if (NULL == fp) { if (NULL == fp) {
fprintf(stderr, "Write foreign content failed: fopen %s error.", fprintf(stderr, "Write foreign content failed: fopen %s error\n",
s_rule->f_keys[track[i].foreign_idx].filename); s_rule->f_keys[track[i].foreign_idx].filename);
} else { } else {
fwrite(reply->str, 1, reply->len, fp); fwrite(reply->str, 1, reply->len, fp);
@@ -811,7 +812,7 @@ int redlock_try_lock(redisContext *c, const char *lock_name, long long expire)
return ret; return ret;
} }
long long _exec_serial_rule_begin(redisContext* c, size_t rule_num, size_t renew_rule_num, long long exec_serial_rule_begin(redisContext* c, size_t rule_num, size_t renew_rule_num,
int *renew_allowed, long long *transaction_version) int *renew_allowed, long long *transaction_version)
{ {
int ret = -1; int ret = -1;
@@ -860,7 +861,7 @@ const char* lua_exec_done=
"redis.call(\'del\', KEYS[4]);" "redis.call(\'del\', KEYS[4]);"
"redis.call(\'zadd\', KEYS[3], ARGV[1], maat_version);" "redis.call(\'zadd\', KEYS[3], ARGV[1], maat_version);"
"return maat_version;"; "return maat_version;";
redisReply* _exec_serial_rule_end(redisContext *c, const char *transaction_list, long long server_time, redisReply* exec_serial_rule_end(redisContext *c, const char *transaction_list, long long server_time,
int renew_allowed, struct expected_reply *expect_reply, size_t *cnt) int renew_allowed, struct expected_reply *expect_reply, size_t *cnt)
{ {
redisReply *data_reply = NULL; redisReply *data_reply = NULL;
@@ -889,7 +890,7 @@ redisReply* _exec_serial_rule_end(redisContext *c, const char *transaction_list,
return data_reply; return data_reply;
} }
void _exec_serial_rule(redisContext *c, const char *transaction_list, struct serial_rule *s_rule, size_t rule_num, void exec_serial_rule(redisContext *c, const char *transaction_list, struct serial_rule *s_rule, size_t rule_num,
struct expected_reply *expect_reply, size_t *cnt, size_t offset, int renew_allowed) struct expected_reply *expect_reply, size_t *cnt, size_t offset, int renew_allowed)
{ {
size_t i = 0; size_t i = 0;
@@ -926,21 +927,6 @@ void _exec_serial_rule(redisContext *c, const char *transaction_list, struct ser
(*cnt)++; (*cnt)++;
append_cmd_cnt++; append_cmd_cnt++;
} }
if (s_rule[i].label_id > 0)
{
redisAppendCommand(c, "ZADD %s %d %s,%lu",
mr_label_sset,
s_rule[i].label_id,
s_rule[i].table_name,
s_rule[i].rule_id);
expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1);
expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 0);
(*cnt)++;
append_cmd_cnt++;
}
break; break;
case MAAT_OP_DEL: case MAAT_OP_DEL:
redisAppendCommand(c, "RENAME %s:%s,%lu %s:%s,%lu", redisAppendCommand(c, "RENAME %s:%s,%lu %s:%s,%lu",
@@ -1050,7 +1036,7 @@ int mr_operation_success(redisReply *actual_reply, struct expected_reply *expect
return 0; return 0;
} }
int maat_cmd_exec_serial_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, long long server_time) int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, long long server_time)
{ {
size_t i = 0; size_t i = 0;
size_t rule_seq = 0; size_t rule_seq = 0;
@@ -1066,7 +1052,7 @@ int maat_cmd_exec_serial_rule(redisContext *c, struct serial_rule *s_rule, size_
char transaction_list[NAME_MAX * 2] = {0}; char transaction_list[NAME_MAX * 2] = {0};
long long transaction_version = 0; long long transaction_version = 0;
long long transaction_finished_version = 0; long long transaction_finished_version = 0;
size_t max_multi_cmd_num = MAX_REDIS_OP_PER_SRULE * serial_rule_num + 2;// 2 for operation in _exec_serial_rule_end() size_t max_multi_cmd_num = MAX_REDIS_OP_PER_SRULE * serial_rule_num + 2;// 2 for operation in exec_serial_rule_end()
struct expected_reply *expected_reply = ALLOC(struct expected_reply, max_multi_cmd_num); struct expected_reply *expected_reply = ALLOC(struct expected_reply, max_multi_cmd_num);
for (i = 0; i < serial_rule_num; i++) { for (i = 0; i < serial_rule_num; i++) {
@@ -1075,7 +1061,7 @@ int maat_cmd_exec_serial_rule(redisContext *c, struct serial_rule *s_rule, size_
} }
} }
int ret = _exec_serial_rule_begin(c, serial_rule_num, renew_num, &renew_allowed, &transaction_version); int ret = exec_serial_rule_begin(c, serial_rule_num, renew_num, &renew_allowed, &transaction_version);
//Preconditions for transaction are not satisfied. //Preconditions for transaction are not satisfied.
if (ret != 0) { if (ret != 0) {
success_cnt = -1; success_cnt = -1;
@@ -1088,13 +1074,13 @@ int maat_cmd_exec_serial_rule(redisContext *c, struct serial_rule *s_rule, size_
while (success_cnt < serial_rule_num) { while (success_cnt < serial_rule_num) {
size_t batch_cnt = MIN(serial_rule_num - success_cnt, max_redis_batch); size_t batch_cnt = MIN(serial_rule_num - success_cnt, max_redis_batch);
_exec_serial_rule(c, transaction_list, s_rule + success_cnt, batch_cnt, expected_reply, &multi_cmd_cnt, exec_serial_rule(c, transaction_list, s_rule + success_cnt, batch_cnt, expected_reply, &multi_cmd_cnt,
success_cnt, renew_allowed); success_cnt, renew_allowed);
assert(multi_cmd_cnt<max_multi_cmd_num); assert(multi_cmd_cnt<max_multi_cmd_num);
success_cnt+=batch_cnt; success_cnt+=batch_cnt;
} }
transaction_reply = _exec_serial_rule_end(c, transaction_list, server_time, renew_allowed, expected_reply, &multi_cmd_cnt); transaction_reply = exec_serial_rule_end(c, transaction_list, server_time, renew_allowed, expected_reply, &multi_cmd_cnt);
if (1 == mr_transaction_success(transaction_reply)) { if (1 == mr_transaction_success(transaction_reply)) {
assert(transaction_reply->elements == multi_cmd_cnt); assert(transaction_reply->elements == multi_cmd_cnt);
for (i = 0; i < multi_cmd_cnt; i++) { for (i = 0; i < multi_cmd_cnt; i++) {
@@ -1106,7 +1092,7 @@ int maat_cmd_exec_serial_rule(redisContext *c, struct serial_rule *s_rule, size_
continue; continue;
} }
rule_seq = expected_reply[i].s_rule_seq; rule_seq = expected_reply[i].s_rule_seq;
fprintf(stderr, "%s %s %lu failed, rule id maybe conflict or not exist.", fprintf(stderr, "%s %s %lu failed, rule id maybe conflict or not exist\n",
mr_op_str[s_rule[rule_seq].op], s_rule[rule_seq].table_name, mr_op_str[s_rule[rule_seq].op], s_rule[rule_seq].table_name,
s_rule[rule_seq].rule_id); s_rule[rule_seq].rule_id);
success_cnt--; success_cnt--;
@@ -1118,7 +1104,7 @@ int maat_cmd_exec_serial_rule(redisContext *c, struct serial_rule *s_rule, size_
if (transaction_version > 0) { if (transaction_version > 0) {
transaction_finished_version = maat_cmd_read_redis_integer(transaction_reply->element[multi_cmd_cnt-1]); transaction_finished_version = maat_cmd_read_redis_integer(transaction_reply->element[multi_cmd_cnt-1]);
fprintf(stdout, "Redis transaction MAAT_PRE_VER = %lld , MAAT_VERSION = %lld ", fprintf(stdout, "Redis transaction MAAT_PRE_VER = %lld , MAAT_VERSION = %lld\n",
transaction_version, transaction_finished_version); transaction_version, transaction_finished_version);
} }
@@ -1129,7 +1115,7 @@ error_out:
if (renew_num > 0 && renew_allowed != 1) { if (renew_num > 0 && renew_allowed != 1) {
for (i = 0; i < (unsigned int)serial_rule_num; i++) { for (i = 0; i < (unsigned int)serial_rule_num; i++) {
if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) { if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) {
fprintf(stdout, "%s %s %lu is not allowed due to lock contention.", fprintf(stdout, "%s %s %lu is not allowed due to lock contention\n",
mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name, mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name,
s_rule[i].rule_id); s_rule[i].rule_id);
} }
@@ -1205,7 +1191,7 @@ void cleanup_update_status(redisContext *c)
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
fprintf(stdout, "Clean up update status from version %lld to %lld (%lld versions, %lld entries).", fprintf(stdout, "Clean up update status from version %lld to %lld (%lld versions, %lld entries)\n",
version_lower_bound, version_upper_bound, version_num, entry_num); version_lower_bound, version_upper_bound, version_num, entry_num);
return; return;
@@ -1243,13 +1229,13 @@ void check_maat_expiration(redisContext *c)
freeReplyObject(data_reply); freeReplyObject(data_reply);
data_reply = NULL; data_reply = NULL;
int success_cnt = maat_cmd_exec_serial_rule(c, s_rule, s_rule_num, server_time); int success_cnt = maat_cmd_write_rule(c, s_rule, s_rule_num, server_time);
if (success_cnt < 0) { if (success_cnt < 0) {
fprintf(stderr, "maat_cmd_exec_serial_rule failed.\n"); fprintf(stderr, "maat_cmd_write_rule failed.\n");
} else if (success_cnt == (int)s_rule_num) { } else if (success_cnt == (int)s_rule_num) {
fprintf(stdout, "Succesfully expired %zu rules in Redis.", s_rule_num); fprintf(stdout, "Succesfully expired %zu rules in Redis\n", s_rule_num);
} else { } else {
fprintf(stderr, "Failed to expired %d of %zu rules in Redis, try later.", fprintf(stderr, "Failed to expired %d of %zu rules in Redis, try later\n",
s_rule_num - success_cnt, s_rule_num); s_rule_num - success_cnt, s_rule_num);
} }
@@ -1292,7 +1278,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
if (mr_ctx->read_ctx != NULL) { if (mr_ctx->read_ctx != NULL) {
redisFree(mr_ctx->read_ctx); redisFree(mr_ctx->read_ctx);
} }
fprintf(stdout, "Reconnecting..."); fprintf(stdout, "Reconnecting...\n");
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
if (NULL == mr_ctx->read_ctx) { if (NULL == mr_ctx->read_ctx) {
@@ -1329,7 +1315,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
if (ret < 0) { if (ret < 0) {
redisFree(mr_ctx->read_ctx); redisFree(mr_ctx->read_ctx);
mr_ctx->read_ctx = NULL; mr_ctx->read_ctx = NULL;
fprintf(stderr, "Get Redis value failed, abandon update and close connection."); fprintf(stderr, "Get Redis value failed, abandon update and close connection\n");
goto clean_up; goto clean_up;
} }
@@ -1340,7 +1326,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
} }
if (empty_value_num == rule_num) { if (empty_value_num == rule_num) {
fprintf(stdout, "All %d rules are empty, abandon update.", empty_value_num); fprintf(stdout, "All %d rules are empty, abandon update\n", empty_value_num);
goto clean_up; goto clean_up;
} }
@@ -1351,7 +1337,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
} }
start_fn(new_version, update_type, u_param); start_fn(new_version, update_type, u_param);
fprintf(stdout, "Start %s update: %lld -> %lld (%d entries).", fprintf(stdout, "Start %s update: %lld -> %lld (%d entries)\n",
update_type==CM_UPDATE_TYPE_INC?"INC":"FULL", version, new_version, rule_num); update_type==CM_UPDATE_TYPE_INC?"INC":"FULL", version, new_version, rule_num);
for (i = 0; i < rule_num; i++) { for (i = 0; i < rule_num; i++) {
@@ -1374,7 +1360,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
valid_column = table_schema_get_valid_flag_column(table_schema); valid_column = table_schema_get_valid_flag_column(table_schema);
ret = invalidate_line(rule_list[i].table_line, table_type, valid_column); ret = invalidate_line(rule_list[i].table_line, table_type, valid_column);
if (ret < 0) { if (ret < 0) {
fprintf(stdout, "Invalidate line failed, invaid format %s .", rule_list[i].table_line); fprintf(stdout, "Invalidate line failed, invaid format %s\n", rule_list[i].table_line);
continue; continue;
} }
} }
@@ -1390,13 +1376,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
finish_fn(u_param); finish_fn(u_param);
if (call_update_num < rule_num) { if (call_update_num < rule_num) {
fprintf(stdout, "Load %d entries to match engine, no tablle: %d, empty value: %d.", fprintf(stdout, "Load %d entries to match engine, no table: %d, empty value: %d\n",
call_update_num, no_table_num, empty_value_num); call_update_num, no_table_num, empty_value_num);
} }
clean_up: clean_up:
for (i = 0; i < rule_num; i++) { for (i = 0; i < rule_num; i++) {
maat_cmd_empty_serial_rule(rule_list + i); maat_cmd_clear_rule_cache(rule_list + i);
} }
FREE(rule_list); FREE(rule_list);

18
src/maat_rule.cpp
View File

@@ -66,7 +66,7 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt)
maat_rt->table_rt_mgr = NULL; maat_rt->table_rt_mgr = NULL;
} }
free(maat_rt); FREE(maat_rt);
} }
int maat_runtime_updating_flag(struct maat_runtime *maat_rt) int maat_runtime_updating_flag(struct maat_runtime *maat_rt)
@@ -122,9 +122,11 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param)
} }
struct table_item *table_item = table_schema_line_to_item(line, table_schema); struct table_item *table_item = table_schema_line_to_item(line, table_schema);
if (table_item != NULL) {
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id); struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id);
table_runtime_update(table_rt, table_schema, line, table_item); table_runtime_update(table_rt, table_schema, line, table_item);
free(table_item); FREE(table_item);
}
return 0; return 0;
} }
@@ -185,7 +187,7 @@ void *rule_monitor_loop(void *arg)
pthread_mutex_lock(&(maat_instance->background_update_mutex)); pthread_mutex_lock(&(maat_instance->background_update_mutex));
/* if deferred load on */ /* if deferred load on */
if (maat_instance->deferred_load != 0) { if (maat_instance->deferred_load != 0) {
fprintf(stdout, "Deferred Loading ON, updating in %s.", __func__); fprintf(stdout, "Deferred Loading ON, updating in %s\n", __func__);
maat_read_full_config(maat_instance); maat_read_full_config(maat_instance);
} }
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_instance->background_update_mutex));
@@ -207,7 +209,7 @@ void *rule_monitor_loop(void *arg)
break; break;
case DATA_SOURCE_IRIS_FILE: case DATA_SOURCE_IRIS_FILE:
config_monitor_traverse(maat_instance->maat_version, config_monitor_traverse(maat_instance->maat_version,
maat_instance->iris_ctx.inc_dir, maat_instance->iris_ctx.inc_idx_dir,
maat_start_cb, maat_start_cb,
maat_update_cb, maat_update_cb,
maat_finish_cb, maat_finish_cb,
@@ -222,7 +224,7 @@ void *rule_monitor_loop(void *arg)
if (0 != strcmp(md5_tmp, maat_instance->json_ctx.effective_json_md5)) { if (0 != strcmp(md5_tmp, maat_instance->json_ctx.effective_json_md5)) {
ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str)); ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str));
if (ret < 0) { if (ret < 0) {
fprintf(stdout, "Maat re-initiate with JSON file %s (md5=%s)failed: %s", fprintf(stdout, "Maat re-initiate with JSON file %s (md5=%s)failed: %s\n",
maat_instance->json_ctx.json_file, md5_tmp, err_str); maat_instance->json_ctx.json_file, md5_tmp, err_str);
} else { } else {
config_monitor_traverse(0, maat_instance->json_ctx.iris_file, config_monitor_traverse(0, maat_instance->json_ctx.iris_file,
@@ -230,7 +232,7 @@ void *rule_monitor_loop(void *arg)
maat_update_cb, maat_update_cb,
maat_finish_cb, maat_finish_cb,
maat_instance); maat_instance);
fprintf(stdout, "Maat re-initiate with JSON file %s success, md5: %s", fprintf(stdout, "Maat re-initiate with JSON file %s success, md5: %s\n",
maat_instance->json_ctx.json_file, md5_tmp); maat_instance->json_ctx.json_file, md5_tmp);
} }
} }
@@ -246,10 +248,10 @@ void *rule_monitor_loop(void *arg)
if (old_maat_rt != NULL) { if (old_maat_rt != NULL) {
if (maat_instance->maat_rt->version > old_maat_rt->version) { if (maat_instance->maat_rt->version > old_maat_rt->version) {
fprintf(stdout, "Maat version updated %lld -> %lld.", fprintf(stdout, "Maat version updated %lld -> %lld\n",
old_maat_rt->version, maat_instance->maat_rt->version); old_maat_rt->version, maat_instance->maat_rt->version);
} else { } else {
fprintf(stdout, "Maat version roll back %lld -> %lld.", fprintf(stdout, "Maat version roll back %lld -> %lld\n",
old_maat_rt->version, maat_instance->maat_rt->version); old_maat_rt->version, maat_instance->maat_rt->version);
} }
maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt, (void (*)(void*))maat_runtime_destroy); maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt, (void (*)(void*))maat_runtime_destroy);

42
src/maat_table_runtime.cpp
View File

@@ -82,6 +82,10 @@ struct table_runtime_manager {
void plugin_ex_data_free(void *user_ctx, void *data) void plugin_ex_data_free(void *user_ctx, void *data)
{ {
if (NULL == user_ctx || NULL == data) {
return;
}
struct plugin_user_ctx *ctx = (struct plugin_user_ctx *)user_ctx; struct plugin_user_ctx *ctx = (struct plugin_user_ctx *)user_ctx;
long argl = ctx->ex_schema->argl; long argl = ctx->ex_schema->argl;
void *argp = ctx->ex_schema->argp; void *argp = ctx->ex_schema->argp;
@@ -90,18 +94,19 @@ void plugin_ex_data_free(void *user_ctx, void *data)
void expr_rule_free(and_expr_t *expr_rule) void expr_rule_free(and_expr_t *expr_rule)
{ {
if (expr_rule != NULL) { if (NULL == expr_rule) {
for (size_t i = 0; i < expr_rule->n_patterns; i++) { return;
free(expr_rule->patterns[i].pat);
expr_rule->patterns[i].pat = NULL;
} }
for (size_t i = 0; i < expr_rule->n_patterns; i++) {
FREE(expr_rule->patterns[i].pat);
} }
} }
void expr_ex_data_free(void *user_ctx, void *data) void expr_ex_data_free(void *user_ctx, void *data)
{ {
and_expr_t *expr_rule = (and_expr_t *)data; and_expr_t *expr_rule = (and_expr_t *)data;
expr_rule_free(expr_rule); expr_rule_free(expr_rule);
free(data); FREE(data);
} }
struct table_runtime *table_runtime_new(struct table_schema *table_schema, int max_thread_num, struct maat_garbage_bin* bin) struct table_runtime *table_runtime_new(struct table_schema *table_schema, int max_thread_num, struct maat_garbage_bin* bin)
@@ -131,6 +136,10 @@ struct table_runtime *table_runtime_new(struct table_schema *table_schema, int m
void table_runtime_free(struct table_runtime * table_rt) void table_runtime_free(struct table_runtime * table_rt)
{ {
if (NULL == table_rt) {
return;
}
switch (table_rt->table_type) { switch (table_rt->table_type) {
case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR:
adapter_hs_destroy(table_rt->expr_rt.hs); adapter_hs_destroy(table_rt->expr_rt.hs);
@@ -147,7 +156,7 @@ void table_runtime_free(struct table_runtime * table_rt)
break; break;
} }
free(table_rt); FREE(table_rt);
} }
struct table_runtime_manager * struct table_runtime_manager *
@@ -184,9 +193,8 @@ void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr)
table_rt_mgr->table_rt[i] = NULL; table_rt_mgr->table_rt[i] = NULL;
} }
free(table_rt_mgr->table_rt); FREE(table_rt_mgr->table_rt);
table_rt_mgr->table_rt = NULL; FREE(table_rt_mgr);
free(table_rt_mgr);
} }
struct table_runtime *table_runtime_get(struct table_runtime_manager *table_rt_mgr, int table_id) struct table_runtime *table_runtime_get(struct table_runtime_manager *table_rt_mgr, int table_id)
@@ -297,7 +305,7 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item)
} }
if (i >= MAAT_MAX_EXPR_ITEM_NUM) { if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
fprintf(stderr, "item_id:%d too many patterns", expr_item->item_id); fprintf(stderr, "item_id:%d too many patterns\n", expr_item->item_id);
return NULL; return NULL;
} }
@@ -340,7 +348,7 @@ void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key
//delete //delete
data = rcu_hash_find(expr_rt->htable, key, key_len); data = rcu_hash_find(expr_rt->htable, key, key_len);
if (NULL == data) { if (NULL == data) {
fprintf(stderr, "the key:%s not exist, so can't be deleted.", key); fprintf(stderr, "the key:%s not exist, so can't be deleted\n", key);
return; return;
} }
rcu_hash_del(expr_rt->htable, key, key_len); rcu_hash_del(expr_rt->htable, key, key_len);
@@ -348,7 +356,7 @@ void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key
//add //add
data = rcu_hash_find(expr_rt->htable, key, key_len); data = rcu_hash_find(expr_rt->htable, key, key_len);
if (data != NULL) { if (data != NULL) {
fprintf(stderr, "the key:%s already exist, so can't be added.", key); fprintf(stderr, "the key:%s already exist, so can't be added\n", key);
return; return;
} }
and_expr_t *data = ALLOC(and_expr_t, 1); and_expr_t *data = ALLOC(and_expr_t, 1);
@@ -443,7 +451,7 @@ void table_runtime_update(struct table_runtime *table_rt, struct table_schema *t
ip_rule = ip_plugin_item_to_ip_rule(&table_item->ip_plugin_item); ip_rule = ip_plugin_item_to_ip_rule(&table_item->ip_plugin_item);
key = (char *)&(table_item->ip_plugin_item.item_id); key = (char *)&(table_item->ip_plugin_item.item_id);
ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, row, key, sizeof(int), ip_rule, is_valid); ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, row, key, sizeof(int), ip_rule, is_valid);
free(ip_rule); FREE(ip_rule);
break; break;
default: default:
break; break;
@@ -489,8 +497,8 @@ int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread)
rule_cnt = rcu_hash_list_updating_data(expr_rt->htable, &ex_data_array); rule_cnt = rcu_hash_list_updating_data(expr_rt->htable, &ex_data_array);
assert(rule_cnt == 0); assert(rule_cnt == 0);
free(rules); FREE(rules);
free(ex_data_array); FREE(ex_data_array);
return ret; return ret;
} }
@@ -535,8 +543,8 @@ int ip_plugin_runtime_commit(struct table_runtime *table_rt)
ex_data_runtime_commit(ip_plugin_rt->ex_data_rt); ex_data_runtime_commit(ip_plugin_rt->ex_data_rt);
table_rt->rule_num = ex_data_runtime_ex_data_count(ip_plugin_rt->ex_data_rt); table_rt->rule_num = ex_data_runtime_ex_data_count(ip_plugin_rt->ex_data_rt);
free(rules); FREE(rules);
free(ex_data_array); FREE(ex_data_array);
return ret; return ret;
} }

50
src/maat_table_schema.cpp
View File

@@ -92,7 +92,7 @@ struct table_schema *table_schema_new(void)
void table_schema_free(struct table_schema *ptable) void table_schema_free(struct table_schema *ptable)
{ {
free(ptable); FREE(ptable);
} }
int read_expr_table_schema(cJSON *root, struct table_schema *ptable, int read_expr_table_schema(cJSON *root, struct table_schema *ptable,
@@ -110,7 +110,7 @@ int read_expr_table_schema(cJSON *root, struct table_schema *ptable,
json = cJSON_GetObjectItem(root, "table_name"); json = cJSON_GetObjectItem(root, "table_name");
if (json != NULL && json->type == cJSON_String) { if (json != NULL && json->type == cJSON_String) {
if (strlen(json->valuestring) >= NAME_MAX) { if (strlen(json->valuestring) >= NAME_MAX) {
fprintf(stderr, "table name %s length too long", json->valuestring); fprintf(stderr, "table name %s length too long\n", json->valuestring);
return -1; return -1;
} }
memcpy(ptable->table_name, json->valuestring, strlen(json->valuestring)); memcpy(ptable->table_name, json->valuestring, strlen(json->valuestring));
@@ -121,7 +121,7 @@ int read_expr_table_schema(cJSON *root, struct table_schema *ptable,
if (json != NULL && json->type == cJSON_String) { if (json != NULL && json->type == cJSON_String) {
int ret = maat_kv_read(reserved_word_map, json->valuestring, (int*)&(ptable->expr.scan_mode)); int ret = maat_kv_read(reserved_word_map, json->valuestring, (int*)&(ptable->expr.scan_mode));
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "scan_mode %s illegal", json->valuestring); fprintf(stderr, "scan_mode %s illegal\n", json->valuestring);
return -1; return -1;
} }
read_cnt++; read_cnt++;
@@ -141,7 +141,7 @@ int read_expr_table_schema(cJSON *root, struct table_schema *ptable,
json = cJSON_GetObjectItem(root, "rule"); json = cJSON_GetObjectItem(root, "rule");
if (json == NULL || json->type != cJSON_Object) { if (json == NULL || json->type != cJSON_Object) {
fprintf(stderr, "table %s has no rule", ptable->table_name); fprintf(stderr, "table %s has no rule\n", ptable->table_name);
return -1; return -1;
} }
@@ -221,7 +221,7 @@ int read_plugin_table_schema(cJSON *root, struct table_schema *ptable)
json = cJSON_GetObjectItem(root, "table_name"); json = cJSON_GetObjectItem(root, "table_name");
if (json != NULL && json->type == cJSON_String) { if (json != NULL && json->type == cJSON_String) {
if (strlen(json->valuestring) >= NAME_MAX) { if (strlen(json->valuestring) >= NAME_MAX) {
fprintf(stderr, "table name %s length too long", json->valuestring); fprintf(stderr, "table name %s length too long\n", json->valuestring);
return -1; return -1;
} }
memcpy(ptable->table_name, json->valuestring, strlen(json->valuestring)); memcpy(ptable->table_name, json->valuestring, strlen(json->valuestring));
@@ -236,7 +236,7 @@ int read_plugin_table_schema(cJSON *root, struct table_schema *ptable)
json = cJSON_GetObjectItem(root, "rule"); json = cJSON_GetObjectItem(root, "rule");
if (json == NULL || json->type != cJSON_Object) { if (json == NULL || json->type != cJSON_Object) {
fprintf(stderr, "table %s has no rule", ptable->table_name); fprintf(stderr, "table %s has no rule\n", ptable->table_name);
return -1; return -1;
} }
@@ -296,7 +296,7 @@ int read_ip_plugin_table_schema(cJSON *root, struct table_schema *ptable)
json = cJSON_GetObjectItem(root, "table_name"); json = cJSON_GetObjectItem(root, "table_name");
if (json != NULL && json->type == cJSON_String) { if (json != NULL && json->type == cJSON_String) {
if (strlen(json->valuestring) >= NAME_MAX) { if (strlen(json->valuestring) >= NAME_MAX) {
fprintf(stderr, "table name %s length too long", json->valuestring); fprintf(stderr, "table name %s length too long\n", json->valuestring);
return -1; return -1;
} }
memcpy(ptable->table_name, json->valuestring, strlen(json->valuestring)); memcpy(ptable->table_name, json->valuestring, strlen(json->valuestring));
@@ -311,7 +311,7 @@ int read_ip_plugin_table_schema(cJSON *root, struct table_schema *ptable)
json = cJSON_GetObjectItem(root, "rule"); json = cJSON_GetObjectItem(root, "rule");
if (NULL == json || json->type != cJSON_Object) { if (NULL == json || json->type != cJSON_Object) {
fprintf(stderr, "table %s has no rule", ptable->table_name); fprintf(stderr, "table %s has no rule\n", ptable->table_name);
return -1; return -1;
} }
@@ -356,7 +356,7 @@ int table_schema_populate(cJSON *json, struct table_schema *ptable,
if (item != NULL && item->type == cJSON_String) { if (item != NULL && item->type == cJSON_String) {
ret = maat_kv_read(reserved_word_map, item->valuestring, (int*)&(ptable->table_type)); ret = maat_kv_read(reserved_word_map, item->valuestring, (int*)&(ptable->table_type));
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "table_type %s is illegal", item->valuestring); fprintf(stderr, "table_type %s is illegal\n", item->valuestring);
return -1; return -1;
} }
} }
@@ -406,14 +406,14 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_
cJSON *root = NULL; cJSON *root = NULL;
root = cJSON_Parse((const char *)json_buff); root = cJSON_Parse((const char *)json_buff);
if (!root) { if (!root) {
fprintf(stderr, "Error before: %-200.200s", cJSON_GetErrorPtr()); fprintf(stderr, "Error before: %-200.200s\n", cJSON_GetErrorPtr());
free(json_buff); FREE(json_buff);
return NULL; return NULL;
} }
int json_array_size = cJSON_GetArraySize(root); int json_array_size = cJSON_GetArraySize(root);
if (json_array_size <= 0) { if (json_array_size <= 0) {
fprintf(stderr, "invalid json content in %s", table_info_path); fprintf(stderr, "invalid json content in %s\n", table_info_path);
free(json_buff); free(json_buff);
return NULL; return NULL;
} }
@@ -440,19 +440,19 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_
struct table_schema *table_schema = table_schema_new(); struct table_schema *table_schema = table_schema_new();
int ret = table_schema_populate(json, table_schema, reserved_word_map); int ret = table_schema_populate(json, table_schema, reserved_word_map);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "Maat populate table info error, table_id:%d", table_schema->table_id); fprintf(stderr, "Maat populate table info error, table_id:%d\n", table_schema->table_id);
goto error; goto error;
} }
if (table_schema->table_id >= MAX_TABLE_NUM) { if (table_schema->table_id >= MAX_TABLE_NUM) {
fprintf(stderr, "Maat read table info %s:%d error: table id %u > %d.\n", fprintf(stderr, "Maat read table info %s:%d error: table id %u > %d\n",
table_info_path, i, table_schema->table_id, MAX_TABLE_NUM); table_info_path, i, table_schema->table_id, MAX_TABLE_NUM);
goto error; goto error;
} }
ret = maat_kv_register(table_schema_mgr->tablename2id_map, table_schema->table_name, table_schema->table_id); ret = maat_kv_register(table_schema_mgr->tablename2id_map, table_schema->table_name, table_schema->table_id);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "Duplicate table %s of table id %d", table_schema->table_name, table_schema->table_id); fprintf(stderr, "Duplicate table %s of table id %d\n", table_schema->table_name, table_schema->table_id);
goto error; goto error;
} }
@@ -473,7 +473,7 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_
} }
maat_kv_store_free(reserved_word_map); maat_kv_store_free(reserved_word_map);
free(json_buff); FREE(json_buff);
return table_schema_mgr; return table_schema_mgr;
} }
@@ -855,7 +855,7 @@ int populate_expr_table_item(const char *line, struct expr_table_schema *expr_sc
expr_item->is_case_sensitive = TRUE; expr_item->is_case_sensitive = TRUE;
break; break;
default: default:
fprintf(stderr, "invalid hexbin value of expr table"); fprintf(stderr, "invalid hexbin value of expr table\n");
return -1; return -1;
} }
ret = get_column_pos(line, expr_schema->is_valid_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->is_valid_column, &column_offset, &column_len);
@@ -878,7 +878,7 @@ int populate_plugin_table_item(const char *line, struct plugin_table_schema *plu
} }
if (column_len > MAX_KEYWORDS_STR) { if (column_len > MAX_KEYWORDS_STR) {
fprintf(stderr, "plugin table's key len:%zu, exceed %d", column_len, MAX_KEYWORDS_STR); fprintf(stderr, "plugin table's key len:%zu, exceed %d\n", column_len, MAX_KEYWORDS_STR);
return -1; return -1;
} }
memcpy(plugin_item->key, (line + column_offset), column_len); memcpy(plugin_item->key, (line + column_offset), column_len);
@@ -971,7 +971,7 @@ table_schema_line_to_item(const char *line, struct table_schema *table_schema)
} }
return table_item; return table_item;
error: error:
free(table_item); FREE(table_item);
return NULL; return NULL;
} }
@@ -996,19 +996,19 @@ int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema,
{ {
if (NULL == table_schema || NULL == new_func || NULL == free_func || NULL == dup_func) { if (NULL == table_schema || NULL == new_func || NULL == free_func || NULL == dup_func) {
assert(0); assert(0);
fprintf(stderr, "%s failed: invalid parameter", __FUNCTION__); fprintf(stderr, "%s failed: invalid parameter\n", __FUNCTION__);
return -1; return -1;
} }
struct ex_data_schema *ex_schema = plugin_table_schema_get_ex_data_schema(table_schema); struct ex_data_schema *ex_schema = plugin_table_schema_get_ex_data_schema(table_schema);
if (NULL == ex_schema) { if (NULL == ex_schema) {
fprintf(stderr, "Error: %s, target table is not a valid plugin table.", __FUNCTION__); fprintf(stderr, "Error: %s, target table is not a valid plugin table\n", __FUNCTION__);
return -1; return -1;
} }
if (ex_schema->set_flag) { if (ex_schema->set_flag) {
assert(0); assert(0);
fprintf(stderr, "Error: %s, EX data schema already registed.", __FUNCTION__); fprintf(stderr, "Error: %s, EX data schema already registed\n", __FUNCTION__);
return -1; return -1;
} }
@@ -1078,7 +1078,7 @@ int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_m
struct table_schema *ptable = table_schema_get(table_schema_mgr, table_id); struct table_schema *ptable = table_schema_get(table_schema_mgr, table_id);
if (NULL == ptable) { if (NULL == ptable) {
fprintf(stderr, "table_id:%d unregistered, can't add callback func", table_id); fprintf(stderr, "table_id:%d unregistered, can't add callback func\n", table_id);
return -1; return -1;
} }
@@ -1086,7 +1086,7 @@ int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_m
case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS: case TABLE_TYPE_EXPR_PLUS:
case TABLE_TYPE_IP: case TABLE_TYPE_IP:
fprintf(stderr, "table %s is not plugin type.", ptable->table_name); fprintf(stderr, "table %s is not plugin type\n", ptable->table_name);
return -1; return -1;
default: default:
break; break;
@@ -1095,7 +1095,7 @@ int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_m
struct plugin_table_schema *plugin_schema = &(ptable->plugin); struct plugin_table_schema *plugin_schema = &(ptable->plugin);
size_t idx = plugin_schema->cb_plug_cnt; size_t idx = plugin_schema->cb_plug_cnt;
if (idx == MAX_PLUGIN_PER_TABLE) { if (idx == MAX_PLUGIN_PER_TABLE) {
fprintf(stderr, "the plugin number of table %s exceed maxium:%d", fprintf(stderr, "the plugin number of table %s exceed maxium:%d\n",
ptable->table_name, MAX_PLUGIN_PER_TABLE); ptable->table_name, MAX_PLUGIN_PER_TABLE);
return -1; return -1;
} }

22
src/maat_utils.cpp
View File

@@ -15,6 +15,7 @@
#include <openssl/md5.h> #include <openssl/md5.h>
#include <openssl/evp.h> #include <openssl/evp.h>
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
char *maat_strdup(const char *s) char *maat_strdup(const char *s)
@@ -51,7 +52,7 @@ int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len
} }
i++; i++;
} }
free(dup_line); FREE(dup_line);
return ret; return ret;
} }
@@ -77,8 +78,7 @@ int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *o
*pp_out = (unsigned char *)calloc(1, *out_sz+1); *pp_out = (unsigned char *)calloc(1, *out_sz+1);
read_size = fread(*pp_out, 1, *out_sz, fp); read_size = fread(*pp_out, 1, *out_sz, fp);
if (read_size != *out_sz) { if (read_size != *out_sz) {
free(*pp_out); FREE(*pp_out);
pp_out = NULL;
return -1; return -1;
} }
@@ -196,6 +196,13 @@ int system_cmd_mkdir(const char *path)
return system(cmd); return system(cmd);
} }
int system_cmd_rmdir(const char *dir)
{
char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
snprintf(cmd,sizeof(cmd), "rm %s -rf", dir);
return system(cmd);
}
char *md5_file(const char *filename, char *md5string) char *md5_file(const char *filename, char *md5string)
{ {
unsigned char md5[MD5_DIGEST_LENGTH] = {0}; unsigned char md5[MD5_DIGEST_LENGTH] = {0};
@@ -217,7 +224,8 @@ char *md5_file(const char *filename, char *md5string)
sprintf(&md5string[i*2], "%02x", (unsigned int)md5[i]); sprintf(&md5string[i*2], "%02x", (unsigned int)md5[i]);
} }
free(file_buff); FREE(file_buff);
return md5string; return md5string;
} }
@@ -284,8 +292,7 @@ int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_ou
return 0; return 0;
error_out: error_out:
free(*pp_out); FREE(*pp_out);
*pp_out = NULL;
return -1; return -1;
} }
@@ -300,8 +307,7 @@ int decrypt_open(const char* file_name, const char* key, const char* algorithm,
} }
ret = crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm, 0, err_str, err_str_sz); ret = crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm, 0, err_str, err_str_sz);
free(file_buff); FREE(file_buff);
file_buff = NULL;
return ret; return ret;
} }

8
src/rcu_hash.cpp
View File

@@ -59,7 +59,7 @@ void rcu_hash_garbage_queue_free(struct rcu_hash_garbage_q* garbage_q)
while ((p = TAILQ_FIRST(garbage_q)) != NULL) { while ((p = TAILQ_FIRST(garbage_q)) != NULL) {
p->garbage_free(p->garbage); p->garbage_free(p->garbage);
TAILQ_REMOVE(garbage_q, p, entries); TAILQ_REMOVE(garbage_q, p, entries);
free(p); FREE(p);
} }
} }
@@ -84,13 +84,13 @@ void rcu_hash_node_free(struct rcu_hash_node *node)
} }
if (node->key != NULL) { if (node->key != NULL) {
free(node->key); FREE(node->key);
} }
if (node->data != NULL) { if (node->data != NULL) {
node->htable->data_free_fn(node->htable->user_ctx, node->data); node->htable->data_free_fn(node->htable->user_ctx, node->data);
} }
free(node); FREE(node);
} }
struct rcu_hash_table *rcu_hash_new(rcu_hash_data_free_fn *free_fn) struct rcu_hash_table *rcu_hash_new(rcu_hash_data_free_fn *free_fn)
@@ -134,7 +134,7 @@ void rcu_hash_free(struct rcu_hash_table *htable)
rcu_hash_garbage_queue_free(&(htable->garbage_q)); rcu_hash_garbage_queue_free(&(htable->garbage_q));
pthread_mutex_destroy(&htable->update_mutex); pthread_mutex_destroy(&htable->update_mutex);
free(htable); FREE(htable);
} }
void rcu_hash_set_user_ctx(struct rcu_hash_table *htable, void *user_ctx) void rcu_hash_set_user_ctx(struct rcu_hash_table *htable, void *user_ctx)

3
test/CMakeLists.txt
View File

@@ -8,6 +8,9 @@ target_link_libraries(maat_api_gtest maat_frame_static gtest_static)
add_executable(rcu_hash_gtest rcu_hash_gtest.cpp) add_executable(rcu_hash_gtest rcu_hash_gtest.cpp)
target_link_libraries(rcu_hash_gtest maat_frame_static gtest_static) target_link_libraries(rcu_hash_gtest maat_frame_static gtest_static)
add_executable(maat_input_mode_gtest maat_input_mode_gtest.cpp)
target_link_libraries(maat_input_mode_gtest maat_frame_static gtest_static)
add_executable(maat_framework_gtest maat_framework_gtest.cpp) add_executable(maat_framework_gtest maat_framework_gtest.cpp)
target_link_libraries(maat_framework_gtest maat_frame_static gtest_static) target_link_libraries(maat_framework_gtest maat_frame_static gtest_static)

1
test/maat_framework_gtest.cpp
View File

@@ -247,7 +247,6 @@ int main(int argc, char ** argv)
ret=RUN_ALL_TESTS(); ret=RUN_ALL_TESTS();
maat_free(g_maat_instance); maat_free(g_maat_instance);
g_maat_instance = NULL;
return ret; return ret;
} }

209
test/maat_input_mode_gtest.cpp Normal file
View File

@@ -0,0 +1,209 @@
#include "utils.h"
#include "maat/maat.h"
#include "maat_utils.h"
#include "maat_rule.h"
#include "maat_table_schema.h"
#include "json2iris.h"
#include "maat_config_monitor.h"
#include <gtest/gtest.h>
const char *table_info_path = "./table_info.conf";
const char *json_filename = "maat_json.json";
TEST(EQ_Test, Always_True) {
EXPECT_EQ(1, 1);
}
TEST(json_mode, maat_scan_string) {
char json_iris_path[128] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename);
if ((access(json_iris_path, F_OK)) == 0) {
system_cmd_rmdir(json_iris_path);
}
struct maat_options *opts = maat_options_new();
char json_path[128] = {0};
snprintf(json_path, sizeof(json_path), "./%s", json_filename);
maat_options_set_json_file(opts, json_path);
struct maat *maat_instance = maat_new(opts, table_info_path);
EXPECT_NE(maat_instance, nullptr);
struct table_schema_manager *table_schema_mgr = maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
char data[128] = "i.ytimg.com";
int result_array[5] = {0};
size_t n_result_array = 0;
int ret = maat_scan_string(maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 1);
EXPECT_EQ(result_array[0], 30);
maat_free(maat_instance);
}
TEST(iris_mode, maat_scan_string) {
char tmp_iris_path[128] = {0};
char json_iris_path[128] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename);
if ((access(json_iris_path, F_OK)) < 0) {
char *json_buff = NULL;
size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_filename, (unsigned char**)&json_buff, &json_buff_sz);
EXPECT_NE(ret, -1);
ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, NULL, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL);
EXPECT_NE(ret, -1);
}
char tmp_iris_full_idx_path[128] = {0};
char tmp_iris_inc_idx_path[128] = {0};
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path);
snprintf(tmp_iris_inc_idx_path, sizeof(tmp_iris_inc_idx_path), "%s/index", json_iris_path);
struct maat_options *opts = maat_options_new();
maat_options_set_iris_full_index_dir(opts, tmp_iris_full_idx_path);
maat_options_set_iris_inc_index_dir(opts, tmp_iris_inc_idx_path);
struct maat *maat_instance = maat_new(opts, table_info_path);
EXPECT_NE(maat_instance, nullptr);
struct table_schema_manager *table_schema_mgr = maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
char data[128] = "i.ytimg.com";
int result_array[5] = {0};
size_t n_result_array = 0;
int ret = maat_scan_string(maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 1);
EXPECT_EQ(result_array[0], 30);
maat_free(maat_instance);
}
int count_line_num_cb(const char *table_name, const char *line, void *u_para)
{
(*((unsigned int *)u_para))++;
return 0;
}
int line_idx = 0;
long long absolute_expire_time=0;
int make_serial_rule(const char *table_name, const char *line, void *u_para)
{
struct serial_rule *s_rule=(struct serial_rule *)u_para;
int rule_id = 0;
char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1);
while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') {
buff[strlen(buff) - 1] = '\0';
}
int j = 0;
char *str1 = NULL;
char *token = NULL;
char *saveptr1 = NULL;
for (j = 0,str1 = buff; ; j++, str1 = NULL) {
token = strtok_r(str1, "\t ", &saveptr1);
if (token == NULL)
break;
if (j == 0) {
sscanf(token,"%d", &rule_id);
}
}
memcpy(buff, line, strlen(line)+1);
while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t') {
buff[strlen(buff)-1]='\0';
}
maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, buff, absolute_expire_time);
line_idx++;
FREE(str1);
return 0;
}
TEST(redis_mode, maat_scan_string) {
char json_iris_path[128] = {0};
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename);
redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db);
EXPECT_NE(c, nullptr);
if ((access(json_iris_path, F_OK)) < 0) {
char tmp_iris_path[128] = {0};
char *json_buff = NULL;
size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz);
EXPECT_NE(ret, -1);
ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL);
EXPECT_NE(ret, -1);
}
size_t total_line_cnt = 0;
char tmp_iris_full_idx_path[128] = {0};
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
long long server_time = maat_cmd_redis_server_time_s(c);
EXPECT_NE(server_time, -1);
absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule);
int success_cnt = 0;
do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time);
} while (success_cnt < 0);
EXPECT_EQ(success_cnt, (int)total_line_cnt);
for (size_t i = 0; i < total_line_cnt; i++) {
maat_cmd_clear_rule_cache(s_rule + i);
}
FREE(s_rule);
redisFree(c);
struct maat_options *opts = maat_options_new();
maat_options_set_redis_ip(opts, redis_ip);
maat_options_set_redis_port(opts, redis_port);
struct maat *maat_instance = maat_new(opts, table_info_path);
struct table_schema_manager *table_schema_mgr = maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
char data[128] = "i.ytimg.com";
int result_array[5] = {0};
size_t n_result_array = 0;
int ret = maat_scan_string(maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 1);
EXPECT_EQ(result_array[0], 30);
maat_free(maat_instance);
}
int main(int argc, char ** argv)
{
int ret=0;
::testing::InitGoogleTest(&argc, argv);
ret=RUN_ALL_TESTS();
return ret;
}

2
test/rcu_hash_gtest.cpp
View File

@@ -10,7 +10,7 @@ struct user_data {
void data_free(void *user_ctx, void *data) void data_free(void *user_ctx, void *data)
{ {
free(data); FREE(data);
} }
TEST(rcu_hash_new, invalid_input_parameter) { TEST(rcu_hash_new, invalid_input_parameter) {

155
tools/maat_redis_tool.cpp
View File

@@ -55,7 +55,7 @@ int set_file_rulenum(const char *path, int rule_num)
} }
if (NULL == fp) { if (NULL == fp) {
fprintf(stderr, "fopen %s failed %s at set rule num.", path, strerror(errno)); fprintf(stderr, "fopen %s failed %s at set rule num\n", path, strerror(errno));
return -1; return -1;
} }
@@ -67,15 +67,18 @@ int set_file_rulenum(const char *path, int rule_num)
void read_rule_from_redis(redisContext *c, long long desire_version, const char* output_path) void read_rule_from_redis(redisContext *c, long long desire_version, const char* output_path)
{ {
struct serial_rule *rule_list; int i = 0;
int line_count=0; int ret = 0;
int i=0,ret=0; int line_count = 0;
int update_type = CM_UPDATE_TYPE_INC; int update_type = CM_UPDATE_TYPE_INC;
long long version=0; long long version = 0;
const char* cur_table=NULL; const char *cur_table = NULL;
char foreign_files_dir[256] = {0}; char foreign_files_dir[256] = {0};
char table_path[256],index_path[256]; char table_path[256] = {0};
FILE *table_fp=NULL, *index_fp=NULL; char index_path[256] = {0};
FILE *table_fp = NULL;
FILE *index_fp = NULL;
struct serial_rule *rule_list = NULL;
int rule_num = maat_cmd_get_rm_key_list(c, 0, desire_version, &version, NULL, &rule_list, &update_type, 0); int rule_num = maat_cmd_get_rm_key_list(c, 0, desire_version, &version, NULL, &rule_list, &update_type, 0);
if (0 == rule_num) { if (0 == rule_num) {
@@ -126,10 +129,10 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char*
maat_cmd_get_foreign_conts(c, rule_list, rule_num, 1); maat_cmd_get_foreign_conts(c, rule_list, rule_num, 1);
} }
snprintf(index_path,sizeof(index_path),"%s/full_config_index.%020lld",output_path,version); snprintf(index_path,sizeof(index_path), "%s/full_config_index.%020lld", output_path, version);
index_fp = fopen(index_path, "w"); index_fp = fopen(index_path, "w");
if (NULL == index_fp) { if (NULL == index_fp) {
printf("Open %s failed.\n",index_path); printf("Open %s failed.\n", index_path);
goto clean_up; goto clean_up;
} }
@@ -174,11 +177,10 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char*
clean_up: clean_up:
for (i = 0; i < rule_num; i++) { for (i = 0; i < rule_num; i++) {
maat_cmd_empty_serial_rule(rule_list+i); maat_cmd_clear_rule_cache(rule_list+i);
} }
free(rule_list); FREE(rule_list);
rule_list = NULL;
if (c != NULL) { if (c != NULL) {
redisFree(c); redisFree(c);
@@ -193,7 +195,7 @@ clean_up:
} }
} }
int count_line_num(const char *table_name,const char *line, void *u_para) int count_line_num_cb(const char *table_name, const char *line, void *u_para)
{ {
(*((unsigned int *)u_para))++; (*((unsigned int *)u_para))++;
return 0; return 0;
@@ -201,10 +203,11 @@ int count_line_num(const char *table_name,const char *line, void *u_para)
int line_idx = 0; int line_idx = 0;
long long absolute_expire_time=0; long long absolute_expire_time=0;
int make_serial_rule(const char *table_name, const char *line,void *u_para) int make_serial_rule(const char *table_name, const char *line, void *u_para)
{ {
struct serial_rule *s_rule=(struct serial_rule *)u_para; struct serial_rule *s_rule=(struct serial_rule *)u_para;
int rule_id=0,is_valid=0; int rule_id = 0;
//int is_valid = 0;
char *buff = ALLOC(char, strlen(line) + 1); char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1); memcpy(buff, line, strlen(line) + 1);
@@ -217,28 +220,19 @@ int make_serial_rule(const char *table_name, const char *line,void *u_para)
char *str1 = NULL; char *str1 = NULL;
char *token = NULL; char *token = NULL;
char *saveptr1 = NULL; char *saveptr1 = NULL;
char *last1 =NULL;
for (j = 0,str1 = buff; ; j++, str1 = NULL) { for (j = 0,str1 = buff; ; j++, str1 = NULL) {
token = strtok_r(str1, "\t ", &saveptr1); token = strtok_r(str1, "\t ", &saveptr1);
if (token == NULL) if (token == NULL)
break; break;
if(j==0) if (j == 0) {
{ sscanf(token,"%d", &rule_id);
sscanf(token,"%d",&rule_id);
} }
last1=token;
} }
sscanf(last1,"%d",&is_valid);
memcpy(buff,line, strlen(line)+1); maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, buff, absolute_expire_time);
while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t')
{
buff[strlen(buff)-1]='\0';
}
//printf("Writing table %s, rule_id=%d, timeout=%lld, is_valid=%d\n", table_name, rule_id,absolute_expire_time,is_valid);
maat_cmd_set_serial_rule(s_rule + line_idx, (enum maat_operation)is_valid, rule_id, table_name, buff, absolute_expire_time);
line_idx++; line_idx++;
free(str1); FREE(str1);
return 0; return 0;
} }
@@ -247,25 +241,17 @@ int make_serial_rule(const char *table_name, const char *line,void *u_para)
#define WORK_MODE_JSON 1 #define WORK_MODE_JSON 1
int main(int argc, char * argv[]) int main(int argc, char * argv[])
{ {
int oc=0,ret=0; int oc = 0;
int model=0; int model = 0;
char redis_ip[64]; char redis_ip[64] = {0};
int redis_port=6379; int redis_port = 6379;
int redis_db=0; int redis_db = 0;
char dump_dir[128] = {0};
char json_file[128] = {0};
int timeout = 0;
long long desired_version = 0;
strncpy(redis_ip, "127.0.0.1", sizeof(redis_ip)); strncpy(redis_ip, "127.0.0.1", sizeof(redis_ip));
char dump_dir[128], json_file[128], tmp_iris_path[128]; strncpy(dump_dir, redis_dump_dir, sizeof(dump_dir));
strncpy(dump_dir,redis_dump_dir,sizeof(dump_dir));
redisContext * ctx=NULL;
unsigned int total_line_cnt=0, success_cnt=0, i=0;
int timeout=0;
FILE* json_fp=NULL;
cJSON *json=NULL, *tmp_obj=NULL;
struct stat fstat_buf;
unsigned long json_file_size=0,read_size=0;
long long desired_version=0;
char* json_buff=NULL;
size_t json_buff_sz=0;
char compile_table_name[128] = {0};
while ((oc = getopt(argc,argv,"h:p:n:d:v:f:j:t:")) != -1) { while ((oc = getopt(argc,argv,"h:p:n:d:v:f:j:t:")) != -1) {
switch (oc) { switch (oc) {
@@ -291,44 +277,9 @@ int main(int argc, char * argv[])
case 'j': case 'j':
strncpy(json_file, optarg, sizeof(json_file)); strncpy(json_file, optarg, sizeof(json_file));
model = WORK_MODE_JSON; model = WORK_MODE_JSON;
ret = stat(json_file, &fstat_buf);
if (ret != 0) {
printf("fstat file %s error.\n", json_file);
return -1;
}
json_fp = fopen(json_file, "r");
if (NULL == json_fp) {
printf("fopen file %s error %s.\n", json_file, strerror(errno));
return -1;
}
json_file_size = fstat_buf.st_size;
json_buff = ALLOC(char, json_file_size);
read_size = fread(json_buff, 1, json_file_size, json_fp);
if (read_size != json_file_size) {
printf("fread file %s error.\n", json_file);
return -1;
}
json = cJSON_Parse(json_buff);
if (!json) {
printf("%s format is error before: %-200.200s\n", json_file, cJSON_GetErrorPtr());
return -1;
}
tmp_obj=cJSON_GetObjectItem(json, "compile_table");
if (NULL == tmp_obj) {
printf("No \"compile_table\" in %s.\n",json_file);
return -1;
}
strncpy(compile_table_name, tmp_obj->valuestring, sizeof(compile_table_name));
free(json_buff);
cJSON_Delete(json);
fclose(json_fp);
break; break;
case 't': case 't':
sscanf(optarg,"%d",&timeout); sscanf(optarg,"%d", &timeout);
break; break;
case '?': case '?':
default: default:
@@ -338,31 +289,34 @@ int main(int argc, char * argv[])
} }
} }
ctx = maat_cmd_connect_redis(redis_ip, redis_port, redis_db); redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db);
if (NULL == ctx) { if (NULL == c) {
return -1; return -1;
} }
char tmp_iris_path[128] = {0};
if (model == WORK_MODE_DUMP) { if (model == WORK_MODE_DUMP) {
printf("Reading key list from %s:%d db%d.\n", redis_ip, redis_port, redis_db); printf("Reading key list from %s:%d db%d.\n", redis_ip, redis_port, redis_db);
read_rule_from_redis(ctx, desired_version, dump_dir); read_rule_from_redis(c, desired_version, dump_dir);
} else if(model == WORK_MODE_JSON) { } else if(model == WORK_MODE_JSON) {
ret = load_file_to_memory(json_file, (unsigned char**)&json_buff, &json_buff_sz); char *json_buff = NULL;
size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_file, (unsigned char**)&json_buff, &json_buff_sz);
if (ret < 0) { if (ret < 0) {
printf("open %s failed.\n", json_file); printf("open %s failed.\n", json_file);
} }
ret = json2iris(json_buff, json_file, NULL, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL); ret = json2iris(json_buff, json_file, NULL, NULL, NULL, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL);
if (ret < 0) { if (ret < 0) {
printf("Invalid json format.\n"); printf("Invalid json format.\n");
} }
total_line_cnt = 0; size_t total_line_cnt = 0;
config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num, NULL, &total_line_cnt); config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt);
printf("Serialize %s to %d lines, write temp file to %s .\n",json_file, total_line_cnt, tmp_iris_path); printf("Serialize %s to %zu lines, write temp file to %s .\n", json_file, total_line_cnt, tmp_iris_path);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
long long server_time = maat_cmd_redis_server_time_s(ctx); long long server_time = maat_cmd_redis_server_time_s(c);
if (!server_time) { if (!server_time) {
printf("Get Redis Time failed.\n"); printf("Get Redis Time failed.\n");
} }
@@ -375,19 +329,20 @@ int main(int argc, char * argv[])
printf("Timeout = %lld\n", absolute_expire_time); printf("Timeout = %lld\n", absolute_expire_time);
ret = 0; ret = 0;
int success_cnt = 0;
do { do {
success_cnt = maat_cmd_exec_serial_rule(ctx, s_rule, total_line_cnt, server_time); success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time);
} while(success_cnt < 0); } while(success_cnt < 0);
if (success_cnt != total_line_cnt) { if (success_cnt != (int)total_line_cnt) {
printf("Only Add %d of %d, rule id maybe conflicts.\n", success_cnt, total_line_cnt); printf("Only Add %d of %zu, rule id maybe conflicts.\n", success_cnt, total_line_cnt);
} }
for (i = 0; i < total_line_cnt; i++) { for (size_t i = 0; i < total_line_cnt; i++) {
maat_cmd_empty_serial_rule(s_rule+i); maat_cmd_clear_rule_cache(s_rule+i);
} }
free(s_rule); FREE(s_rule);
redisFree(ctx); redisFree(c);
} }
return 0; return 0;