gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[OPTIMIZE]reduce config memory usage

Browse Source
This commit is contained in:
liuwentan
2023-06-16 15:59:30 +08:00
parent 28dc76e987
commit 0b73681bd1
27 changed files with 1902 additions and 1862 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
include/maat.h
View File

@@ -70,32 +70,55 @@ enum log_level {
LOG_LEVEL_ERROR, LOG_LEVEL_ERROR,
LOG_LEVEL_FATAL LOG_LEVEL_FATAL
}; };
/* update_type: MAAT_UPDATE_TYPE_FULL or MAAT_UPDATE_TYPE_INC */ /* update_type: MAAT_UPDATE_TYPE_FULL or MAAT_UPDATE_TYPE_INC */
typedef void maat_start_callback_t(int update_type, void *u_param); typedef void maat_start_callback_t(int update_type, void *u_param);
typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para); typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para);
typedef void maat_finish_callback_t(void *u_para); typedef void maat_finish_callback_t(void *u_para);
typedef void maat_ex_new_func_t(const char *table_name, int table_id, const char *key, typedef void maat_ex_new_func_t(const char *table_name, int table_id, const char *key,
const char *table_line, void **ad, long argl, void *argp); const char *table_line, void **ad, long argl, void *argp);
typedef void maat_ex_free_func_t(int table_id, void **ad, long argl, void *argp); typedef void maat_ex_free_func_t(int table_id, void **ad, long argl, void *argp);
typedef void maat_ex_dup_func_t(int table_id, void **to, void **from, long argl, void *argp); typedef void maat_ex_dup_func_t(int table_id, void **to, void **from, long argl, void *argp);
/* maat_instance options API */ /* maat_instance options API */
struct maat_options; struct maat_options;
struct maat_options *maat_options_new(void); struct maat_options *maat_options_new(void);
void maat_options_free(struct maat_options *opts); void maat_options_free(struct maat_options *opts);
int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread); /**
int maat_options_set_accept_tags(struct maat_options *opts, const char *accept_tags); * @brief set maat instance name
int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms); *
int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms); * @note The maximum length of instance_name is 15 bytes
int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms); */
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name); int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name);
int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread);
int maat_options_set_accept_tags(struct maat_options *opts, const char *accept_tags);
int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_deferred_load_on(struct maat_options *opts); int maat_options_set_deferred_load_on(struct maat_options *opts);
int maat_options_set_stat_on(struct maat_options *opts); int maat_options_set_stat_on(struct maat_options *opts);
int maat_options_set_perf_on(struct maat_options *opts); int maat_options_set_perf_on(struct maat_options *opts);
int maat_options_set_foreign_cont_dir(struct maat_options *opts, const char *dir); int maat_options_set_foreign_cont_dir(struct maat_options *opts, const char *dir);
int maat_options_set_logger(struct maat_options *opts, const char *log_path, enum log_level level);
int maat_options_set_logger(struct maat_options *opts, const char *log_path,
enum log_level level);
int maat_options_set_iris(struct maat_options *opts, const char *full_directory, int maat_options_set_iris(struct maat_options *opts, const char *full_directory,
const char *increment_directory); const char *increment_directory);

198
scanner/adapter_hs/adapter_hs.cpp
View File

@@ -103,8 +103,8 @@ struct adapter_hs_stream {
struct log_handle *logger; struct log_handle *logger;
}; };
int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratches, size_t n_worker_thread, int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratches,
struct log_handle *logger) size_t n_worker_thread, struct log_handle *logger)
{ {
size_t scratch_size = 0; size_t scratch_size = 0;
@@ -119,14 +119,16 @@ int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratches, size_t n_work
hs_error_t err = hs_clone_scratch(scratches[0], &scratches[i]); hs_error_t err = hs_clone_scratch(scratches[0], &scratches[i]);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS, log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] Unable to clone scratch", __FUNCTION__, __LINE__); "[%s:%d] Unable to clone scratch",
__FUNCTION__, __LINE__);
return -1; return -1;
} }
err = hs_scratch_size(scratches[i], &scratch_size); err = hs_scratch_size(scratches[i], &scratch_size);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS, log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] Unable to query scratch size", __FUNCTION__, __LINE__); "[%s:%d] Unable to query scratch size",
__FUNCTION__, __LINE__);
return -1; return -1;
} }
} }
@@ -134,8 +136,10 @@ int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratches, size_t n_work
return 0; return 0;
} }
static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_worker_thread, static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt,
enum hs_pattern_type pattern_type, struct log_handle *logger) size_t n_worker_thread,
enum hs_pattern_type pattern_type,
struct log_handle *logger)
{ {
int ret = 0; int ret = 0;
@@ -178,9 +182,11 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
} }
if (literal_cd != NULL) { if (literal_cd != NULL) {
err = hs_compile_lit_multi((const char *const *)literal_cd->patterns, literal_cd->flags, err = hs_compile_lit_multi((const char *const *)literal_cd->patterns,
literal_cd->ids, literal_cd->pattern_lens, literal_cd->n_patterns, literal_cd->flags,literal_cd->ids,
HS_MODE_STREAM, NULL, &hs_rt->literal_db, &compile_err); literal_cd->pattern_lens, literal_cd->n_patterns,
HS_MODE_STREAM | HS_MODE_SOM_HORIZON_SMALL, NULL,
&hs_rt->literal_db, &compile_err);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
if (compile_err) { if (compile_err) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] compile error: %s", log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] compile error: %s",
@@ -193,9 +199,10 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
} }
if (regex_cd != NULL) { if (regex_cd != NULL) {
err = hs_compile_multi((const char *const *)regex_cd->patterns, regex_cd->flags, regex_cd->ids, err = hs_compile_multi((const char *const *)regex_cd->patterns,
regex_cd->n_patterns, HS_MODE_STREAM | HS_MODE_SOM_HORIZON_SMALL, NULL, regex_cd->flags, regex_cd->ids, regex_cd->n_patterns,
&hs_rt->regex_db, &compile_err); HS_MODE_STREAM | HS_MODE_SOM_HORIZON_SMALL,
NULL, &hs_rt->regex_db, &compile_err);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
if (compile_err) { if (compile_err) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] compile error: %s", log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] compile error: %s",
@@ -250,8 +257,8 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd)
FREE(hs_cd); FREE(hs_cd);
} }
void populate_compile_data(struct adpt_hs_compile_data *compile_data, int index, int pattern_id, void populate_compile_data(struct adpt_hs_compile_data *compile_data, int index,
char *pat, size_t pat_len, int case_sensitive) int pattern_id, char *pat, size_t pat_len, int case_sensitive)
{ {
compile_data->ids[index] = pattern_id; compile_data->ids[index] = pattern_id;
@@ -266,8 +273,10 @@ void populate_compile_data(struct adpt_hs_compile_data *compile_data, int index,
memcpy(compile_data->patterns[index], pat, pat_len); memcpy(compile_data->patterns[index], pat, pat_len);
} }
struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule, struct pattern_attribute *pattern_attr, struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule,
struct adpt_hs_compile_data *literal_cd, struct adpt_hs_compile_data *regex_cd, struct pattern_attribute *pattern_attr,
struct adpt_hs_compile_data *literal_cd,
struct adpt_hs_compile_data *regex_cd,
size_t *n_pattern) size_t *n_pattern)
{ {
uint32_t pattern_index = 0; uint32_t pattern_index = 0;
@@ -329,7 +338,8 @@ int verify_regex_expression(const char *regex_str, struct log_handle *logger)
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
// Expression will fail compilation and report error elsewhere. // Expression will fail compilation and report error elsewhere.
if (logger != NULL) { if (logger != NULL) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] illegal regex expression: \"%s\": %s", log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] illegal regex expression: \"%s\": %s",
__FUNCTION__, __LINE__, regex_str, error->message); __FUNCTION__, __LINE__, regex_str, error->message);
} }
@@ -354,13 +364,12 @@ int adapter_hs_verify_regex_expression(const char *regex_expr, struct log_handle
return verify_regex_expression(regex_expr, logger); return verify_regex_expression(regex_expr, logger);
} }
struct adapter_hs *adapter_hs_new(size_t n_worker_thread, struct adapter_hs *adapter_hs_new(struct expr_rule *rules, size_t n_rule,
struct expr_rule *rules, size_t n_rule, size_t n_worker_thread, struct log_handle *logger)
struct log_handle *logger)
{ {
if (0 == n_worker_thread || NULL == rules || 0 == n_rule) { if (0 == n_worker_thread || NULL == rules || 0 == n_rule) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] input parameters illegal!", log_error(logger, MODULE_ADAPTER_HS,
__FUNCTION__, __LINE__); "[%s:%d] input parameters illegal!", __FUNCTION__, __LINE__);
return NULL; return NULL;
} }
@@ -370,8 +379,9 @@ struct adapter_hs *adapter_hs_new(size_t n_worker_thread,
for (size_t i = 0; i < n_rule; i++) { for (size_t i = 0; i < n_rule; i++) {
if (rules[i].n_patterns > MAX_EXPR_PATTERN_NUM) { if (rules[i].n_patterns > MAX_EXPR_PATTERN_NUM) {
log_error(logger, MODULE_ADAPTER_HS, log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] the number of patterns in one expression should less than %d", "[%s:%d] the number of patterns in one expression "
__FUNCTION__, __LINE__, MAX_EXPR_PATTERN_NUM); "should less than %d", __FUNCTION__, __LINE__,
MAX_EXPR_PATTERN_NUM);
return NULL; return NULL;
} }
@@ -379,7 +389,8 @@ struct adapter_hs *adapter_hs_new(size_t n_worker_thread,
/* pat_len should not 0 */ /* pat_len should not 0 */
if (0 == rules[i].patterns[j].pat_len) { if (0 == rules[i].patterns[j].pat_len) {
log_error(logger, MODULE_ADAPTER_HS, log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] expr pattern length should not 0", __FUNCTION__, __LINE__); "[%s:%d] expr pattern length should not 0",
__FUNCTION__, __LINE__);
return NULL; return NULL;
} }
@@ -392,8 +403,8 @@ struct adapter_hs *adapter_hs_new(size_t n_worker_thread,
} }
if (0 == literal_pattern_num && 0 == regex_pattern_num) { if (0 == literal_pattern_num && 0 == regex_pattern_num) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] exprs has no valid pattern", log_error(logger, MODULE_ADAPTER_HS,
__FUNCTION__, __LINE__); "[%s:%d] exprs has no valid pattern", __FUNCTION__, __LINE__);
return NULL; return NULL;
} }
@@ -408,30 +419,32 @@ struct adapter_hs *adapter_hs_new(size_t n_worker_thread,
} }
size_t pattern_cnt = literal_pattern_num + regex_pattern_num; size_t pattern_cnt = literal_pattern_num + regex_pattern_num;
struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1); struct adapter_hs *hs_inst = ALLOC(struct adapter_hs, 1);
hs_instance->hs_attr = ALLOC(struct pattern_attribute, pattern_cnt); hs_inst->hs_attr = ALLOC(struct pattern_attribute, pattern_cnt);
hs_instance->logger = logger; hs_inst->logger = logger;
hs_instance->n_worker_thread = n_worker_thread; hs_inst->n_worker_thread = n_worker_thread;
hs_instance->n_expr = n_rule; hs_inst->n_expr = n_rule;
struct bool_expr *bool_exprs = bool_exprs_new(rules, n_rule, hs_instance->hs_attr, struct bool_expr *bool_exprs = bool_exprs_new(rules, n_rule, hs_inst->hs_attr,
literal_cd, regex_cd, &pattern_cnt); literal_cd, regex_cd, &pattern_cnt);
if (NULL == bool_exprs) { if (NULL == bool_exprs) {
return NULL; return NULL;
} }
hs_instance->n_patterns = pattern_cnt; hs_inst->n_patterns = pattern_cnt;
/* create bool matcher */ /* create bool matcher */
size_t mem_size = 0; size_t mem_size = 0;
int hs_ret = 0; int hs_ret = 0;
hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1);
hs_instance->hs_rt->bm = bool_matcher_new(bool_exprs, n_rule, &mem_size); hs_inst->hs_rt = ALLOC(struct adapter_hs_runtime, 1);
if (hs_instance->hs_rt->bm != NULL) { hs_inst->hs_rt->bm = bool_matcher_new(bool_exprs, n_rule, &mem_size);
if (hs_inst->hs_rt->bm != NULL) {
log_info(logger, MODULE_ADAPTER_HS, log_info(logger, MODULE_ADAPTER_HS,
"Adapter_hs module: build bool matcher of %zu expressions with %zu bytes memory", "Adapter_hs module: build bool matcher of %zu expressions"
n_rule, mem_size); " with %zu bytes memory", n_rule, mem_size);
} else { } else {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] Adapter_hs module: build bool matcher failed", log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] Adapter_hs module: build bool matcher failed",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
hs_ret = -1; hs_ret = -1;
@@ -439,7 +452,7 @@ struct adapter_hs *adapter_hs_new(size_t n_worker_thread,
FREE(bool_exprs); FREE(bool_exprs);
/* build hs database */ /* build hs database */
int ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, logger); int ret = adpt_hs_build_database(hs_inst->hs_rt, literal_cd, regex_cd, logger);
if (ret < 0) { if (ret < 0) {
hs_ret = -1; hs_ret = -1;
} }
@@ -456,97 +469,101 @@ struct adapter_hs *adapter_hs_new(size_t n_worker_thread,
goto error; goto error;
} }
hs_instance->hs_rt->scratch = ALLOC(struct adapter_hs_scratch, 1); hs_inst->hs_rt->scratch = ALLOC(struct adapter_hs_scratch, 1);
hs_instance->hs_rt->scratch->bool_match_buffs = ALLOC(struct bool_expr_match *, n_worker_thread); hs_inst->hs_rt->scratch->bool_match_buffs = ALLOC(struct bool_expr_match *,
n_worker_thread);
for (size_t i = 0; i < n_worker_thread; i++) { for (size_t i = 0; i < n_worker_thread; i++) {
hs_instance->hs_rt->scratch->bool_match_buffs[i] = ALLOC(struct bool_expr_match, hs_instance->n_expr); hs_inst->hs_rt->scratch->bool_match_buffs[i] = ALLOC(struct bool_expr_match,
hs_inst->n_expr);
} }
/* literal and regex scratch can't reuse */ /* literal and regex scratch can't reuse */
if (literal_pattern_num > 0) { if (literal_pattern_num > 0) {
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, HS_PATTERN_TYPE_STR, logger); ret = adpt_hs_alloc_scratch(hs_inst->hs_rt, n_worker_thread,
HS_PATTERN_TYPE_STR, logger);
if (ret < 0) { if (ret < 0) {
goto error; goto error;
} }
} }
if (regex_pattern_num > 0) { if (regex_pattern_num > 0) {
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, HS_PATTERN_TYPE_REG, logger); ret = adpt_hs_alloc_scratch(hs_inst->hs_rt, n_worker_thread,
HS_PATTERN_TYPE_REG, logger);
if (ret < 0) { if (ret < 0) {
goto error; goto error;
} }
} }
return hs_instance; return hs_inst;
error: error:
adapter_hs_free(hs_instance); adapter_hs_free(hs_inst);
return NULL; return NULL;
} }
void adapter_hs_free(struct adapter_hs *hs_instance) void adapter_hs_free(struct adapter_hs *hs_inst)
{ {
if (NULL == hs_instance) { if (NULL == hs_inst) {
return; return;
} }
if (hs_instance->hs_rt != NULL) { if (hs_inst->hs_rt != NULL) {
if (hs_instance->hs_rt->literal_db != NULL) { if (hs_inst->hs_rt->literal_db != NULL) {
hs_free_database(hs_instance->hs_rt->literal_db); hs_free_database(hs_inst->hs_rt->literal_db);
hs_instance->hs_rt->literal_db = NULL; hs_inst->hs_rt->literal_db = NULL;
} }
if (hs_instance->hs_rt->regex_db != NULL) { if (hs_inst->hs_rt->regex_db != NULL) {
hs_free_database(hs_instance->hs_rt->regex_db); hs_free_database(hs_inst->hs_rt->regex_db);
hs_instance->hs_rt->regex_db = NULL; hs_inst->hs_rt->regex_db = NULL;
} }
if (hs_instance->hs_rt->scratch != NULL) { if (hs_inst->hs_rt->scratch != NULL) {
if (hs_instance->hs_rt->scratch->literal_scratches != NULL) { if (hs_inst->hs_rt->scratch->literal_scratches != NULL) {
for (size_t i = 0; i < hs_instance->n_worker_thread; i++) { for (size_t i = 0; i < hs_inst->n_worker_thread; i++) {
if (hs_instance->hs_rt->scratch->literal_scratches[i] != NULL) { if (hs_inst->hs_rt->scratch->literal_scratches[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->scratch->literal_scratches[i]); hs_free_scratch(hs_inst->hs_rt->scratch->literal_scratches[i]);
hs_instance->hs_rt->scratch->literal_scratches[i] = NULL; hs_inst->hs_rt->scratch->literal_scratches[i] = NULL;
} }
} }
FREE(hs_instance->hs_rt->scratch->literal_scratches); FREE(hs_inst->hs_rt->scratch->literal_scratches);
} }
if (hs_instance->hs_rt->scratch->regex_scratches != NULL) { if (hs_inst->hs_rt->scratch->regex_scratches != NULL) {
for (size_t i = 0; i < hs_instance->n_worker_thread; i++) { for (size_t i = 0; i < hs_inst->n_worker_thread; i++) {
if (hs_instance->hs_rt->scratch->regex_scratches[i] != NULL) { if (hs_inst->hs_rt->scratch->regex_scratches[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->scratch->regex_scratches[i]); hs_free_scratch(hs_inst->hs_rt->scratch->regex_scratches[i]);
hs_instance->hs_rt->scratch->regex_scratches[i] = NULL; hs_inst->hs_rt->scratch->regex_scratches[i] = NULL;
} }
} }
FREE(hs_instance->hs_rt->scratch->regex_scratches); FREE(hs_inst->hs_rt->scratch->regex_scratches);
} }
if (hs_instance->hs_rt->scratch->bool_match_buffs != NULL) { if (hs_inst->hs_rt->scratch->bool_match_buffs != NULL) {
for (size_t i = 0; i < hs_instance->n_worker_thread; i++) { for (size_t i = 0; i < hs_inst->n_worker_thread; i++) {
if (hs_instance->hs_rt->scratch->bool_match_buffs[i] != NULL) { if (hs_inst->hs_rt->scratch->bool_match_buffs[i] != NULL) {
FREE(hs_instance->hs_rt->scratch->bool_match_buffs[i]); FREE(hs_inst->hs_rt->scratch->bool_match_buffs[i]);
} }
} }
FREE(hs_instance->hs_rt->scratch->bool_match_buffs); FREE(hs_inst->hs_rt->scratch->bool_match_buffs);
} }
FREE(hs_instance->hs_rt->scratch); FREE(hs_inst->hs_rt->scratch);
} }
if (hs_instance->hs_rt->bm != NULL) { if (hs_inst->hs_rt->bm != NULL) {
bool_matcher_free(hs_instance->hs_rt->bm); bool_matcher_free(hs_inst->hs_rt->bm);
hs_instance->hs_rt->bm = NULL; hs_inst->hs_rt->bm = NULL;
} }
FREE(hs_instance->hs_rt); FREE(hs_inst->hs_rt);
} }
if (hs_instance->hs_attr != NULL) { if (hs_inst->hs_attr != NULL) {
FREE(hs_instance->hs_attr); FREE(hs_inst->hs_attr);
} }
FREE(hs_instance); FREE(hs_inst);
} }
static inline int compare_pattern_id(const void *a, const void *b) static inline int compare_pattern_id(const void *a, const void *b)
@@ -641,7 +658,8 @@ int matched_event_cb(unsigned int id, unsigned long long from,
} }
UT_icd ut_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL}; UT_icd ut_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL};
struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id) struct adapter_hs_stream *
adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id)
{ {
if (NULL == hs_instance || thread_id < 0) { if (NULL == hs_instance || thread_id < 0) {
return NULL; return NULL;
@@ -662,17 +680,21 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
int err_count = 0; int err_count = 0;
if (hs_instance->hs_rt->literal_db != NULL) { if (hs_instance->hs_rt->literal_db != NULL) {
err = hs_open_stream(hs_instance->hs_rt->literal_db, 0, &hs_stream->literal_stream); err = hs_open_stream(hs_instance->hs_rt->literal_db, 0,
&hs_stream->literal_stream);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err); log_error(hs_instance->logger, MODULE_ADAPTER_HS,
"hs_open_stream failed, hs err:%d", err);
err_count++; err_count++;
} }
} }
if (hs_instance->hs_rt->regex_db != NULL) { if (hs_instance->hs_rt->regex_db != NULL) {
err = hs_open_stream(hs_instance->hs_rt->regex_db, 0, &hs_stream->regex_stream); err = hs_open_stream(hs_instance->hs_rt->regex_db, 0,
&hs_stream->regex_stream);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err); log_error(hs_instance->logger, MODULE_ADAPTER_HS,
"hs_open_stream failed, hs err:%d", err);
err_count++; err_count++;
} }
} }

7
scanner/adapter_hs/adapter_hs.h
View File

@@ -86,15 +86,14 @@ int adapter_hs_verify_regex_expression(const char *regex_expr,
/** /**
* @brief new adapter_hs instance * @brief new adapter_hs instance
* *
* @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan()
* @param rules: logic AND expression's array * @param rules: logic AND expression's array
* @param n_rule: the number of logic AND expression's array * @param n_rule: the number of logic AND expression's array
* @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan()
* *
* @retval the pointer to adapter_hs instance * @retval the pointer to adapter_hs instance
*/ */
struct adapter_hs *adapter_hs_new(size_t n_worker_thread, struct adapter_hs *adapter_hs_new(struct expr_rule *rules, size_t n_rule,
struct expr_rule *rules, size_t n_rule, size_t n_worker_thread, struct log_handle *logger);
struct log_handle *logger);
/** /**
* @brief scan input data to match logic AND expression, return all matched expr_id * @brief scan input data to match logic AND expression, return all matched expr_id

5
src/inc_internal/maat_compile.h
View File

@@ -49,8 +49,7 @@ void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
struct log_handle *logger); struct log_handle *logger);
void compile_runtime_free(void *compile_runtime); void compile_runtime_free(void *compile_runtime);
void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt, void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt);
void *g2c_runtime);
int compile_runtime_update(void *compile_runtime, void *compile_schema, int compile_runtime_update(void *compile_runtime, void *compile_schema,
const char *table_name, const char *line, const char *table_name, const char *line,
@@ -81,7 +80,9 @@ void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt,
void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num, void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger); struct log_handle *logger);
void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime); void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime);
void group2compile_runtime_free(void *g2c_runtime); void group2compile_runtime_free(void *g2c_runtime);
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,

4
src/inc_internal/maat_limits.h
View File

@@ -17,9 +17,13 @@ extern "C"
#endif #endif
#define MAX_KEYWORDS_STR 1024 #define MAX_KEYWORDS_STR 1024
#define MAX_MAAT_STAT_NUM 64 #define MAX_MAAT_STAT_NUM 64
#define MAX_NAME_STR_LEN 64 #define MAX_NAME_STR_LEN 64
#define MAX_INSTANCE_NAME_LEN 15
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

7
src/inc_internal/maat_rule.h
View File

@@ -36,7 +36,6 @@ extern "C"
#include "hiredis/hiredis.h" #include "hiredis/hiredis.h"
#define MAX_TABLE_NUM 1024 #define MAX_TABLE_NUM 1024
#define MAX_COMPILE_TABLE_NUM 16
#define DISTRICT_ANY -1 #define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2 #define DISTRICT_UNKNOWN -2
@@ -109,13 +108,13 @@ struct source_redis_ctx {
}; };
struct maat_options { struct maat_options {
char instance_name[NAME_MAX]; char inst_name[MAX_INSTANCE_NAME_LEN + 1];
char foreign_cont_dir[NAME_MAX]; char foreign_cont_dir[NAME_MAX];
char decrypt_algo[MAX_KEYWORDS_STR]; char decrypt_algo[MAX_KEYWORDS_STR];
char decrypt_key[MAX_KEYWORDS_STR]; char decrypt_key[MAX_KEYWORDS_STR];
char log_path[PATH_MAX]; char log_path[PATH_MAX];
int log_level; int log_level;
char stat_file[PATH_MAX]; char stat_file[NAME_MAX];
size_t nr_worker_thread; size_t nr_worker_thread;
char *accept_tags; char *accept_tags;
@@ -194,7 +193,7 @@ enum last_scan_flag {
}; };
struct maat_state { struct maat_state {
struct maat *maat_instance; struct maat *maat_inst;
struct maat_compile_state *compile_state; struct maat_compile_state *compile_state;
int scan_cnt; int scan_cnt;
int district_id; //-1: Any District; -2: Unkonwn District; int district_id; //-1: Any District; -2: Unkonwn District;

562
src/maat_api.c
View File

File diff suppressed because it is too large Load Diff

6
src/maat_bool_plugin.c
View File

@@ -32,7 +32,6 @@ struct bool_plugin_schema {
struct bool_plugin_runtime { struct bool_plugin_runtime {
struct bool_matcher *matcher; struct bool_matcher *matcher;
struct ex_data_runtime *ex_data_rt; struct ex_data_runtime *ex_data_rt;
long long version;
long long rule_num; long long rule_num;
size_t n_worker_thread; size_t n_worker_thread;
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
@@ -493,13 +492,10 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name
} }
bool_plugin_rt->rule_num = rule_cnt; bool_plugin_rt->rule_num = rule_cnt;
if (maat_rt_version != 0) {
bool_plugin_rt->version = maat_rt_version;
}
log_info(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN, log_info(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
"table[%s] commit %zu bool_plugin rules and rebuild bool_matcher completed" "table[%s] commit %zu bool_plugin rules and rebuild bool_matcher completed"
", version:%lld", table_name, rule_cnt, bool_plugin_rt->version); ", version:%lld", table_name, rule_cnt, maat_rt_version);
if (rules != NULL) { if (rules != NULL) {
FREE(rules); FREE(rules);

66
src/maat_command.c
View File

@@ -184,31 +184,31 @@ static int connect_redis_for_write(struct source_redis_ctx *redis_ctx,
} }
} }
redisContext *get_redis_ctx_for_write(struct maat *maat_instance) redisContext *get_redis_ctx_for_write(struct maat *maat_inst)
{ {
if (NULL == maat_instance->opts.redis_ctx.write_ctx) { if (NULL == maat_inst->opts.redis_ctx.write_ctx) {
int ret = connect_redis_for_write(&(maat_instance->opts.redis_ctx), int ret = connect_redis_for_write(&(maat_inst->opts.redis_ctx),
maat_instance->logger); maat_inst->logger);
if (ret != 0) { if (ret != 0) {
return NULL; return NULL;
} }
} }
return maat_instance->opts.redis_ctx.write_ctx; return maat_inst->opts.redis_ctx.write_ctx;
} }
int maat_cmd_flushDB(struct maat *maat_instance) int maat_cmd_flushDB(struct maat *maat_inst)
{ {
int ret = 0; int ret = 0;
redisContext *write_ctx = get_redis_ctx_for_write(maat_instance); redisContext *write_ctx = get_redis_ctx_for_write(maat_inst);
if (NULL == write_ctx) { if (NULL == write_ctx) {
return -1; return -1;
} }
do { do {
ret = redis_flushDB(maat_instance->opts.redis_ctx.write_ctx, ret = redis_flushDB(maat_inst->opts.redis_ctx.write_ctx,
maat_instance->opts.redis_ctx.redis_db, maat_inst->opts.redis_ctx.redis_db,
maat_instance->logger); maat_inst->logger);
} while(0 == ret); } while(0 == ret);
return 0; return 0;
@@ -317,13 +317,13 @@ int maat_cmd_wrap_redis_get_reply(redisContext *c, redisReply **reply)
return redisGetReply(c, (void **)reply); return redisGetReply(c, (void **)reply);
} }
int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule) int maat_cmd_set_line(struct maat *maat_inst, const struct maat_cmd_line *line_rule)
{ {
int i = 0; int i = 0;
int ret = 0; int ret = 0;
long long absolute_expire_time = 0; long long absolute_expire_time = 0;
redisContext *write_ctx = get_redis_ctx_for_write(maat_instance); redisContext *write_ctx = get_redis_ctx_for_write(maat_inst);
if (NULL == write_ctx) { if (NULL == write_ctx) {
return -1; return -1;
} }
@@ -335,18 +335,18 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
struct serial_rule *s_rule = ALLOC(struct serial_rule, 1); struct serial_rule *s_rule = ALLOC(struct serial_rule, 1);
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, line_rule->table_name); int table_id = table_manager_get_table_id(maat_inst->tbl_mgr, line_rule->table_name);
if (table_id < 0) { if (table_id < 0) {
log_error(maat_instance->logger, MODULE_MAAT_COMMAND, log_error(maat_inst->logger, MODULE_MAAT_COMMAND,
"[%s:%d] Command set line id %lld failed: unknown table %s", "[%s:%d] Command set line id %lld failed: unknown table %s",
__FUNCTION__, __LINE__, line_rule->rule_id, line_rule->table_name); __FUNCTION__, __LINE__, line_rule->rule_id, line_rule->table_name);
FREE(s_rule); FREE(s_rule);
return -1; return -1;
} }
int valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id); int valid_column = table_manager_get_valid_column(maat_inst->tbl_mgr, table_id);
if (valid_column < 0) { if (valid_column < 0) {
log_error(maat_instance->logger, MODULE_MAAT_COMMAND, log_error(maat_inst->logger, MODULE_MAAT_COMMAND,
"[%s:%d] Command set line id %lld failed: table %s is not a plugin or ip_plugin table", "[%s:%d] Command set line id %lld failed: table %s is not a plugin or ip_plugin table",
__FUNCTION__, __LINE__, line_rule->rule_id, line_rule->table_name); __FUNCTION__, __LINE__, line_rule->rule_id, line_rule->table_name);
FREE(s_rule); FREE(s_rule);
@@ -355,7 +355,7 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
int valid_offset = maat_cmd_get_valid_flag_offset(line_rule->table_line, valid_column); int valid_offset = maat_cmd_get_valid_flag_offset(line_rule->table_line, valid_column);
if (valid_offset < 0) { if (valid_offset < 0) {
log_error(maat_instance->logger, MODULE_MAAT_COMMAND, log_error(maat_inst->logger, MODULE_MAAT_COMMAND,
"[%s:%d] Command set line id %lld failed: table %s valid_offset error", "[%s:%d] Command set line id %lld failed: table %s valid_offset error",
__FUNCTION__, __LINE__, line_rule->rule_id, line_rule->table_name); __FUNCTION__, __LINE__, line_rule->rule_id, line_rule->table_name);
FREE(s_rule); FREE(s_rule);
@@ -370,14 +370,14 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id, maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id,
line_rule->table_name, line_rule->table_line, absolute_expire_time); line_rule->table_name, line_rule->table_line, absolute_expire_time);
int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time, maat_instance->logger); int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time, maat_inst->logger);
if (success_cnt != 1) { if (success_cnt != 1) {
ret = -1; ret = -1;
goto error_out; goto error_out;
} }
ret = success_cnt; ret = success_cnt;
maat_instance->stat->line_cmd_acc_num += success_cnt; maat_inst->stat->line_cmd_acc_num += success_cnt;
error_out: error_out:
maat_cmd_clear_rule_cache(s_rule); maat_cmd_clear_rule_cache(s_rule);
@@ -386,12 +386,12 @@ error_out:
return ret; return ret;
} }
int maat_cmd_set_file(struct maat *maat_instance, const char *key, const char *value, int maat_cmd_set_file(struct maat *maat_inst, const char *key, const char *value,
size_t size, enum maat_operation op) size_t size, enum maat_operation op)
{ {
redisContext *ctx = maat_instance->opts.redis_ctx.write_ctx; redisContext *ctx = maat_inst->opts.redis_ctx.write_ctx;
if (NULL == ctx) { if (NULL == ctx) {
log_error(maat_instance->logger, MODULE_MAAT_COMMAND, log_error(maat_inst->logger, MODULE_MAAT_COMMAND,
"[%s:%d] failed: Redis is not connected.", "[%s:%d] failed: Redis is not connected.",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
return -1; return -1;
@@ -411,7 +411,7 @@ int maat_cmd_set_file(struct maat *maat_instance, const char *key, const char *v
redisReply *reply = NULL; redisReply *reply = NULL;
if (0 != strncmp(key, foreign_key_prefix, strlen(foreign_key_prefix))) { if (0 != strncmp(key, foreign_key_prefix, strlen(foreign_key_prefix))) {
log_error(maat_instance->logger, MODULE_MAAT_COMMAND, log_error(maat_inst->logger, MODULE_MAAT_COMMAND,
"Invalid File key, prefix %s is mandatory.", foreign_key_prefix); "Invalid File key, prefix %s is mandatory.", foreign_key_prefix);
return -1; return -1;
} }
@@ -430,7 +430,7 @@ int maat_cmd_set_file(struct maat *maat_instance, const char *key, const char *v
} }
if (NULL == reply || reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) { if (NULL == reply || reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) {
log_error(maat_instance->logger, MODULE_MAAT_COMMAND, log_error(maat_inst->logger, MODULE_MAAT_COMMAND,
"Set file failed, maybe Redis is busy."); "Set file failed, maybe Redis is busy.");
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
@@ -442,11 +442,11 @@ int maat_cmd_set_file(struct maat *maat_instance, const char *key, const char *v
return 1; return 1;
} }
long long maat_cmd_incrby(struct maat *maat_instance, const char *key, int increment) long long maat_cmd_incrby(struct maat *maat_inst, const char *key, int increment)
{ {
long long result = 0; long long result = 0;
redisContext *write_ctx = get_redis_ctx_for_write(maat_instance); redisContext *write_ctx = get_redis_ctx_for_write(maat_inst);
if (NULL == write_ctx) { if (NULL == write_ctx) {
return -1; return -1;
} }
@@ -464,26 +464,26 @@ long long maat_cmd_incrby(struct maat *maat_instance, const char *key, int incre
return result; return result;
} }
long long maat_cmd_get_config_version(struct maat *maat_instance) long long maat_cmd_get_config_version(struct maat *maat_inst)
{ {
long long new_version = -1; long long new_version = -1;
if (maat_instance->new_version != INVALID_VERSION) { if (maat_inst->new_version != INVALID_VERSION) {
new_version = maat_instance->new_version; new_version = maat_inst->new_version;
} else { } else {
new_version = maat_instance->maat_version; new_version = maat_inst->maat_version;
} }
return new_version; return new_version;
} }
int maat_cmd_config_is_updating(struct maat *maat_instance) int maat_cmd_config_is_updating(struct maat *maat_inst)
{ {
int ret = 0; int ret = 0;
if (0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) { if (0 == pthread_mutex_trylock(&(maat_inst->background_update_mutex))) {
ret = 0; ret = 0;
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_inst->background_update_mutex));
} else { } else {
ret = 1; ret = 1;
} }

127
src/maat_compile.c
View File

@@ -96,12 +96,11 @@ struct compile_rule {
/* compile_runtime and group2compile_runtime share compile_hash_map */ /* compile_runtime and group2compile_runtime share compile_hash_map */
struct compile_runtime { struct compile_runtime {
struct bool_matcher *bm; struct bool_matcher *bm;
struct rcu_hash_table *cfg_hash_tbl; // <compile_id, struct maat_compile> struct rcu_hash_table *cfg_hash; // <compile_id, struct maat_compile>
struct maat_runtime *ref_maat_rt; struct maat_runtime *ref_maat_rt;
time_t version; time_t version;
struct maat_clause *clause_by_literals_hash; struct maat_clause *clause_by_literals_hash;
struct literal_clause *literal2clause_hash; struct literal_clause *literal2clause_hash;
struct group2compile_runtime *ref_g2c_rt;
long long rule_num; long long rule_num;
long long update_err_cnt; long long update_err_cnt;
@@ -222,7 +221,7 @@ void maat_compile_free(struct maat_compile *compile)
FREE(compile); FREE(compile);
} }
void rcu_maat_compile_free(void *user_ctx, void *data) void rcu_compile_cfg_free(void *user_ctx, void *data)
{ {
struct maat_compile *compile = (struct maat_compile *)data; struct maat_compile *compile = (struct maat_compile *)data;
maat_compile_free(compile); maat_compile_free(compile);
@@ -253,7 +252,7 @@ int compile_table_set_ex_data_schema(struct compile_schema *compile_schema, int
void *compile_runtime_get_user_data(struct compile_runtime *compile_rt, long long compile_id) void *compile_runtime_get_user_data(struct compile_runtime *compile_rt, long long compile_id)
{ {
struct maat_compile *compile = rcu_hash_find(compile_rt->cfg_hash_tbl, struct maat_compile *compile = rcu_hash_find(compile_rt->cfg_hash,
(char *)&compile_id, sizeof(long long)); (char *)&compile_id, sizeof(long long));
void *ret = NULL; void *ret = NULL;
if (compile != NULL) { if (compile != NULL) {
@@ -291,9 +290,9 @@ void compile_runtime_user_data_iterate(struct compile_runtime *compile_rt,
void (*callback)(void *user_data, void *param, const char *table_name, int table_id), void (*callback)(void *user_data, void *param, const char *table_name, int table_id),
void *param, int table_id) void *param, int table_id)
{ {
/* I'm in background_update_mutex, config update can't happen, so no need to lock cfg_hash_tbl */ /* I'm in background_update_mutex, config update can't happen, so no need to lock cfg_hash */
void **data_array = NULL; void **data_array = NULL;
size_t data_cnt = rcu_hash_list(compile_rt->cfg_hash_tbl, &data_array); size_t data_cnt = rcu_hash_list(compile_rt->cfg_hash, &data_array);
for (size_t i = 0; i < data_cnt; i++) { for (size_t i = 0; i < data_cnt; i++) {
struct maat_compile *compile = (struct maat_compile *)data_array[i]; struct maat_compile *compile = (struct maat_compile *)data_array[i];
@@ -573,7 +572,7 @@ void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
compile_rt->expr_match_buff = ALLOC(struct bool_expr_match, compile_rt->expr_match_buff = ALLOC(struct bool_expr_match,
max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM); max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM);
compile_rt->version = time(NULL); compile_rt->version = time(NULL);
compile_rt->cfg_hash_tbl = rcu_hash_new(rcu_maat_compile_free, NULL); compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL);
compile_rt->clause_by_literals_hash = NULL; compile_rt->clause_by_literals_hash = NULL;
compile_rt->literal2clause_hash = NULL; compile_rt->literal2clause_hash = NULL;
compile_rt->logger = logger; compile_rt->logger = logger;
@@ -628,9 +627,9 @@ void compile_runtime_free(void *compile_runtime)
compile_rt->bm = NULL; compile_rt->bm = NULL;
} }
if (compile_rt->cfg_hash_tbl != NULL) { if (compile_rt->cfg_hash != NULL) {
rcu_hash_free(compile_rt->cfg_hash_tbl); rcu_hash_free(compile_rt->cfg_hash);
compile_rt->cfg_hash_tbl = NULL; compile_rt->cfg_hash = NULL;
} }
if (compile_rt->literal2clause_hash != NULL) { if (compile_rt->literal2clause_hash != NULL) {
@@ -650,21 +649,14 @@ void compile_runtime_free(void *compile_runtime)
FREE(compile_rt); FREE(compile_rt);
} }
void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt, void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt)
void *g2c_runtime)
{ {
if (NULL == compile_runtime) { if (NULL == compile_runtime) {
return; return;
} }
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
if (maat_rt != NULL) {
compile_rt->ref_maat_rt = maat_rt; compile_rt->ref_maat_rt = maat_rt;
}
if (g2c_runtime != NULL) {
compile_rt->ref_g2c_rt = g2c_runtime;
}
} }
void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num, void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num,
@@ -896,22 +888,22 @@ maat_clause_hash_fetch_clause(struct compile_runtime *compile_rt,
return clause; return clause;
} }
struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compile_rt, struct bool_matcher *
size_t *compile_cnt) maat_compile_bool_matcher_new(struct compile_runtime *compile_rt, size_t *compile_cnt)
{ {
if (NULL == compile_rt) { if (NULL == compile_rt) {
return NULL; return NULL;
} }
size_t i = 0, j = 0; size_t i = 0, j = 0, idx = 0;
int has_clause_num = 0; int has_clause_num = 0;
const struct maat_clause *clause = NULL; const struct maat_clause *clause = NULL;
struct maat_literal_id *literal_ids = NULL;
// STEP 1, update clause_id of each compile and literal // STEP 1, update clause_id of each compile and literal
void **data_array = NULL; void **data_array = NULL;
size_t idx = 0;
struct maat_compile *iter_compile = NULL; struct maat_compile *iter_compile = NULL;
size_t rule_cnt = rcu_updating_hash_list(compile_rt->cfg_hash_tbl, &data_array); size_t rule_cnt = rcu_updating_hash_list(compile_rt->cfg_hash, &data_array);
*compile_cnt = rule_cnt; *compile_cnt = rule_cnt;
for (idx = 0; idx < rule_cnt; idx++) { for (idx = 0; idx < rule_cnt; idx++) {
@@ -924,7 +916,7 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
} }
has_clause_num++; has_clause_num++;
struct maat_literal_id *literal_ids = (struct maat_literal_id *)utarray_eltptr(clause_state->ut_literal_ids, 0); literal_ids = (struct maat_literal_id *)utarray_eltptr(clause_state->ut_literal_ids, 0);
size_t n_literal_id = utarray_len(clause_state->ut_literal_ids); size_t n_literal_id = utarray_len(clause_state->ut_literal_ids);
clause = maat_clause_hash_fetch_clause(compile_rt, literal_ids, n_literal_id); clause = maat_clause_hash_fetch_clause(compile_rt, literal_ids, n_literal_id);
clause_state->clause_id = clause->clause_id; clause_state->clause_id = clause->clause_id;
@@ -974,7 +966,8 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
size_t mem_size = 0; size_t mem_size = 0;
if (0 == expr_cnt) { if (0 == expr_cnt) {
log_error(compile_rt->logger, MODULE_COMPILE, log_error(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] No bool expression to build bool matcher.", __FUNCTION__, __LINE__); "[%s:%d] No bool expression to build bool matcher.",
__FUNCTION__, __LINE__);
FREE(bool_expr_array); FREE(bool_expr_array);
return NULL; return NULL;
} }
@@ -985,8 +978,8 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
"Build bool matcher of %zu expressions with %zu bytes memory.", "Build bool matcher of %zu expressions with %zu bytes memory.",
expr_cnt, mem_size); expr_cnt, mem_size);
} else { } else {
log_error(compile_rt->logger, MODULE_COMPILE, "[%s:%d] Build bool matcher failed!", log_error(compile_rt->logger, MODULE_COMPILE,
__FUNCTION__, __LINE__); "[%s:%d] Build bool matcher failed!", __FUNCTION__, __LINE__);
} }
FREE(bool_expr_array); FREE(bool_expr_array);
@@ -1019,7 +1012,8 @@ static inline int compare_hit_group(const void *pa, const void *pb)
return ret; return ret;
} }
struct literal_clause *maat_compile_build_literal2clause_hash(struct compile_runtime *compile_rt) struct literal_clause *
maat_compile_build_literal2clause_hash(struct compile_runtime *compile_rt)
{ {
if (NULL == compile_rt) { if (NULL == compile_rt) {
return NULL; return NULL;
@@ -1030,7 +1024,7 @@ struct literal_clause *maat_compile_build_literal2clause_hash(struct compile_run
struct maat_literal_id *tmp_literal_id = NULL; struct maat_literal_id *tmp_literal_id = NULL;
struct literal_clause *l2c_value = NULL; struct literal_clause *l2c_value = NULL;
struct literal_clause *literal2clause_hash = NULL; struct literal_clause *literal2clause_hash = NULL;
size_t compile_cnt = rcu_updating_hash_list(compile_rt->cfg_hash_tbl, &data_array); size_t compile_cnt = rcu_updating_hash_list(compile_rt->cfg_hash, &data_array);
for (size_t idx = 0; idx < compile_cnt; idx++) { for (size_t idx = 0; idx < compile_cnt; idx++) {
struct maat_compile *compile = (struct maat_compile *)data_array[idx]; struct maat_compile *compile = (struct maat_compile *)data_array[idx];
@@ -1216,7 +1210,8 @@ void compile_rule_free(struct compile_rule *compile_rule)
assert(compile_rule->magic_num == COMPILE_RULE_MAGIC); assert(compile_rule->magic_num == COMPILE_RULE_MAGIC);
if (1 == schema->set_flag) { if (1 == schema->set_flag) {
rule_ex_data_free(schema->table_id, compile_rule->ex_data, &(schema->ex_schema)); rule_ex_data_free(schema->table_id, compile_rule->ex_data,
&(schema->ex_schema));
*compile_rule->ex_data = NULL; *compile_rule->ex_data = NULL;
} }
@@ -1419,8 +1414,9 @@ int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
} }
} else { } else {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] Remove group_id:%lld from compile_id:%lld failed, compile is not exisited.", "[%s:%d] Remove group_id:%lld from compile_id:%lld failed, "
__FUNCTION__, __LINE__, g2c_item->group_id, compile_id); "compile is not exisited.", __FUNCTION__, __LINE__,
g2c_item->group_id, compile_id);
return -1; return -1;
} }
} }
@@ -1456,7 +1452,7 @@ void maat_compile_state_reset(struct maat_compile_state *compile_state)
} }
void maat_compile_state_free(struct maat_compile_state *compile_state, void maat_compile_state_free(struct maat_compile_state *compile_state,
struct maat *maat_instance, int thread_id) struct maat *maat_inst, int thread_id)
{ {
if (NULL == compile_state) { if (NULL == compile_state) {
return; return;
@@ -1484,7 +1480,7 @@ void maat_compile_state_free(struct maat_compile_state *compile_state,
FREE(compile_state); FREE(compile_state);
free_bytes += sizeof(struct maat_compile_state); free_bytes += sizeof(struct maat_compile_state);
alignment_int64_array_add(maat_instance->stat->maat_state_free_bytes, thread_id, free_bytes); alignment_int64_array_add(maat_inst->stat->maat_state_free_bytes, thread_id, free_bytes);
} }
static int maat_compile_hit_path_add(UT_array *hit_paths, long long item_id, static int maat_compile_hit_path_add(UT_array *hit_paths, long long item_id,
@@ -1602,8 +1598,8 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr
} }
void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state,
long long item_id, long long group_id, int vtable_id, long long item_id, long long group_id,
int Nth_scan, int Nth_item_result) int vtable_id, int Nth_scan, int Nth_item_result)
{ {
if (compile_state->Nth_scan != Nth_scan) { if (compile_state->Nth_scan != Nth_scan) {
assert(compile_state->this_scan_hit_item_flag == 0); assert(compile_state->this_scan_hit_item_flag == 0);
@@ -1618,16 +1614,15 @@ void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state
} }
void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state,
void *compile_runtime, long long group_id, struct compile_runtime *compile_rt,
int vtable_id) long long group_id, int vtable_id)
{ {
if (NULL == compile_state || NULL == compile_runtime) { if (NULL == compile_state || NULL == compile_rt) {
return; return;
} }
struct maat_literal_id literal_id = {group_id, vtable_id}; struct maat_literal_id literal_id = {group_id, vtable_id};
struct literal_clause *l2c_val = NULL; struct literal_clause *l2c_val = NULL;
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
HASH_FIND(hh, compile_rt->literal2clause_hash, &literal_id, sizeof(literal_id), l2c_val); HASH_FIND(hh, compile_rt->literal2clause_hash, &literal_id, sizeof(literal_id), l2c_val);
if (!l2c_val) { if (!l2c_val) {
@@ -1685,7 +1680,8 @@ void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt,
} }
struct compile_rule *compile_rule = NULL; struct compile_rule *compile_rule = NULL;
compile_rule = (struct compile_rule *)compile_runtime_get_user_data(compile_rt, compile_id); compile_rule = (struct compile_rule *)compile_runtime_get_user_data(compile_rt,
compile_id);
if (NULL == compile_rule) { if (NULL == compile_rule) {
return NULL; return NULL;
} }
@@ -1712,9 +1708,9 @@ int compile_runtime_add_compile(struct compile_runtime *compile_rt, struct compi
struct compile_rule *compile_rule = compile_rule_new(compile_item, schema, table_name, line); struct compile_rule *compile_rule = compile_rule_new(compile_item, schema, table_name, line);
compile_item_free(compile_item); compile_item_free(compile_item);
int updating_flag = rcu_hash_is_updating(compile_rt->cfg_hash_tbl); int updating_flag = rcu_hash_is_updating(compile_rt->cfg_hash);
if (1 == updating_flag) { if (1 == updating_flag) {
compile = rcu_updating_hash_find(compile_rt->cfg_hash_tbl, (char *)&compile_id, compile = rcu_updating_hash_find(compile_rt->cfg_hash, (char *)&compile_id,
sizeof(long long)); sizeof(long long));
if (compile != NULL) { if (compile != NULL) {
/**************************************************************** /****************************************************************
@@ -1734,10 +1730,10 @@ int compile_runtime_add_compile(struct compile_runtime *compile_rt, struct compi
assert(compile != NULL); assert(compile != NULL);
maat_compile_set(compile, table_name, compile_rule->declared_clause_num, maat_compile_set(compile, table_name, compile_rule->declared_clause_num,
compile_rule, (void (*)(void *))compile_rule_free); compile_rule, (void (*)(void *))compile_rule_free);
rcu_hash_add(compile_rt->cfg_hash_tbl, (char *)&compile_id, sizeof(long long), compile); rcu_hash_add(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long), compile);
} }
} else { } else {
compile = rcu_hash_find(compile_rt->cfg_hash_tbl, (char *)&compile_id, sizeof(long long)); compile = rcu_hash_find(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long));
if (compile != NULL) { if (compile != NULL) {
/******************************************************************************** /********************************************************************************
compile found in effective hash(added by group2compile runtime), which means compile found in effective hash(added by group2compile runtime), which means
@@ -1752,19 +1748,19 @@ int compile_runtime_add_compile(struct compile_runtime *compile_rt, struct compi
assert(copy_compile != NULL); assert(copy_compile != NULL);
/* delete compile from rcu hash */ /* delete compile from rcu hash */
rcu_hash_del(compile_rt->cfg_hash_tbl, (char *)&compile_id, sizeof(long long)); rcu_hash_del(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long));
/* copy_compile has group2compile_table info, so set compile_table info */ /* copy_compile has group2compile_table info, so set compile_table info */
maat_compile_set(copy_compile, table_name, compile_rule->declared_clause_num, maat_compile_set(copy_compile, table_name, compile_rule->declared_clause_num,
compile_rule, (void (*)(void *))compile_rule_free); compile_rule, (void (*)(void *))compile_rule_free);
/* add copy_compile to rcu hash */ /* add copy_compile to rcu hash */
rcu_hash_add(compile_rt->cfg_hash_tbl, (char *)&compile_id, sizeof(long long), copy_compile); rcu_hash_add(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long), copy_compile);
} else { } else {
compile = maat_compile_new(compile_rule->compile_id); compile = maat_compile_new(compile_rule->compile_id);
assert(compile != NULL); assert(compile != NULL);
maat_compile_set(compile, table_name, compile_rule->declared_clause_num, maat_compile_set(compile, table_name, compile_rule->declared_clause_num,
compile_rule, (void (*)(void *))compile_rule_free); compile_rule, (void (*)(void *))compile_rule_free);
rcu_hash_add(compile_rt->cfg_hash_tbl, (char *)&compile_id, sizeof(long long), compile); rcu_hash_add(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long), compile);
} }
} }
@@ -1775,11 +1771,11 @@ void compile_runtime_del_compile(struct compile_runtime *compile_rt, long long c
{ {
struct maat_compile *compile = NULL; struct maat_compile *compile = NULL;
int updating_flag = rcu_hash_is_updating(compile_rt->cfg_hash_tbl); int updating_flag = rcu_hash_is_updating(compile_rt->cfg_hash);
if (1 == updating_flag) { if (1 == updating_flag) {
// find in updating hash // find in updating hash
compile = rcu_updating_hash_find(compile_rt->cfg_hash_tbl, (char *)&compile_id, compile = rcu_updating_hash_find(compile_rt->cfg_hash, (char *)&compile_id,
sizeof(long long)); sizeof(long long));
if (compile != NULL) { if (compile != NULL) {
/**************************************************************** /****************************************************************
@@ -1795,12 +1791,12 @@ void compile_runtime_del_compile(struct compile_runtime *compile_rt, long long c
} }
if (0 == compile->actual_clause_num) { if (0 == compile->actual_clause_num) {
rcu_hash_del(compile_rt->cfg_hash_tbl, (char *)&compile_id, sizeof(long long)); rcu_hash_del(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long));
} }
} }
} else { } else {
// find in effective hash // find in effective hash
compile = rcu_hash_find(compile_rt->cfg_hash_tbl, (char *)&compile_id, sizeof(long long)); compile = rcu_hash_find(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long));
if (compile != NULL) { if (compile != NULL) {
/******************************************************************* /*******************************************************************
compile found in effective hash, which means compile found in effective hash, which means
@@ -1815,12 +1811,12 @@ void compile_runtime_del_compile(struct compile_runtime *compile_rt, long long c
assert(copy_compile != NULL); assert(copy_compile != NULL);
/* delete compile from rcu hash */ /* delete compile from rcu hash */
rcu_hash_del(compile_rt->cfg_hash_tbl, (char *)&compile_id, sizeof(long long)); rcu_hash_del(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long));
if (0 == copy_compile->actual_clause_num) { if (0 == copy_compile->actual_clause_num) {
maat_compile_free(copy_compile); maat_compile_free(copy_compile);
} else { } else {
rcu_hash_add(compile_rt->cfg_hash_tbl, (char *)&compile_id, rcu_hash_add(compile_rt->cfg_hash, (char *)&compile_id,
sizeof(long long), copy_compile); sizeof(long long), copy_compile);
} }
} }
@@ -1904,7 +1900,7 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
if (0 == is_valid) { if (0 == is_valid) {
//delete //delete
ret = maat_remove_group_from_compile(compile_rt->cfg_hash_tbl, g2c_item, ret = maat_remove_group_from_compile(compile_rt->cfg_hash, g2c_item,
compile_rt->logger); compile_rt->logger);
if (0 == ret) { if (0 == ret) {
if (g2c_item->not_flag) { if (g2c_item->not_flag) {
@@ -1916,7 +1912,8 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
} }
} else { } else {
//add //add
ret = maat_add_group_to_compile(compile_rt->cfg_hash_tbl, g2c_item, compile_rt->logger); ret = maat_add_group_to_compile(compile_rt->cfg_hash, g2c_item,
compile_rt->logger);
if (0 == ret) { if (0 == ret) {
if (g2c_item->not_flag) { if (g2c_item->not_flag) {
g2c_rt->not_flag_group++; g2c_rt->not_flag_group++;
@@ -1970,7 +1967,7 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name,
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
int updating_flag = rcu_hash_is_updating(compile_rt->cfg_hash_tbl); int updating_flag = rcu_hash_is_updating(compile_rt->cfg_hash);
if (0 == updating_flag) { if (0 == updating_flag) {
return 0; return 0;
} }
@@ -2003,14 +2000,14 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name,
compile_rt->bm = new_bool_matcher; compile_rt->bm = new_bool_matcher;
compile_rt->literal2clause_hash = new_literal2clause; compile_rt->literal2clause_hash = new_literal2clause;
rcu_hash_commit(compile_rt->cfg_hash_tbl); rcu_hash_commit(compile_rt->cfg_hash);
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, NULL, maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, NULL,
garbage_bool_matcher_free); garbage_bool_matcher_free);
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_literal2clause, NULL, maat_garbage_bagging(compile_rt->ref_garbage_bin, old_literal2clause, NULL,
garbage_literal2clause_hash_free); garbage_literal2clause_hash_free);
compile_rt->rule_num = rcu_hash_count(compile_rt->cfg_hash_tbl); compile_rt->rule_num = rcu_hash_count(compile_rt->cfg_hash);
return ret; return ret;
} }
@@ -2100,10 +2097,10 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
hit_cnt = MAX_SCANNER_HIT_GROUP_NUM; hit_cnt = MAX_SCANNER_HIT_GROUP_NUM;
} }
struct maat *maat_instance = state->maat_instance; struct maat *maat_inst = state->maat_inst;
if (NULL == state->compile_state) { if (NULL == state->compile_state) {
state->compile_state = maat_compile_state_new(); state->compile_state = maat_compile_state_new();
alignment_int64_array_add(maat_instance->stat->maat_compile_state_cnt, alignment_int64_array_add(maat_inst->stat->maat_compile_state_cnt,
state->thread_id, 1); state->thread_id, 1);
} }
@@ -2118,17 +2115,17 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
if (state->compile_table_id > 0) { if (state->compile_table_id > 0) {
compile_table_id = state->compile_table_id; compile_table_id = state->compile_table_id;
} else { } else {
compile_table_id = table_manager_get_default_compile_table_id(maat_instance->tbl_mgr); compile_table_id = table_manager_get_default_compile_table_id(maat_inst->tbl_mgr);
} }
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, struct compile_runtime *compile_rt = table_manager_get_runtime(maat_inst->tbl_mgr,
compile_table_id); compile_table_id);
if (NULL == compile_rt) { if (NULL == compile_rt) {
return; return;
} }
int g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr); int g2g_table_id = table_manager_get_group2group_table_id(maat_inst->tbl_mgr);
void *g2g_rt = table_manager_get_runtime(maat_instance->tbl_mgr, g2g_table_id); void *g2g_rt = table_manager_get_runtime(maat_inst->tbl_mgr, g2g_table_id);
if (NULL == g2g_rt) { if (NULL == g2g_rt) {
return; return;
} }

42
src/maat_config_monitor.c
View File

@@ -392,7 +392,7 @@ void config_monitor_traverse(long long current_version, const char *idx_dir,
FREE(idx_path_array); FREE(idx_path_array);
} }
int load_maat_json_file(struct maat *maat_instance, const char *json_filename, int load_maat_json_file(struct maat *maat_inst, const char *json_filename,
char *err_str, size_t err_str_sz) char *err_str, size_t err_str_sz)
{ {
int ret = 0; int ret = 0;
@@ -404,18 +404,18 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
size_t decrypted_buff_sz = 0; size_t decrypted_buff_sz = 0;
size_t uncompressed_buff_sz = 0; size_t uncompressed_buff_sz = 0;
log_info(maat_instance->logger, MODULE_CONFIG_MONITOR, log_info(maat_inst->logger, MODULE_CONFIG_MONITOR,
"Maat initial with JSON file %s, formating...", "Maat initial with JSON file %s, formating...",
json_filename); json_filename);
if (strlen(maat_instance->opts.decrypt_key) && strlen(maat_instance->opts.decrypt_algo)) { if (strlen(maat_inst->opts.decrypt_key) && strlen(maat_inst->opts.decrypt_algo)) {
ret = decrypt_open(json_filename, maat_instance->opts.decrypt_key, ret = decrypt_open(json_filename, maat_inst->opts.decrypt_key,
maat_instance->opts.decrypt_algo, maat_inst->opts.decrypt_algo,
(unsigned char **)&decrypted_buff, (unsigned char **)&decrypted_buff,
&decrypted_buff_sz, &decrypted_buff_sz,
err_str, err_str_sz); err_str, err_str_sz);
if (ret < 0) { if (ret < 0) {
log_error(maat_instance->logger, MODULE_CONFIG_MONITOR, log_error(maat_inst->logger, MODULE_CONFIG_MONITOR,
"[%s:%d] Decrypt Maat JSON file %s failed", "[%s:%d] Decrypt Maat JSON file %s failed",
__FUNCTION__, __LINE__, json_filename); __FUNCTION__, __LINE__, json_filename);
return -1; return -1;
@@ -425,12 +425,12 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
json_buff_sz = decrypted_buff_sz; json_buff_sz = decrypted_buff_sz;
} }
if (maat_instance->opts.maat_json_is_gzipped) { if (maat_inst->opts.maat_json_is_gzipped) {
ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff,
&uncompressed_buff_sz); &uncompressed_buff_sz);
FREE(json_buff); FREE(json_buff);
if (ret < 0) { if (ret < 0) {
log_error(maat_instance->logger, MODULE_CONFIG_MONITOR, log_error(maat_inst->logger, MODULE_CONFIG_MONITOR,
"[%s:%d] Uncompress Maat JSON file %s failed", "[%s:%d] Uncompress Maat JSON file %s failed",
__FUNCTION__, __LINE__, json_filename); __FUNCTION__, __LINE__, json_filename);
return -1; return -1;
@@ -444,7 +444,7 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
if (NULL == json_buff) { if (NULL == json_buff) {
ret = load_file_to_memory(json_filename, &json_buff, &json_buff_sz); ret = load_file_to_memory(json_filename, &json_buff, &json_buff_sz);
if (ret < 0) { if (ret < 0) {
log_error(maat_instance->logger, MODULE_CONFIG_MONITOR, log_error(maat_inst->logger, MODULE_CONFIG_MONITOR,
"[%s:%d] Read Maat JSON file %s failed", "[%s:%d] Read Maat JSON file %s failed",
__FUNCTION__, __LINE__, json_filename); __FUNCTION__, __LINE__, json_filename);
return -1; return -1;
@@ -452,27 +452,27 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
} }
ret = json2iris((const char*)json_buff, json_filename, NULL, ret = json2iris((const char*)json_buff, json_filename, NULL,
maat_instance->opts.json_ctx.iris_file, maat_inst->opts.json_ctx.iris_file,
sizeof(maat_instance->opts.json_ctx.iris_file), sizeof(maat_inst->opts.json_ctx.iris_file),
strlen(maat_instance->opts.decrypt_key) ? maat_instance->opts.decrypt_key : NULL, strlen(maat_inst->opts.decrypt_key) ? maat_inst->opts.decrypt_key : NULL,
strlen(maat_instance->opts.decrypt_algo) ? maat_instance->opts.decrypt_algo : NULL, strlen(maat_inst->opts.decrypt_algo) ? maat_inst->opts.decrypt_algo : NULL,
maat_instance->logger); maat_inst->logger);
FREE(json_buff); FREE(json_buff);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
ret = stat(json_filename, &fstat_buf); ret = stat(json_filename, &fstat_buf);
maat_instance->opts.json_ctx.last_md5_time = fstat_buf.st_ctim; maat_inst->opts.json_ctx.last_md5_time = fstat_buf.st_ctim;
md5_file(maat_instance->opts.json_ctx.json_file, maat_instance->opts.json_ctx.effective_json_md5); md5_file(maat_inst->opts.json_ctx.json_file, maat_inst->opts.json_ctx.effective_json_md5);
log_info(maat_instance->logger, MODULE_CONFIG_MONITOR, log_info(maat_inst->logger, MODULE_CONFIG_MONITOR,
"JSON file %s md5: %s, generate index file %s OK", "JSON file %s md5: %s, generate index file %s OK",
maat_instance->opts.json_ctx.json_file, maat_inst->opts.json_ctx.json_file,
maat_instance->opts.json_ctx.effective_json_md5, maat_inst->opts.json_ctx.effective_json_md5,
maat_instance->opts.json_ctx.iris_file); maat_inst->opts.json_ctx.iris_file);
maat_instance->opts.input_mode = DATA_SOURCE_JSON_FILE; maat_inst->opts.input_mode = DATA_SOURCE_JSON_FILE;
return 0; return 0;
} }

2
src/maat_expr.c
View File

@@ -875,7 +875,7 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name,
struct adapter_hs *old_adapter_hs = NULL; struct adapter_hs *old_adapter_hs = NULL;
if (rule_cnt > 0) { if (rule_cnt > 0) {
new_adapter_hs = adapter_hs_new(expr_rt->n_worker_thread, rules, real_rule_cnt, new_adapter_hs = adapter_hs_new(rules, real_rule_cnt, expr_rt->n_worker_thread,
expr_rt->logger); expr_rt->logger);
if (NULL == new_adapter_hs) { if (NULL == new_adapter_hs) {
log_error(expr_rt->logger, MODULE_EXPR, log_error(expr_rt->logger, MODULE_EXPR,

4
src/maat_flag.c
View File

@@ -49,7 +49,6 @@ struct flag_runtime {
struct rcu_hash_table *item_hash; // <item_id, struct flag_item> struct rcu_hash_table *item_hash; // <item_id, struct flag_item>
long long rule_num; long long rule_num;
long long version;
size_t n_worker_thread; size_t n_worker_thread;
struct log_handle *logger; struct log_handle *logger;
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
@@ -510,11 +509,10 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name,
} }
flag_rt->rule_num = rule_cnt; flag_rt->rule_num = rule_cnt;
flag_rt->version = maat_rt_version;
log_info(flag_rt->logger, MODULE_FLAG, log_info(flag_rt->logger, MODULE_FLAG,
"table[%s] commit %zu flag rules and rebuild flag_matcher completed," "table[%s] commit %zu flag rules and rebuild flag_matcher completed,"
" version:%lld", table_name, rule_cnt, flag_rt->version); " version:%lld", table_name, rule_cnt, maat_rt_version);
if (rules != NULL) { if (rules != NULL) {
FREE(rules); FREE(rules);

6
src/maat_fqdn_plugin.c
View File

@@ -33,7 +33,6 @@ struct fqdn_plugin_schema {
struct fqdn_plugin_runtime { struct fqdn_plugin_runtime {
struct FQDN_engine *engine; struct FQDN_engine *engine;
struct ex_data_runtime *ex_data_rt; struct ex_data_runtime *ex_data_rt;
long long version;
long long rule_num; long long rule_num;
long long update_err_cnt; long long update_err_cnt;
size_t n_worker_thread; size_t n_worker_thread;
@@ -499,13 +498,10 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name
} }
fqdn_plugin_rt->rule_num = rule_cnt; fqdn_plugin_rt->rule_num = rule_cnt;
if (maat_rt_version != 0) {
fqdn_plugin_rt->version = maat_rt_version;
}
log_info(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN, log_info(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
"table[%s] commit %zu fqdn_plugin rules and rebuild FQDN engine completed" "table[%s] commit %zu fqdn_plugin rules and rebuild FQDN engine completed"
", version:%lld", table_name, rule_cnt, fqdn_plugin_rt->version); ", version:%lld", table_name, rule_cnt, maat_rt_version);
if (rules != NULL) { if (rules != NULL) {
FREE(rules); FREE(rules);

4
src/maat_group.c
View File

@@ -62,7 +62,6 @@ struct maat_group_topology {
struct group2group_runtime { struct group2group_runtime {
struct maat_group_topology *group_topo; struct maat_group_topology *group_topo;
struct maat_group_topology *updating_group_topo; struct maat_group_topology *updating_group_topo;
long long version;
long long rule_num; long long rule_num;
long long excl_rule_num; //exclude g2g rule num long long excl_rule_num; //exclude g2g rule num
long long update_err_cnt; long long update_err_cnt;
@@ -774,11 +773,10 @@ int group2group_runtime_commit(void *g2g_runtime, const char *table_name,
maat_garbage_bagging(g2g_rt->ref_garbage_bin, old_group_topo, NULL, maat_garbage_bagging(g2g_rt->ref_garbage_bin, old_group_topo, NULL,
garbage_maat_group_topology_free); garbage_maat_group_topology_free);
g2g_rt->version = maat_rt_version;
log_info(g2g_rt->logger, MODULE_GROUP, log_info(g2g_rt->logger, MODULE_GROUP,
"table[%s] commit %zu g2g rules and rebuild super_groups completed," "table[%s] commit %zu g2g rules and rebuild super_groups completed,"
" version:%lld", table_name, g2g_rt->rule_num, g2g_rt->version); " version:%lld", table_name, g2g_rt->rule_num, maat_rt_version);
return 0; return 0;
} }

4
src/maat_interval.c
View File

@@ -45,7 +45,6 @@ struct interval_runtime {
struct interval_matcher *matcher; struct interval_matcher *matcher;
struct rcu_hash_table *item_hash; // <item_id, struct interval_item> struct rcu_hash_table *item_hash; // <item_id, struct interval_item>
long long version;
long long rule_num; long long rule_num;
size_t n_worker_thread; size_t n_worker_thread;
struct log_handle *logger; struct log_handle *logger;
@@ -514,11 +513,10 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name,
} }
interval_rt->rule_num = rule_cnt; interval_rt->rule_num = rule_cnt;
interval_rt->version = maat_rt_version;
log_info(interval_rt->logger, MODULE_INTERVAL, log_info(interval_rt->logger, MODULE_INTERVAL,
"table[%s] commit %zu interval rules and rebuild interval_matcher " "table[%s] commit %zu interval rules and rebuild interval_matcher "
"completed, version:%lld", table_name, rule_cnt, interval_rt->version); "completed, version:%lld", table_name, rule_cnt, maat_rt_version);
if (rules != NULL) { if (rules != NULL) {
FREE(rules); FREE(rules);

4
src/maat_ip.c
View File

@@ -69,7 +69,6 @@ struct ip_runtime {
struct interval_matcher *intval_matcher; struct interval_matcher *intval_matcher;
struct rcu_hash_table *item_hash; // <item_id, struct ip_item> struct rcu_hash_table *item_hash; // <item_id, struct ip_item>
long long version;
long long rule_num; long long rule_num;
long long ipv6_rule_num; long long ipv6_rule_num;
size_t n_worker_thread; size_t n_worker_thread;
@@ -641,11 +640,10 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name,
} }
ip_rt->rule_num = rule_cnt; ip_rt->rule_num = rule_cnt;
ip_rt->version = maat_rt_version;
log_info(ip_rt->logger, MODULE_IP, log_info(ip_rt->logger, MODULE_IP,
"table[%s] commit %zu ip rules and rebuild ip_matcher completed" "table[%s] commit %zu ip rules and rebuild ip_matcher completed"
", version:%lld", table_name, rule_cnt, ip_rt->version); ", version:%lld", table_name, rule_cnt, maat_rt_version);
if (rules != NULL) { if (rules != NULL) {
FREE(rules); FREE(rules);

6
src/maat_ip_plugin.c
View File

@@ -37,7 +37,6 @@ struct ip_plugin_schema {
struct ip_plugin_runtime { struct ip_plugin_runtime {
struct ip_matcher *ip_matcher; struct ip_matcher *ip_matcher;
struct ex_data_runtime *ex_data_rt; struct ex_data_runtime *ex_data_rt;
long long version;
long long rule_num; long long rule_num;
long long update_err_cnt; long long update_err_cnt;
size_t n_worker_thread; size_t n_worker_thread;
@@ -539,13 +538,10 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,
} }
ip_plugin_rt->rule_num = rule_cnt; ip_plugin_rt->rule_num = rule_cnt;
if (maat_rt_version != 0) {
ip_plugin_rt->version = maat_rt_version;
}
log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN, log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"table[%s] commit %zu ip_plugin rules and rebuild ip_matcher " "table[%s] commit %zu ip_plugin rules and rebuild ip_matcher "
"completed, version:%lld", table_name, rule_cnt, ip_plugin_rt->version); "completed, version:%lld", table_name, rule_cnt, maat_rt_version);
if (rules != NULL) { if (rules != NULL) {
FREE(rules); FREE(rules);

6
src/maat_plugin.c
View File

@@ -33,7 +33,6 @@ struct plugin_callback_schema {
struct plugin_runtime { struct plugin_runtime {
long long acc_line_num; long long acc_line_num;
struct ex_data_runtime *ex_data_rt; struct ex_data_runtime *ex_data_rt;
long long version;
long long rule_num; long long rule_num;
long long update_err_cnt; long long update_err_cnt;
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
@@ -572,13 +571,10 @@ int plugin_runtime_commit(void *plugin_runtime, const char *table_name,
ex_data_runtime_commit(ex_data_rt); ex_data_runtime_commit(ex_data_rt);
plugin_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt); plugin_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt);
if (maat_rt_version != 0) {
plugin_rt->version = maat_rt_version;
}
log_info(plugin_rt->logger, MODULE_PLUGIN, log_info(plugin_rt->logger, MODULE_PLUGIN,
"table[%s] commit %zu plugin rules, version:%lld", "table[%s] commit %zu plugin rules, version:%lld",
table_name, plugin_rt->rule_num, plugin_rt->version); table_name, plugin_rt->rule_num, maat_rt_version);
return 0; return 0;
} }

50
src/maat_redis_monitor.c
View File

@@ -118,7 +118,7 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns,
} }
int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list,
int rule_num, struct maat *maat_instance, const char *dir) int rule_num, struct maat *maat_inst, const char *dir)
{ {
int rule_with_foreign_key = 0; int rule_with_foreign_key = 0;
@@ -127,9 +127,9 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list,
continue; continue;
} }
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, rule_list[i].table_name); int table_id = table_manager_get_table_id(maat_inst->tbl_mgr, rule_list[i].table_name);
void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); void *schema = table_manager_get_schema(maat_inst->tbl_mgr, table_id);
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
if (!schema || table_type != TABLE_TYPE_PLUGIN) { if (!schema || table_type != TABLE_TYPE_PLUGIN) {
continue; continue;
} }
@@ -141,7 +141,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list,
continue; continue;
} }
_get_foregin_keys(rule_list+i, foreign_columns, n_foreign_column, dir, maat_instance->logger); _get_foregin_keys(rule_list+i, foreign_columns, n_foreign_column, dir, maat_inst->logger);
rule_with_foreign_key++; rule_with_foreign_key++;
} }
@@ -1380,14 +1380,14 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
int no_table_num = 0; int no_table_num = 0;
int call_update_num = 0; int call_update_num = 0;
int valid_column = -1; int valid_column = -1;
struct maat *maat_instance = (struct maat *)u_param; struct maat *maat_inst = (struct maat *)u_param;
//authorized to write //authorized to write
if (mr_ctx->write_ctx != NULL && mr_ctx->write_ctx->err == 0) { if (mr_ctx->write_ctx != NULL && mr_ctx->write_ctx->err == 0) {
//For thread safe, deliberately use redis_read_ctx but not redis_write_ctx. //For thread safe, deliberately use redis_read_ctx but not redis_write_ctx.
if (1 == redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock, mr_expire_lock_timeout_ms)) { if (1 == redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock, mr_expire_lock_timeout_ms)) {
check_maat_expiration(mr_ctx->read_ctx, maat_instance->logger); check_maat_expiration(mr_ctx->read_ctx, maat_inst->logger);
cleanup_update_status(mr_ctx->read_ctx, maat_instance->logger); cleanup_update_status(mr_ctx->read_ctx, maat_inst->logger);
redlock_unlock(mr_ctx->read_ctx, mr_expire_lock); redlock_unlock(mr_ctx->read_ctx, mr_expire_lock);
} }
} }
@@ -1402,12 +1402,12 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
redisFree(mr_ctx->read_ctx); redisFree(mr_ctx->read_ctx);
mr_ctx->read_ctx = NULL; mr_ctx->read_ctx = NULL;
} }
log_info(maat_instance->logger, MODULE_REDIS_MONITOR, "Reconnecting..."); log_info(maat_inst->logger, MODULE_REDIS_MONITOR, "Reconnecting...");
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip,
mr_ctx->redis_port, mr_ctx->redis_port,
mr_ctx->redis_db, mr_ctx->redis_db,
maat_instance->logger); maat_inst->logger);
if (NULL == mr_ctx->read_ctx) { if (NULL == mr_ctx->read_ctx) {
return; return;
} else { } else {
@@ -1420,11 +1420,11 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
int update_type = MAAT_UPDATE_TYPE_INC; int update_type = MAAT_UPDATE_TYPE_INC;
int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version, int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version,
maat_instance->load_specific_version, maat_inst->load_specific_version,
&new_version, maat_instance->tbl_mgr, &new_version, maat_inst->tbl_mgr,
&rule_list, &update_type, &rule_list, &update_type,
maat_instance->opts.cumulative_update_off, maat_inst->opts.cumulative_update_off,
maat_instance->logger); maat_inst->logger);
//redis communication error //redis communication error
if (rule_num < 0) { if (rule_num < 0) {
redisFree(mr_ctx->read_ctx); redisFree(mr_ctx->read_ctx);
@@ -1432,7 +1432,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
return; return;
} }
maat_instance->load_specific_version = 0;//only valid for one time. maat_inst->load_specific_version = 0;//only valid for one time.
//error or nothing changed //error or nothing changed
if (0 == rule_num && update_type == MAAT_UPDATE_TYPE_INC) { if (0 == rule_num && update_type == MAAT_UPDATE_TYPE_INC) {
return; return;
@@ -1440,12 +1440,12 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
if (rule_num > 0) { if (rule_num > 0) {
ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num, ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num,
0, maat_instance->logger); 0, maat_inst->logger);
//redis communication error //redis communication error
if (ret < 0) { if (ret < 0) {
redisFree(mr_ctx->read_ctx); redisFree(mr_ctx->read_ctx);
mr_ctx->read_ctx = NULL; mr_ctx->read_ctx = NULL;
log_error(maat_instance->logger, MODULE_REDIS_MONITOR, log_error(maat_inst->logger, MODULE_REDIS_MONITOR,
"[%s:%d] Get Redis value failed, abandon update and close connection", "[%s:%d] Get Redis value failed, abandon update and close connection",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
goto clean_up; goto clean_up;
@@ -1458,21 +1458,21 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
} }
if (empty_value_num == rule_num) { if (empty_value_num == rule_num) {
log_info(maat_instance->logger, MODULE_REDIS_MONITOR, log_info(maat_inst->logger, MODULE_REDIS_MONITOR,
"All %d rules are empty, abandon update", empty_value_num); "All %d rules are empty, abandon update", empty_value_num);
goto clean_up; goto clean_up;
} }
ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num, ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num,
maat_instance, maat_instance->opts.foreign_cont_dir); maat_inst, maat_inst->opts.foreign_cont_dir);
if (ret > 0) { if (ret > 0) {
maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0, maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0,
maat_instance->logger); maat_inst->logger);
} }
} }
start_fn(new_version, update_type, u_param); start_fn(new_version, update_type, u_param);
log_info(maat_instance->logger, MODULE_REDIS_MONITOR, log_info(maat_inst->logger, MODULE_REDIS_MONITOR,
"Start %s update: %lld -> %lld (%d entries)", "Start %s update: %lld -> %lld (%d entries)",
update_type == MAAT_UPDATE_TYPE_INC ? "INC" : "FULL", update_type == MAAT_UPDATE_TYPE_INC ? "INC" : "FULL",
version, new_version, rule_num); version, new_version, rule_num);
@@ -1482,7 +1482,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
continue; continue;
} }
table_id = table_manager_get_table_id(maat_instance->tbl_mgr, rule_list[i].table_name); table_id = table_manager_get_table_id(maat_inst->tbl_mgr, rule_list[i].table_name);
//Unrecognized table. //Unrecognized table.
if (table_id < 0) { if (table_id < 0) {
no_table_num++; no_table_num++;
@@ -1490,10 +1490,10 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
} }
if (rule_list[i].op == MAAT_OP_DEL) { if (rule_list[i].op == MAAT_OP_DEL) {
valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id); valid_column = table_manager_get_valid_column(maat_inst->tbl_mgr, table_id);
ret = invalidate_line(rule_list[i].table_line, valid_column); ret = invalidate_line(rule_list[i].table_line, valid_column);
if (ret < 0) { if (ret < 0) {
log_error(maat_instance->logger, MODULE_REDIS_MONITOR, log_error(maat_inst->logger, MODULE_REDIS_MONITOR,
"[%s:%d] Invalidate line failed, invaid format %s", "[%s:%d] Invalidate line failed, invaid format %s",
__FUNCTION__, __LINE__, rule_list[i].table_line); __FUNCTION__, __LINE__, rule_list[i].table_line);
continue; continue;
@@ -1511,7 +1511,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
finish_fn(u_param); finish_fn(u_param);
if (call_update_num < rule_num) { if (call_update_num < rule_num) {
log_error(maat_instance->logger, MODULE_REDIS_MONITOR, log_error(maat_inst->logger, MODULE_REDIS_MONITOR,
"[%s:%d] Load %d entries to match engine, no table: %d, empty value: %d", "[%s:%d] Load %d entries to match engine, no table: %d, empty value: %d",
__FUNCTION__, __LINE__, call_update_num, no_table_num, empty_value_num); __FUNCTION__, __LINE__, call_update_num, no_table_num, empty_value_num);
} }

319
src/maat_rule.c
View File

@@ -32,25 +32,25 @@
#define MODULE_MAAT_RULE module_name_str("maat.rule") #define MODULE_MAAT_RULE module_name_str("maat.rule")
struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance) struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_inst)
{ {
struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1); struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1);
maat_rt->version = version; maat_rt->version = version;
int ret = table_manager_runtime_create(maat_instance->tbl_mgr, int ret = table_manager_runtime_create(maat_inst->tbl_mgr,
maat_instance->opts.nr_worker_thread, maat_inst->opts.nr_worker_thread,
maat_instance->garbage_bin); maat_inst->garbage_bin);
if (ret < 0) { if (ret < 0) {
FREE(maat_rt); FREE(maat_rt);
return NULL; return NULL;
} }
maat_rt->ref_tbl_mgr = maat_instance->tbl_mgr; maat_rt->ref_tbl_mgr = maat_inst->tbl_mgr;
maat_rt->max_table_num = table_manager_table_size(maat_instance->tbl_mgr); maat_rt->max_table_num = table_manager_table_size(maat_inst->tbl_mgr);
maat_rt->sequence_map = maat_kv_store_new(); maat_rt->sequence_map = maat_kv_store_new();
maat_rt->logger = maat_instance->logger; maat_rt->logger = maat_inst->logger;
maat_rt->ref_garbage_bin = maat_instance->garbage_bin; maat_rt->ref_garbage_bin = maat_inst->garbage_bin;
maat_rt->ref_cnt = alignment_int64_array_alloc(maat_instance->opts.nr_worker_thread); maat_rt->ref_cnt = alignment_int64_array_alloc(maat_inst->opts.nr_worker_thread);
return maat_rt; return maat_rt;
} }
@@ -67,30 +67,32 @@ void maat_runtime_commit(struct maat_runtime *maat_rt, int update_type,
void maat_start_cb(long long new_version, int update_type, void *u_param) void maat_start_cb(long long new_version, int update_type, void *u_param)
{ {
struct maat *maat_instance = (struct maat *)u_param;
maat_instance->new_version = new_version;
size_t i = 0; size_t i = 0;
size_t max_table_cnt = table_manager_table_size(maat_instance->tbl_mgr);
enum table_type table_type = TABLE_TYPE_INVALID; enum table_type table_type = TABLE_TYPE_INVALID;
struct maat *maat_inst = (struct maat *)u_param;
size_t max_table_cnt = table_manager_table_size(maat_inst->tbl_mgr);
maat_inst->new_version = new_version;
if (update_type == MAAT_UPDATE_TYPE_FULL) { if (update_type == MAAT_UPDATE_TYPE_FULL) {
maat_instance->creating_maat_rt = maat_runtime_create(new_version, maat_instance); maat_inst->creating_maat_rt = maat_runtime_create(new_version, maat_inst);
for (i = 0; i < max_table_cnt; i++) { for (i = 0; i < max_table_cnt; i++) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, i); table_type = table_manager_get_table_type(maat_inst->tbl_mgr, i);
if (table_type == TABLE_TYPE_COMPILE) { if (table_type == TABLE_TYPE_COMPILE) {
// compile runtime need a reference to maat runtime // compile runtime need a reference to maat runtime
void *compile_rt = table_manager_get_updating_runtime(maat_instance->tbl_mgr, i); void *compile_rt = table_manager_get_updating_runtime(maat_inst->tbl_mgr, i);
compile_runtime_init(compile_rt, maat_instance->creating_maat_rt, NULL); compile_runtime_init(compile_rt, maat_inst->creating_maat_rt);
} }
} }
} else { } else {
maat_instance->maat_version = new_version; maat_inst->maat_version = new_version;
} }
for (i = 0; i < max_table_cnt; i++) { for (i = 0; i < max_table_cnt; i++) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, i); table_type = table_manager_get_table_type(maat_inst->tbl_mgr, i);
if (table_type == TABLE_TYPE_PLUGIN) { if (table_type == TABLE_TYPE_PLUGIN) {
void *schema = table_manager_get_schema(maat_instance->tbl_mgr, i); void *schema = table_manager_get_schema(maat_inst->tbl_mgr, i);
plugin_table_all_callback_start((struct plugin_schema *)schema, update_type); plugin_table_all_callback_start((struct plugin_schema *)schema, update_type);
} }
} }
@@ -102,30 +104,30 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param)
return 0; return 0;
} }
struct maat *maat_instance =(struct maat *)u_param; struct maat *maat_inst =(struct maat *)u_param;
struct maat_runtime* maat_rt = NULL; struct maat_runtime* maat_rt = NULL;
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, table_name); int table_id = table_manager_get_table_id(maat_inst->tbl_mgr, table_name);
if (table_id < 0) { if (table_id < 0) {
log_error(maat_instance->logger, MODULE_MAAT_RULE, log_error(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] update warning, unknown table name %s", "[%s:%d] update warning, unknown table name %s",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
return -1; return -1;
} }
void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); void *schema = table_manager_get_schema(maat_inst->tbl_mgr, table_id);
if (NULL == schema) { if (NULL == schema) {
log_error(maat_instance->logger, MODULE_MAAT_RULE, log_error(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] update warning, table name %s doesn't have table schema", "[%s:%d] update warning, table name %s doesn't have table schema",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
return -1; return -1;
} }
int update_type = MAAT_UPDATE_TYPE_INC; int update_type = MAAT_UPDATE_TYPE_INC;
if (maat_instance->creating_maat_rt != NULL) { //Full update if (maat_inst->creating_maat_rt != NULL) { //Full update
maat_rt = maat_instance->creating_maat_rt; maat_rt = maat_inst->creating_maat_rt;
update_type = MAAT_UPDATE_TYPE_FULL; update_type = MAAT_UPDATE_TYPE_FULL;
} else { } else {
maat_rt = maat_instance->maat_rt; maat_rt = maat_inst->maat_rt;
} }
table_manager_update_runtime(maat_rt->ref_tbl_mgr, table_name, table_id, line, update_type); table_manager_update_runtime(maat_rt->ref_tbl_mgr, table_name, table_id, line, update_type);
@@ -166,107 +168,107 @@ void maat_plugin_table_all_callback_finish(struct table_manager *tbl_mgr)
void maat_finish_cb(void *u_param) void maat_finish_cb(void *u_param)
{ {
struct maat *maat_instance = (struct maat *)u_param; struct maat *maat_inst = (struct maat *)u_param;
maat_plugin_table_all_callback_finish(maat_instance->tbl_mgr); maat_plugin_table_all_callback_finish(maat_inst->tbl_mgr);
if (maat_instance->creating_maat_rt != NULL) { if (maat_inst->creating_maat_rt != NULL) {
maat_runtime_commit(maat_instance->creating_maat_rt, MAAT_UPDATE_TYPE_FULL, maat_runtime_commit(maat_inst->creating_maat_rt, MAAT_UPDATE_TYPE_FULL,
maat_instance->creating_maat_rt->version, maat_instance->logger); maat_inst->creating_maat_rt->version, maat_inst->logger);
maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt); maat_inst->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_inst->creating_maat_rt);
log_info(maat_instance->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Full config version %llu load %d entries complete", "Full config version %llu load %d entries complete",
maat_instance->creating_maat_rt->version, maat_inst->creating_maat_rt->version,
maat_instance->creating_maat_rt->rule_num); maat_inst->creating_maat_rt->rule_num);
} else if (maat_instance->maat_rt != NULL) { } else if (maat_inst->maat_rt != NULL) {
maat_instance->maat_rt->version = maat_instance->maat_version; maat_inst->maat_rt->version = maat_inst->maat_version;
maat_runtime_commit(maat_instance->maat_rt, MAAT_UPDATE_TYPE_INC, maat_runtime_commit(maat_inst->maat_rt, MAAT_UPDATE_TYPE_INC,
maat_instance->maat_rt->version, maat_instance->logger); maat_inst->maat_rt->version, maat_inst->logger);
maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt); maat_inst->maat_rt->rule_num = maat_runtime_rule_num(maat_inst->maat_rt);
log_info(maat_instance->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Inc config version %llu load %d entries complete", "Inc config version %llu load %d entries complete",
maat_instance->maat_rt->version, maat_inst->maat_rt->version,
maat_instance->maat_rt->rule_num); maat_inst->maat_rt->rule_num);
} else { } else {
log_info(maat_instance->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Version %d has no valid rules, plugin callback complete.", "Version %d has no valid rules, plugin callback complete.",
maat_instance->maat_version); maat_inst->maat_version);
} }
maat_instance->new_version = INVALID_VERSION; maat_inst->new_version = INVALID_VERSION;
} }
void maat_read_full_config(struct maat *maat_instance) void maat_read_full_config(struct maat *maat_inst)
{ {
int ret = -1; int ret = -1;
char err_str[NAME_MAX] = {0}; char err_str[NAME_MAX] = {0};
struct source_redis_ctx *redis_ctx = NULL; struct source_redis_ctx *redis_ctx = NULL;
switch (maat_instance->opts.input_mode) { switch (maat_inst->opts.input_mode) {
case DATA_SOURCE_REDIS: case DATA_SOURCE_REDIS:
redis_ctx = &(maat_instance->opts.redis_ctx); redis_ctx = &(maat_inst->opts.redis_ctx);
log_info(maat_instance->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Maat initiate from Redis %s:%hu db%d", "Maat initiate from Redis %s:%hu db%d",
redis_ctx->redis_ip, redis_ctx->redis_port, redis_ctx->redis_db); redis_ctx->redis_ip, redis_ctx->redis_port, redis_ctx->redis_db);
redis_ctx->read_ctx = maat_cmd_connect_redis(redis_ctx->redis_ip, redis_ctx->read_ctx = maat_cmd_connect_redis(redis_ctx->redis_ip,
redis_ctx->redis_port, redis_ctx->redis_port,
redis_ctx->redis_db, redis_ctx->redis_db,
maat_instance->logger); maat_inst->logger);
if (redis_ctx->read_ctx != NULL) { if (redis_ctx->read_ctx != NULL) {
redis_monitor_traverse(maat_instance->maat_version, redis_ctx, redis_monitor_traverse(maat_inst->maat_version, redis_ctx,
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance); maat_inst);
} }
if (NULL == maat_instance->creating_maat_rt) { if (NULL == maat_inst->creating_maat_rt) {
log_error(maat_instance->logger, MODULE_MAAT_RULE, log_error(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] At initiation: NO effective rule in redis %s:%hu db%d", "[%s:%d] At initiation: NO effective rule in redis %s:%hu db%d",
__FUNCTION__, __LINE__, redis_ctx->redis_ip, redis_ctx->redis_port, __FUNCTION__, __LINE__, redis_ctx->redis_ip, redis_ctx->redis_port,
redis_ctx->redis_db); redis_ctx->redis_db);
} }
break; break;
case DATA_SOURCE_IRIS_FILE: case DATA_SOURCE_IRIS_FILE:
config_monitor_traverse(maat_instance->maat_version, config_monitor_traverse(maat_inst->maat_version,
maat_instance->opts.iris_ctx.full_idx_dir, maat_inst->opts.iris_ctx.full_idx_dir,
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance, maat_instance->opts.decrypt_key, maat_inst, maat_inst->opts.decrypt_key,
maat_instance->logger); maat_inst->logger);
if (NULL == maat_instance->creating_maat_rt) { if (NULL == maat_inst->creating_maat_rt) {
log_error(maat_instance->logger, MODULE_MAAT_RULE, log_error(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] At initiation: NO effective rule in %s", "[%s:%d] At initiation: NO effective rule in %s",
__FUNCTION__, __LINE__, maat_instance->opts.iris_ctx.full_idx_dir); __FUNCTION__, __LINE__, maat_inst->opts.iris_ctx.full_idx_dir);
} }
break; break;
case DATA_SOURCE_JSON_FILE: case DATA_SOURCE_JSON_FILE:
ret = load_maat_json_file(maat_instance, maat_instance->opts.json_ctx.json_file, ret = load_maat_json_file(maat_inst, maat_inst->opts.json_ctx.json_file,
err_str, sizeof(err_str)); err_str, sizeof(err_str));
if (ret < 0) { if (ret < 0) {
log_error(maat_instance->logger, MODULE_MAAT_RULE, log_error(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] Maat re-initiate with JSON file %s failed: %s", "[%s:%d] Maat re-initiate with JSON file %s failed: %s",
__FUNCTION__, __LINE__, maat_instance->opts.json_ctx.json_file, err_str); __FUNCTION__, __LINE__, maat_inst->opts.json_ctx.json_file, err_str);
} }
config_monitor_traverse(maat_instance->maat_version, config_monitor_traverse(maat_inst->maat_version,
maat_instance->opts.json_ctx.iris_file, maat_inst->opts.json_ctx.iris_file,
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance, maat_instance->opts.decrypt_key, maat_inst, maat_inst->opts.decrypt_key,
maat_instance->logger); maat_inst->logger);
if (NULL == maat_instance->creating_maat_rt) { if (NULL == maat_inst->creating_maat_rt) {
log_error(maat_instance->logger, MODULE_MAAT_RULE, log_error(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] At initiation: NO effective rule in %s", "[%s:%d] At initiation: NO effective rule in %s",
__FUNCTION__, __LINE__, maat_instance->opts.json_ctx.iris_file); __FUNCTION__, __LINE__, maat_inst->opts.json_ctx.iris_file);
} }
break; break;
default: default:
break; break;
} }
maat_instance->maat_rt = maat_instance->creating_maat_rt; maat_inst->maat_rt = maat_inst->creating_maat_rt;
maat_instance->creating_maat_rt = NULL; maat_inst->creating_maat_rt = NULL;
maat_instance->is_running = 1; maat_inst->is_running = 1;
if (maat_instance->maat_rt != NULL) { if (maat_inst->maat_rt != NULL) {
maat_instance->maat_version = maat_instance->maat_rt->version; maat_inst->maat_version = maat_inst->maat_rt->version;
maat_instance->last_full_version = maat_instance->maat_rt->version; maat_inst->last_full_version = maat_inst->maat_rt->version;
} }
} }
@@ -326,74 +328,77 @@ void *rule_monitor_loop(void *arg)
{ {
/* Defined by prctl: The name can be up to 16 bytes long, and should /* Defined by prctl: The name can be up to 16 bytes long, and should
be null terminated if it contains fewer bytes. */ be null terminated if it contains fewer bytes. */
char maat_name[16] = {0}; char maat_name[MAX_INSTANCE_NAME_LEN + 1] = {0};
struct maat *maat_instance = (struct maat *)arg; struct maat *maat_inst = (struct maat *)arg;
if (strlen(maat_instance->opts.instance_name) > 0) { if (strlen(maat_inst->opts.inst_name) > 0) {
snprintf(maat_name, sizeof(maat_name), "MAAT_%s", maat_instance->opts.instance_name); snprintf(maat_name, sizeof(maat_name), "%s", maat_inst->opts.inst_name);
} else { } else {
snprintf(maat_name, sizeof(maat_name), "MAAT"); snprintf(maat_name, sizeof(maat_name), "MAAT_LOOP");
} }
int ret = prctl(PR_SET_NAME, (unsigned long long)maat_name, NULL, NULL, NULL); int ret = prctl(PR_SET_NAME, (unsigned long long)maat_name, NULL, NULL, NULL);
assert(ret >= 0); assert(ret >= 0);
pthread_mutex_lock(&(maat_instance->background_update_mutex)); pthread_mutex_lock(&(maat_inst->background_update_mutex));
/* if deferred load on */ /* if deferred load on */
if (maat_instance->opts.deferred_load_on != 0) { if (maat_inst->opts.deferred_load_on != 0) {
log_info(maat_instance->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Deferred Loading ON, updating in %s:%d", __FUNCTION__, __LINE__); "Deferred Loading ON, updating in %s:%d", __FUNCTION__, __LINE__);
maat_read_full_config(maat_instance); maat_read_full_config(maat_inst);
} }
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_inst->background_update_mutex));
char md5_tmp[MD5_DIGEST_LENGTH * 2 + 1] = {0}; char md5_tmp[MD5_DIGEST_LENGTH * 2 + 1] = {0};
char err_str[NAME_MAX] = {0}; char err_str[NAME_MAX] = {0};
struct stat attrib; struct stat attrib;
while (maat_instance->is_running) {
while (maat_inst->is_running) {
if (time(NULL) % 10 == 0) { if (time(NULL) % 10 == 0) {
log_info(maat_instance->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"%s thread still alive.........", __FUNCTION__); "%s thread still alive.........", __FUNCTION__);
} }
usleep(maat_instance->opts.rule_update_checking_interval_ms * 1000); usleep(maat_inst->opts.rule_update_checking_interval_ms * 1000);
if (0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
switch (maat_instance->opts.input_mode) { if (0 == pthread_mutex_trylock(&(maat_inst->background_update_mutex))) {
switch (maat_inst->opts.input_mode) {
case DATA_SOURCE_REDIS: case DATA_SOURCE_REDIS:
redis_monitor_traverse(maat_instance->maat_version, redis_monitor_traverse(maat_inst->maat_version,
&(maat_instance->opts.redis_ctx), &(maat_inst->opts.redis_ctx),
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance); maat_inst);
break; break;
case DATA_SOURCE_IRIS_FILE: case DATA_SOURCE_IRIS_FILE:
config_monitor_traverse(maat_instance->maat_version, config_monitor_traverse(maat_inst->maat_version,
maat_instance->opts.iris_ctx.inc_idx_dir, maat_inst->opts.iris_ctx.inc_idx_dir,
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance, maat_instance->opts.decrypt_key, maat_inst, maat_inst->opts.decrypt_key,
maat_instance->logger); maat_inst->logger);
break; break;
case DATA_SOURCE_JSON_FILE: case DATA_SOURCE_JSON_FILE:
memset(md5_tmp, 0, sizeof(md5_tmp)); memset(md5_tmp, 0, sizeof(md5_tmp));
stat(maat_instance->opts.json_ctx.json_file, &attrib); stat(maat_inst->opts.json_ctx.json_file, &attrib);
if (memcmp(&attrib.st_ctim, &(maat_instance->opts.json_ctx.last_md5_time), sizeof(attrib.st_ctim))) { if (memcmp(&attrib.st_ctim, &(maat_inst->opts.json_ctx.last_md5_time),
maat_instance->opts.json_ctx.last_md5_time = attrib.st_ctim; sizeof(attrib.st_ctim))) {
md5_file(maat_instance->opts.json_ctx.json_file, md5_tmp); maat_inst->opts.json_ctx.last_md5_time = attrib.st_ctim;
if (0 != strcmp(md5_tmp, maat_instance->opts.json_ctx.effective_json_md5)) { md5_file(maat_inst->opts.json_ctx.json_file, md5_tmp);
ret = load_maat_json_file(maat_instance, maat_instance->opts.json_ctx.json_file, if (0 != strcmp(md5_tmp, maat_inst->opts.json_ctx.effective_json_md5)) {
ret = load_maat_json_file(maat_inst, maat_inst->opts.json_ctx.json_file,
err_str, sizeof(err_str)); err_str, sizeof(err_str));
if (ret < 0) { if (ret < 0) {
log_error(maat_instance->logger, MODULE_MAAT_RULE, log_error(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] Maat re-initiate with JSON file %s (md5=%s)failed: %s\n", "[%s:%d] Maat re-initiate with JSON file %s (md5=%s)failed: %s\n",
__FUNCTION__, __LINE__, maat_instance->opts.json_ctx.json_file, __FUNCTION__, __LINE__, maat_inst->opts.json_ctx.json_file,
md5_tmp, err_str); md5_tmp, err_str);
} else { } else {
config_monitor_traverse(0, maat_instance->opts.json_ctx.iris_file, config_monitor_traverse(0, maat_inst->opts.json_ctx.iris_file,
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance, maat_instance->opts.decrypt_key, maat_inst, maat_inst->opts.decrypt_key,
maat_instance->logger); maat_inst->logger);
log_info(maat_instance->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Maat re-initiate with JSON file %s success, md5: %s\n", "Maat re-initiate with JSON file %s success, md5: %s\n",
maat_instance->opts.json_ctx.json_file, md5_tmp); maat_inst->opts.json_ctx.json_file, md5_tmp);
} }
} }
} }
@@ -402,77 +407,81 @@ void *rule_monitor_loop(void *arg)
break; break;
} }
if (maat_instance->creating_maat_rt != NULL) { if (maat_inst->creating_maat_rt != NULL) {
struct maat_runtime *old_maat_rt = maat_instance->maat_rt; struct maat_runtime *old_maat_rt = maat_inst->maat_rt;
maat_instance->maat_rt = maat_instance->creating_maat_rt; maat_inst->maat_rt = maat_inst->creating_maat_rt;
if (old_maat_rt != NULL) { if (old_maat_rt != NULL) {
if (maat_instance->maat_rt->version > old_maat_rt->version) { if (maat_inst->maat_rt->version > old_maat_rt->version) {
log_info(maat_instance->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Maat version updated %lld -> %lld\n", "Maat version updated %lld -> %lld\n",
old_maat_rt->version, maat_instance->maat_rt->version); old_maat_rt->version, maat_inst->maat_rt->version);
} else { } else {
log_info(maat_instance->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Maat version roll back %lld -> %lld\n", "Maat version roll back %lld -> %lld\n",
old_maat_rt->version, maat_instance->maat_rt->version); old_maat_rt->version, maat_inst->maat_rt->version);
} }
maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt, NULL,
maat_inst->stat->zombie_rs_stream += alignment_int64_array_sum(old_maat_rt->ref_cnt,
maat_inst->opts.nr_worker_thread);
maat_garbage_bagging(maat_inst->garbage_bin, old_maat_rt, NULL,
garbage_maat_runtime_destroy); garbage_maat_runtime_destroy);
} }
maat_instance->creating_maat_rt = NULL; maat_inst->creating_maat_rt = NULL;
maat_instance->maat_version = maat_instance->maat_rt->version; maat_inst->maat_version = maat_inst->maat_rt->version;
maat_instance->last_full_version = maat_instance->maat_rt->version; maat_inst->last_full_version = maat_inst->maat_rt->version;
} }
if (maat_instance->maat_rt != NULL) { if (maat_inst->maat_rt != NULL) {
time_t time_window = time(NULL) - maat_instance->maat_rt->last_update_time; time_t time_window = time(NULL) - maat_inst->maat_rt->last_update_time;
if (time_window >= maat_instance->opts.rule_effect_interval_ms / 1000) { if (time_window >= maat_inst->opts.rule_effect_interval_ms / 1000) {
maat_runtime_commit(maat_instance->maat_rt, MAAT_UPDATE_TYPE_INC, maat_runtime_commit(maat_inst->maat_rt, MAAT_UPDATE_TYPE_INC,
maat_instance->maat_rt->version, maat_instance->logger); maat_inst->maat_rt->version, maat_inst->logger);
log_info(maat_instance->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Actual update config version %u, %d entries load to maat runtime.", "Actual update config version %u, %d entries load to maat runtime.",
maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num); maat_inst->maat_rt->version, maat_inst->maat_rt->rule_num);
} }
} }
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_inst->background_update_mutex));
} }
maat_garbage_collect_routine(maat_instance->garbage_bin);
if ((1 == maat_instance->opts.stat_on) && (time(NULL) % 2 == 0)) { maat_garbage_collect_routine(maat_inst->garbage_bin);
maat_stat_output(maat_instance->stat, maat_instance->maat_version, maat_instance->opts.perf_on); if ((1 == maat_inst->opts.stat_on) && (time(NULL) % 2 == 0)) {
maat_stat_output(maat_inst->stat, maat_inst->maat_version, maat_inst->opts.perf_on);
} }
} }
maat_runtime_destroy(maat_instance->maat_rt); maat_runtime_destroy(maat_inst->maat_rt);
maat_garbage_bin_free(maat_instance->garbage_bin); maat_garbage_bin_free(maat_inst->garbage_bin);
table_manager_destroy(maat_instance->tbl_mgr); //table manager MUST be freed at last. table_manager_destroy(maat_inst->tbl_mgr); //table manager MUST be freed at last.
if (maat_instance->stat != NULL) { if (maat_inst->stat != NULL) {
maat_stat_free(maat_instance->stat); maat_stat_free(maat_inst->stat);
maat_instance->stat = NULL; maat_inst->stat = NULL;
} }
if (maat_instance->opts.input_mode == DATA_SOURCE_REDIS) { if (maat_inst->opts.input_mode == DATA_SOURCE_REDIS) {
if (maat_instance->opts.redis_ctx.read_ctx != NULL) { if (maat_inst->opts.redis_ctx.read_ctx != NULL) {
redisFree(maat_instance->opts.redis_ctx.read_ctx); redisFree(maat_inst->opts.redis_ctx.read_ctx);
maat_instance->opts.redis_ctx.read_ctx = NULL; maat_inst->opts.redis_ctx.read_ctx = NULL;
} }
if (maat_instance->opts.redis_ctx.write_ctx != NULL) { if (maat_inst->opts.redis_ctx.write_ctx != NULL) {
redisFree(maat_instance->opts.redis_ctx.write_ctx); redisFree(maat_inst->opts.redis_ctx.write_ctx);
maat_instance->opts.redis_ctx.write_ctx = NULL; maat_inst->opts.redis_ctx.write_ctx = NULL;
} }
} }
if (maat_instance->opts.accept_tags != NULL) { if (maat_inst->opts.accept_tags != NULL) {
FREE(maat_instance->opts.accept_tags); FREE(maat_inst->opts.accept_tags);
maat_instance->opts.accept_tags = NULL; maat_inst->opts.accept_tags = NULL;
} }
log_handle_destroy(maat_instance->logger); log_handle_destroy(maat_inst->logger);
FREE(maat_instance); FREE(maat_inst);
return NULL; return NULL;
} }

2
src/rcu_hash.c
View File

@@ -44,7 +44,7 @@ struct rcu_hash_table {
struct rcu_hash_node { struct rcu_hash_node {
char *key; char *key;
size_t key_len; size_t key_len;
void *data; //table_runtime解析成两个成员 void *data;
/* htable the node belongs to */ /* htable the node belongs to */
struct rcu_hash_table *htable; struct rcu_hash_table *htable;

30
test/adapter_hs_gtest.cpp
View File

@@ -235,23 +235,23 @@ TEST(adapter_hs_init, invalid_input_parameter)
struct expr_rule rules[64]; struct expr_rule rules[64];
size_t n_rule = 0; size_t n_rule = 0;
struct adapter_hs *hs_instance = adapter_hs_new(1, NULL, 0, g_logger); struct adapter_hs *hs_instance = adapter_hs_new(NULL, 0, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL); EXPECT_TRUE(hs_instance == NULL);
hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL); EXPECT_TRUE(hs_instance == NULL);
n_rule = 1; n_rule = 1;
rules[0].expr_id = 101; rules[0].expr_id = 101;
rules[0].n_patterns = 10; rules[0].n_patterns = 10;
hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL); EXPECT_TRUE(hs_instance == NULL);
memset(rules, 0, sizeof(rules)); memset(rules, 0, sizeof(rules));
n_rule = 1; n_rule = 1;
rules[0].expr_id = 101; rules[0].expr_id = 101;
rules[0].n_patterns = 1; rules[0].n_patterns = 1;
hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL); EXPECT_TRUE(hs_instance == NULL);
} }
@@ -263,7 +263,7 @@ TEST(adapter_hs_scan, literal_sub_has_normal_offset)
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule); expr_array_free(rules, n_rule);
@@ -310,7 +310,7 @@ TEST(adapter_hs_scan, literal_sub_has_left_unlimit_offset)
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule); expr_array_free(rules, n_rule);
@@ -358,7 +358,7 @@ TEST(adapter_hs_scan, literal_sub_has_right_unlimit_offset)
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule); expr_array_free(rules, n_rule);
@@ -413,7 +413,7 @@ TEST(adapter_hs_scan, literal_sub_with_no_offset)
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule); expr_array_free(rules, n_rule);
@@ -460,7 +460,7 @@ TEST(adapter_hs_scan, literal_exactly)
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule); expr_array_free(rules, n_rule);
@@ -501,7 +501,7 @@ TEST(adapter_hs_scan, literal_prefix)
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule); expr_array_free(rules, n_rule);
@@ -551,7 +551,7 @@ TEST(adapter_hs_scan, literal_suffix)
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule); expr_array_free(rules, n_rule);
@@ -601,7 +601,7 @@ TEST(adapter_hs_scan, literal_sub_with_hexbin)
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule); expr_array_free(rules, n_rule);
@@ -633,7 +633,7 @@ TEST(adapter_hs_scan, literal_with_chinese)
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule); expr_array_free(rules, n_rule);
@@ -657,7 +657,7 @@ TEST(adapter_hs_scan, same_pattern_different_offset)
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule); expr_array_free(rules, n_rule);
@@ -681,7 +681,7 @@ TEST(adapter_hs_scan, long_scan_data)
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(1, rules, n_rule, g_logger); struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule); expr_array_free(rules, n_rule);

12
test/maat_ex_data_gtest.cpp
View File

@@ -12,7 +12,7 @@
const char *table_info_path = "./table_info.conf"; const char *table_info_path = "./table_info.conf";
const char *json_filename = "maat_json.json"; const char *json_filename = "maat_json.json";
struct log_handle *g_logger = NULL; struct log_handle *g_logger = NULL;
struct maat *g_maat_instance = NULL; struct maat *g_maat_inst = NULL;
struct user_info { struct user_info {
char name[256]; char name[256];
@@ -55,7 +55,7 @@ void ex_data_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
TEST(EXDataRuntime, Update) { TEST(EXDataRuntime, Update) {
const char *table_name = "TEST_PLUGIN_EXDATA_TABLE"; const char *table_name = "TEST_PLUGIN_EXDATA_TABLE";
int table_id = maat_get_table_id(g_maat_instance, table_name); int table_id = maat_get_table_id(g_maat_inst, table_name);
ASSERT_GT(table_id, 0); ASSERT_GT(table_id, 0);
int ex_data_counter = 0; int ex_data_counter = 0;
@@ -145,14 +145,14 @@ int main(int argc, char ** argv)
snprintf(json_path, sizeof(json_path), "./%s", json_filename); snprintf(json_path, sizeof(json_path), "./%s", json_filename);
maat_options_set_json_file(opts, json_path); maat_options_set_json_file(opts, json_path);
g_maat_instance = maat_new(opts, table_info_path); g_maat_inst = maat_new(opts, table_info_path);
maat_options_free(opts); maat_options_free(opts);
EXPECT_TRUE(g_maat_instance != NULL); EXPECT_TRUE(g_maat_inst != NULL);
ret=RUN_ALL_TESTS(); ret=RUN_ALL_TESTS();
log_handle_destroy(g_logger); log_handle_destroy(g_logger);
maat_free(g_maat_instance); maat_free(g_maat_inst);
g_maat_instance = NULL; g_maat_inst = NULL;
return ret; return ret;
} }

1964
test/maat_framework_gtest.cpp
View File

File diff suppressed because it is too large Load Diff

232
test/maat_framework_perf_gtest.cpp
View File

@@ -22,7 +22,7 @@ const char *json_filename = "maat_json.json";
struct thread_param { struct thread_param {
int thread_id; int thread_id;
int test_count; int test_count;
struct maat *maat_instance; struct maat *maat_inst;
const char *table_name; const char *table_name;
long long time_elapse_ms; long long time_elapse_ms;
struct log_handle *logger; struct log_handle *logger;
@@ -162,7 +162,7 @@ static int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
return 0; return 0;
} }
static int compile_table_set_line(struct maat *maat_instance, const char *table_name, static int compile_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long compile_id, enum maat_operation op, long long compile_id,
const char *user_region, int clause_num, int expire_after) const char *user_region, int clause_num, int expire_after)
{ {
@@ -176,10 +176,10 @@ static int compile_table_set_line(struct maat *maat_instance, const char *table_
line_rule.table_name = table_name; line_rule.table_name = table_name;
line_rule.expire_after = expire_after; line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_instance, &line_rule); return maat_cmd_set_line(maat_inst, &line_rule);
} }
static int group2compile_table_set_line(struct maat *maat_instance, const char *table_name, static int group2compile_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long group_id, long long compile_id, enum maat_operation op, long long group_id, long long compile_id,
int not_flag, const char *vtable_name, int clause_num, int not_flag, const char *vtable_name, int clause_num,
int expire_after) int expire_after)
@@ -194,20 +194,20 @@ static int group2compile_table_set_line(struct maat *maat_instance, const char *
line_rule.table_name = table_name; line_rule.table_name = table_name;
line_rule.expire_after = expire_after; line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_instance, &line_rule); return maat_cmd_set_line(maat_inst, &line_rule);
} }
static int expr_table_set_line(struct maat *maat_instance, const char *table_name, enum maat_operation op, static int expr_table_set_line(struct maat *maat_inst, const char *table_name, enum maat_operation op,
long long item_id, long long group_id, const char *keywords, const char *district, long long item_id, long long group_id, const char *keywords, const char *district,
int expr_type, int match_method, int is_hexbin, int expire_after) int expr_type, int match_method, int is_hexbin, int expire_after)
{ {
char table_line[1024] = {0}; char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
if (table_id < 0) { if (table_id < 0) {
return 0; return 0;
} }
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
assert(table_type == TABLE_TYPE_EXPR || table_type == TABLE_TYPE_EXPR_PLUS); assert(table_type == TABLE_TYPE_EXPR || table_type == TABLE_TYPE_EXPR_PLUS);
if (table_type == TABLE_TYPE_EXPR_PLUS) { if (table_type == TABLE_TYPE_EXPR_PLUS) {
@@ -224,15 +224,15 @@ static int expr_table_set_line(struct maat *maat_instance, const char *table_nam
line_rule.table_name = table_name; line_rule.table_name = table_name;
line_rule.expire_after = expire_after; line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_instance, &line_rule); return maat_cmd_set_line(maat_inst, &line_rule);
} }
static int ip_table_set_line(struct maat *maat_instance, const char *table_name, enum maat_operation op, static int ip_table_set_line(struct maat *maat_inst, const char *table_name, enum maat_operation op,
long long item_id, long long group_id, enum IP_TYPE type, const char *ip1, long long item_id, long long group_id, enum IP_TYPE type, const char *ip1,
const char *ip2, uint16_t port_min, uint16_t port_max, int expire_after) const char *ip2, uint16_t port_min, uint16_t port_max, int expire_after)
{ {
char table_line[1024] = {0}; char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
if (table_id < 0) { if (table_id < 0) {
return 0; return 0;
} }
@@ -252,41 +252,41 @@ static int ip_table_set_line(struct maat *maat_instance, const char *table_name,
line_rule.table_name = table_name; line_rule.table_name = table_name;
line_rule.expire_after = expire_after; line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_instance, &line_rule); return maat_cmd_set_line(maat_inst, &line_rule);
} }
void test_add_expr_command(struct maat *maat_instance, const char *table_name, void test_add_expr_command(struct maat *maat_inst, const char *table_name,
const char *keywords) const char *keywords)
{ {
long long compile_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1); long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1);
int ret = compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_ADD, compile_id, "null", 1, 0); int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, compile_id, "null", 1, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1);
ret = group2compile_table_set_line(maat_instance, "GROUP2COMPILE", MAAT_OP_ADD, group_id, ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, group_id,
compile_id, 0, "null", 1, 0); compile_id, 0, "null", 1, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); long long item_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1);
ret = expr_table_set_line(maat_instance, table_name, MAAT_OP_ADD, item_id, group_id, ret = expr_table_set_line(maat_inst, table_name, MAAT_OP_ADD, item_id, group_id,
keywords, NULL, 1, 0, 0, 0); keywords, NULL, 1, 0, 0, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
} }
void test_add_ip_command(struct maat *maat_instance, const char *table_name, void test_add_ip_command(struct maat *maat_inst, const char *table_name,
const char *ip, uint16_t port) const char *ip, uint16_t port)
{ {
long long compile_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1); long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1);
int ret = compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_ADD, compile_id, "null", 1, 0); int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, compile_id, "null", 1, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1);
ret = group2compile_table_set_line(maat_instance, "GROUP2COMPILE", MAAT_OP_ADD, group_id, ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, group_id,
compile_id, 0, "null", 1, 0); compile_id, 0, "null", 1, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); long long item_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1);
ret = ip_table_set_line(maat_instance, table_name, MAAT_OP_ADD, item_id, group_id, ret = ip_table_set_line(maat_inst, table_name, MAAT_OP_ADD, item_id, group_id,
IPv4, ip, ip, port, port, 0); IPv4, ip, ip, port, port, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
} }
@@ -316,9 +316,9 @@ protected:
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info_path); _shared_maat_inst = maat_new(opts, table_info_path);
maat_options_free(opts); maat_options_free(opts);
if (NULL == _shared_maat_instance) { if (NULL == _shared_maat_inst) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatFlagScan failed.", "[%s:%d] create maat instance in MaatFlagScan failed.",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
@@ -326,34 +326,34 @@ protected:
} }
static void TearDownTestCase() { static void TearDownTestCase() {
maat_free(_shared_maat_instance); maat_free(_shared_maat_inst);
log_handle_destroy(logger); log_handle_destroy(logger);
} }
static struct log_handle *logger; static struct log_handle *logger;
static struct maat *_shared_maat_instance; static struct maat *_shared_maat_inst;
}; };
struct maat *MaatPerfStringScan::_shared_maat_instance; struct maat *MaatPerfStringScan::_shared_maat_inst;
struct log_handle *MaatPerfStringScan::logger; struct log_handle *MaatPerfStringScan::logger;
void *perf_string_scan_thread(void *arg) void *perf_string_scan_thread(void *arg)
{ {
struct thread_param *param = (struct thread_param *)arg; struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance; struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name; const char *table_name = param->table_name;
struct timespec start, end; struct timespec start, end;
const char *scan_data = "String TEST should hit"; const char *scan_data = "String TEST should hit";
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
int hit_times = 0; int hit_times = 0;
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = maat_state_new(maat_instance, param->thread_id); struct maat_state *state = maat_state_new(maat_inst, param->thread_id);
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_count; i++) { for (int i = 0; i < param->test_count; i++) {
int ret = maat_scan_string(maat_instance, table_id, scan_data, strlen(scan_data), int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, state); results, ARRAY_SIZE, &n_hit_result, state);
if (ret == MAAT_SCAN_HIT) { if (ret == MAAT_SCAN_HIT) {
hit_times++; hit_times++;
@@ -374,14 +374,14 @@ void *perf_string_scan_thread(void *arg)
void *perf_string_update_thread(void *arg) void *perf_string_update_thread(void *arg)
{ {
struct thread_param *param = (struct thread_param *)arg; struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance; struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name; const char *table_name = param->table_name;
const int CMD_EXPR_NUM = 10; const int CMD_EXPR_NUM = 10;
char keyword_buf[128]; char keyword_buf[128];
for (int i = 0; i < CMD_EXPR_NUM; i++) { for (int i = 0; i < CMD_EXPR_NUM; i++) {
random_keyword_generate(keyword_buf, sizeof(keyword_buf)); random_keyword_generate(keyword_buf, sizeof(keyword_buf));
test_add_expr_command(maat_instance, table_name, keyword_buf); test_add_expr_command(maat_inst, table_name, keyword_buf);
sleep(1); sleep(1);
} }
@@ -394,7 +394,7 @@ void *perf_string_update_thread(void *arg)
void *perf_ip_scan_thread(void *arg) void *perf_ip_scan_thread(void *arg)
{ {
struct thread_param *param = (struct thread_param *)arg; struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance; struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name; const char *table_name = param->table_name;
struct timespec start, end; struct timespec start, end;
char ip_str[32] = "10.0.7.100"; char ip_str[32] = "10.0.7.100";
@@ -407,12 +407,12 @@ void *perf_ip_scan_thread(void *arg)
int hit_times = 0; int hit_times = 0;
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = maat_state_new(maat_instance, param->thread_id); struct maat_state *state = maat_state_new(maat_inst, param->thread_id);
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_count; i++) { for (int i = 0; i < param->test_count; i++) {
int ret = maat_scan_ipv4(maat_instance, table_id, ip_addr, port, 6, int ret = maat_scan_ipv4(maat_inst, table_id, ip_addr, port, 6,
results, ARRAY_SIZE, &n_hit_result, state); results, ARRAY_SIZE, &n_hit_result, state);
if (ret == MAAT_SCAN_HIT) { if (ret == MAAT_SCAN_HIT) {
hit_times++; hit_times++;
@@ -433,7 +433,7 @@ void *perf_ip_scan_thread(void *arg)
void *perf_ip_update_thread(void *arg) void *perf_ip_update_thread(void *arg)
{ {
struct thread_param *param = (struct thread_param *)arg; struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance; struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name; const char *table_name = param->table_name;
const int CMD_EXPR_NUM = 10; const int CMD_EXPR_NUM = 10;
char ip_str[10][32] = { char ip_str[10][32] = {
@@ -450,7 +450,7 @@ void *perf_ip_update_thread(void *arg)
uint16_t port = 65530; uint16_t port = 65530;
for (int i = 0; i < CMD_EXPR_NUM; i++) { for (int i = 0; i < CMD_EXPR_NUM; i++) {
test_add_ip_command(maat_instance, table_name, ip_str[i], port); test_add_ip_command(maat_inst, table_name, ip_str[i], port);
sleep(1); sleep(1);
} }
@@ -462,9 +462,9 @@ void *perf_ip_update_thread(void *arg)
TEST_F(MaatPerfStringScan, MultiThread) { TEST_F(MaatPerfStringScan, MultiThread) {
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
struct maat *maat_instance = MaatPerfStringScan::_shared_maat_instance; struct maat *maat_inst = MaatPerfStringScan::_shared_maat_inst;
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0); ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM + 1]; pthread_t threads[PERF_THREAD_NUM + 1];
@@ -473,7 +473,7 @@ TEST_F(MaatPerfStringScan, MultiThread) {
int *is_all_hit = NULL; int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) { for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_instance = maat_instance; thread_params[i].maat_inst = maat_inst;
thread_params[i].thread_id = i; thread_params[i].thread_id = i;
thread_params[i].table_name = table_name; thread_params[i].table_name = table_name;
thread_params[i].test_count = PERF_SCAN_COUNT; thread_params[i].test_count = PERF_SCAN_COUNT;
@@ -502,7 +502,7 @@ TEST_F(MaatPerfStringScan, MultiThread) {
} }
scan_per_second = scan_count * 1000 / time_elapse_ms; scan_per_second = scan_count * 1000 / time_elapse_ms;
//EXPECT_GT(scan_per_second, 800 * 1000); //EXPECT_GT(scan_per_second, 800 * 1000);
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST, log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST,
"StringScan match rate on %d-threads speed %lld lookups/s/thread", "StringScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second); PERF_THREAD_NUM, scan_per_second);
} }
@@ -532,9 +532,9 @@ protected:
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info_path); _shared_maat_inst = maat_new(opts, table_info_path);
maat_options_free(opts); maat_options_free(opts);
if (NULL == _shared_maat_instance) { if (NULL == _shared_maat_inst) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatFlagScan failed.", "[%s:%d] create maat instance in MaatFlagScan failed.",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
@@ -542,21 +542,21 @@ protected:
} }
static void TearDownTestCase() { static void TearDownTestCase() {
maat_free(_shared_maat_instance); maat_free(_shared_maat_inst);
log_handle_destroy(logger); log_handle_destroy(logger);
} }
static struct log_handle *logger; static struct log_handle *logger;
static struct maat *_shared_maat_instance; static struct maat *_shared_maat_inst;
}; };
struct maat *MaatPerfStreamScan::_shared_maat_instance; struct maat *MaatPerfStreamScan::_shared_maat_inst;
struct log_handle *MaatPerfStreamScan::logger; struct log_handle *MaatPerfStreamScan::logger;
void *perf_stream_scan_thread(void *arg) void *perf_stream_scan_thread(void *arg)
{ {
struct thread_param *param = (struct thread_param *)arg; struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance; struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name; const char *table_name = param->table_name;
struct timespec start, end; struct timespec start, end;
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
@@ -565,13 +565,13 @@ void *perf_stream_scan_thread(void *arg)
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state_array[ARRAY_SIZE]; struct maat_state *state_array[ARRAY_SIZE];
struct maat_stream *sp[ARRAY_SIZE]; struct maat_stream *sp[ARRAY_SIZE];
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_count; i++) { for (int i = 0; i < param->test_count; i++) {
for (int j = 0; j < ARRAY_SIZE; j++) { for (int j = 0; j < ARRAY_SIZE; j++) {
state_array[j] = maat_state_new(maat_instance, param->thread_id); state_array[j] = maat_state_new(maat_inst, param->thread_id);
sp[j] = maat_stream_new(maat_instance, table_id, state_array[j]); sp[j] = maat_stream_new(maat_inst, table_id, state_array[j]);
ret = maat_stream_scan(sp[j], scan_data, strlen(scan_data), results, ARRAY_SIZE, ret = maat_stream_scan(sp[j], scan_data, strlen(scan_data), results, ARRAY_SIZE,
&n_hit_result, state_array[j]); &n_hit_result, state_array[j]);
@@ -597,14 +597,14 @@ void *perf_stream_scan_thread(void *arg)
void *perf_stream_update_thread(void *arg) void *perf_stream_update_thread(void *arg)
{ {
struct thread_param *param = (struct thread_param *)arg; struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance; struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name; const char *table_name = param->table_name;
const int CMD_EXPR_NUM = 10; const int CMD_EXPR_NUM = 10;
char keyword_buf[128]; char keyword_buf[128];
for (int i = 0; i < CMD_EXPR_NUM; i++) { for (int i = 0; i < CMD_EXPR_NUM; i++) {
random_keyword_generate(keyword_buf, sizeof(keyword_buf)); random_keyword_generate(keyword_buf, sizeof(keyword_buf));
test_add_expr_command(maat_instance, table_name, keyword_buf); test_add_expr_command(maat_inst, table_name, keyword_buf);
sleep(1); sleep(1);
} }
@@ -616,9 +616,9 @@ void *perf_stream_update_thread(void *arg)
TEST_F(MaatPerfStreamScan, MultiThread) { TEST_F(MaatPerfStreamScan, MultiThread) {
const char *table_name = "HTTP_URL"; const char *table_name = "HTTP_URL";
struct maat *maat_instance = MaatPerfStreamScan::_shared_maat_instance; struct maat *maat_inst = MaatPerfStreamScan::_shared_maat_inst;
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0); ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM + 1]; pthread_t threads[PERF_THREAD_NUM + 1];
@@ -627,7 +627,7 @@ TEST_F(MaatPerfStreamScan, MultiThread) {
int *is_all_hit = NULL; int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) { for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_instance = maat_instance; thread_params[i].maat_inst = maat_inst;
thread_params[i].thread_id = i; thread_params[i].thread_id = i;
thread_params[i].table_name = table_name; thread_params[i].table_name = table_name;
thread_params[i].test_count = PERF_SCAN_COUNT / 10; thread_params[i].test_count = PERF_SCAN_COUNT / 10;
@@ -656,7 +656,7 @@ TEST_F(MaatPerfStreamScan, MultiThread) {
} }
scan_per_second = scan_count * 1000 / time_elapse_ms; scan_per_second = scan_count * 1000 / time_elapse_ms;
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST, log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST,
"StreamScan match rate on %d-threads speed %lld lookups/s/thread", "StreamScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second); PERF_THREAD_NUM, scan_per_second);
} }
@@ -686,9 +686,9 @@ protected:
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info_path); _shared_maat_inst = maat_new(opts, table_info_path);
maat_options_free(opts); maat_options_free(opts);
if (NULL == _shared_maat_instance) { if (NULL == _shared_maat_inst) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatFlagScan failed.", "[%s:%d] create maat instance in MaatFlagScan failed.",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
@@ -696,23 +696,23 @@ protected:
} }
static void TearDownTestCase() { static void TearDownTestCase() {
maat_free(_shared_maat_instance); maat_free(_shared_maat_inst);
log_handle_destroy(logger); log_handle_destroy(logger);
} }
static struct log_handle *logger; static struct log_handle *logger;
static struct maat *_shared_maat_instance; static struct maat *_shared_maat_inst;
}; };
struct maat *MaatPerfIPScan::_shared_maat_instance; struct maat *MaatPerfIPScan::_shared_maat_inst;
struct log_handle *MaatPerfIPScan::logger; struct log_handle *MaatPerfIPScan::logger;
TEST_F(MaatPerfIPScan, MultiThread) TEST_F(MaatPerfIPScan, MultiThread)
{ {
const char *table_name = "IP_PLUS_CONFIG"; const char *table_name = "IP_PLUS_CONFIG";
struct maat *maat_instance = MaatPerfIPScan::_shared_maat_instance; struct maat *maat_inst = MaatPerfIPScan::_shared_maat_inst;
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0); ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM + 1]; pthread_t threads[PERF_THREAD_NUM + 1];
@@ -721,7 +721,7 @@ TEST_F(MaatPerfIPScan, MultiThread)
int *is_all_hit = NULL; int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) { for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_instance = maat_instance; thread_params[i].maat_inst = maat_inst;
thread_params[i].thread_id = i; thread_params[i].thread_id = i;
thread_params[i].table_name = table_name; thread_params[i].table_name = table_name;
thread_params[i].test_count = PERF_SCAN_COUNT; thread_params[i].test_count = PERF_SCAN_COUNT;
@@ -750,7 +750,7 @@ TEST_F(MaatPerfIPScan, MultiThread)
} }
scan_per_second = scan_count * 1000 / time_elapse_ms; scan_per_second = scan_count * 1000 / time_elapse_ms;
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST, log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST,
"IPScan match rate on %d-threads speed %lld lookups/s/thread", "IPScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second); PERF_THREAD_NUM, scan_per_second);
} }
@@ -780,9 +780,9 @@ protected:
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info_path); _shared_maat_inst = maat_new(opts, table_info_path);
maat_options_free(opts); maat_options_free(opts);
if (NULL == _shared_maat_instance) { if (NULL == _shared_maat_inst) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatFlagScan failed.", "[%s:%d] create maat instance in MaatFlagScan failed.",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
@@ -790,15 +790,15 @@ protected:
} }
static void TearDownTestCase() { static void TearDownTestCase() {
maat_free(_shared_maat_instance); maat_free(_shared_maat_inst);
log_handle_destroy(logger); log_handle_destroy(logger);
} }
static struct log_handle *logger; static struct log_handle *logger;
static struct maat *_shared_maat_instance; static struct maat *_shared_maat_inst;
}; };
struct maat *MaatPerfFQDNPluginScan::_shared_maat_instance; struct maat *MaatPerfFQDNPluginScan::_shared_maat_inst;
struct log_handle *MaatPerfFQDNPluginScan::logger; struct log_handle *MaatPerfFQDNPluginScan::logger;
struct perf_fqdn_plugin_ud { struct perf_fqdn_plugin_ud {
@@ -846,11 +846,11 @@ void perf_fqdn_plugin_EX_dup_cb(int table_id, void **to, void **from, long argl,
void* perf_fqdn_plugin_scan_thread(void *arg) void* perf_fqdn_plugin_scan_thread(void *arg)
{ {
struct thread_param *param = (struct thread_param *)arg; struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance; struct maat *maat_inst = param->maat_inst;
struct perf_fqdn_plugin_ud *result[ARRAY_SIZE]; struct perf_fqdn_plugin_ud *result[ARRAY_SIZE];
int i=0, j=0, ret=0, hit_times=0; int i=0, j=0, ret=0, hit_times=0;
int table_id = maat_get_table_id(maat_instance, param->table_name); int table_id = maat_get_table_id(maat_inst, param->table_name);
memset(&result, 0, sizeof(result)); memset(&result, 0, sizeof(result));
struct timespec start, end; struct timespec start, end;
@@ -858,7 +858,7 @@ void* perf_fqdn_plugin_scan_thread(void *arg)
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < param->test_count; i++) { for (i = 0; i < param->test_count; i++) {
ret = maat_fqdn_plugin_table_get_ex_data(maat_instance, table_id, ret = maat_fqdn_plugin_table_get_ex_data(maat_inst, table_id,
"r3---sn-i3belne6.example2.com", "r3---sn-i3belne6.example2.com",
(void**)result, ARRAY_SIZE); (void**)result, ARRAY_SIZE);
if (ret == 2) { if (ret == 2) {
@@ -907,20 +907,20 @@ void random_fqdn_generate(char *fqdn_buff, int sz)
void *perf_fqdn_plugin_update_thread(void *arg) void *perf_fqdn_plugin_update_thread(void *arg)
{ {
struct thread_param *param = (struct thread_param *)arg; struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance; struct maat *maat_inst = param->maat_inst;
const int CMD_EXPR_NUM = 20; const int CMD_EXPR_NUM = 20;
int i = 0; int i = 0;
struct maat_cmd_line line_rule; struct maat_cmd_line line_rule;
char line_buff[1024], fqdn_buff[256]; char line_buff[1024], fqdn_buff[256];
for (i = 0; i < CMD_EXPR_NUM; i++) { for (i = 0; i < CMD_EXPR_NUM; i++) {
line_rule.rule_id = (int)maat_cmd_incrby(maat_instance, "TEST_PLUG_SEQ", 1); line_rule.rule_id = (int)maat_cmd_incrby(maat_inst, "TEST_PLUG_SEQ", 1);
line_rule.table_name = param->table_name; line_rule.table_name = param->table_name;
random_fqdn_generate(fqdn_buff, sizeof(fqdn_buff)); random_fqdn_generate(fqdn_buff, sizeof(fqdn_buff));
snprintf(line_buff, 1024, "%lld\t1\t%s\tcatid=4\t1", line_rule.rule_id, fqdn_buff); snprintf(line_buff, 1024, "%lld\t1\t%s\tcatid=4\t1", line_rule.rule_id, fqdn_buff);
line_rule.table_line = line_buff; line_rule.table_line = line_buff;
line_rule.expire_after = 0; line_rule.expire_after = 0;
maat_cmd_set_line(maat_instance, &line_rule); maat_cmd_set_line(maat_inst, &line_rule);
sleep(WAIT_FOR_EFFECTIVE_S); sleep(WAIT_FOR_EFFECTIVE_S);
} }
@@ -931,11 +931,11 @@ void *perf_fqdn_plugin_update_thread(void *arg)
} }
TEST_F(MaatPerfFQDNPluginScan, MultiThread) { TEST_F(MaatPerfFQDNPluginScan, MultiThread) {
struct maat *maat_instance = MaatPerfFQDNPluginScan::_shared_maat_instance; struct maat *maat_inst = MaatPerfFQDNPluginScan::_shared_maat_inst;
const char *table_name = "TEST_FQDN_PLUGIN_WITH_EXDATA"; const char *table_name = "TEST_FQDN_PLUGIN_WITH_EXDATA";
int fqdn_plugin_ex_data_counter = 0; int fqdn_plugin_ex_data_counter = 0;
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0); ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM + 1]; pthread_t threads[PERF_THREAD_NUM + 1];
@@ -943,7 +943,7 @@ TEST_F(MaatPerfFQDNPluginScan, MultiThread) {
int i = 0; int i = 0;
int *is_all_hit = NULL; int *is_all_hit = NULL;
int ret = maat_plugin_table_ex_schema_register(maat_instance, table_name, int ret = maat_plugin_table_ex_schema_register(maat_inst, table_name,
perf_fqdn_plugin_EX_new_cb, perf_fqdn_plugin_EX_new_cb,
perf_fqdn_plugin_EX_free_cb, perf_fqdn_plugin_EX_free_cb,
perf_fqdn_plugin_EX_dup_cb, perf_fqdn_plugin_EX_dup_cb,
@@ -951,7 +951,7 @@ TEST_F(MaatPerfFQDNPluginScan, MultiThread) {
ASSERT_TRUE(ret>=0); ASSERT_TRUE(ret>=0);
for (i = 0; i < PERF_THREAD_NUM + 1; i++) { for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_instance = maat_instance; thread_params[i].maat_inst = maat_inst;
thread_params[i].thread_id = i; thread_params[i].thread_id = i;
thread_params[i].table_name = table_name; thread_params[i].table_name = table_name;
thread_params[i].test_count = PERF_SCAN_COUNT; thread_params[i].test_count = PERF_SCAN_COUNT;
@@ -980,7 +980,7 @@ TEST_F(MaatPerfFQDNPluginScan, MultiThread) {
} }
scan_per_second = scan_count * 1000 / time_elapse_ms; scan_per_second = scan_count * 1000 / time_elapse_ms;
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST, log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST,
"FQDNPluginScan match rate on %d-threads speed %lld lookups/s/thread", "FQDNPluginScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second); PERF_THREAD_NUM, scan_per_second);
} }
@@ -1010,9 +1010,9 @@ protected:
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info_path); _shared_maat_inst = maat_new(opts, table_info_path);
maat_options_free(opts); maat_options_free(opts);
if (NULL == _shared_maat_instance) { if (NULL == _shared_maat_inst) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in BoolPluginScan failed.", "[%s:%d] create maat instance in BoolPluginScan failed.",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
@@ -1020,15 +1020,15 @@ protected:
} }
static void TearDownTestCase() { static void TearDownTestCase() {
maat_free(_shared_maat_instance); maat_free(_shared_maat_inst);
log_handle_destroy(logger); log_handle_destroy(logger);
} }
static struct log_handle *logger; static struct log_handle *logger;
static struct maat *_shared_maat_instance; static struct maat *_shared_maat_inst;
}; };
struct maat *MaatPerfBoolPluginScan::_shared_maat_instance; struct maat *MaatPerfBoolPluginScan::_shared_maat_inst;
struct log_handle *MaatPerfBoolPluginScan::logger; struct log_handle *MaatPerfBoolPluginScan::logger;
struct bool_plugin_ud { struct bool_plugin_ud {
@@ -1076,11 +1076,11 @@ void perf_bool_plugin_ex_dup_cb(int table_id, void **to, void **from, long argl,
void* perf_bool_plugin_scan_thread(void *arg) void* perf_bool_plugin_scan_thread(void *arg)
{ {
struct thread_param *param = (struct thread_param *)arg; struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance; struct maat *maat_inst = param->maat_inst;
struct perf_fqdn_plugin_ud *result[ARRAY_SIZE]; struct perf_fqdn_plugin_ud *result[ARRAY_SIZE];
int i=0, j=0, ret=0, hit_times=0; int i=0, j=0, ret=0, hit_times=0;
int table_id = maat_get_table_id(maat_instance, param->table_name); int table_id = maat_get_table_id(maat_inst, param->table_name);
memset(&result, 0, sizeof(result)); memset(&result, 0, sizeof(result));
struct timespec start, end; struct timespec start, end;
@@ -1089,7 +1089,7 @@ void* perf_bool_plugin_scan_thread(void *arg)
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < param->test_count; i++) { for (i = 0; i < param->test_count; i++) {
ret = maat_bool_plugin_table_get_ex_data(maat_instance, table_id, items_4, ret = maat_bool_plugin_table_get_ex_data(maat_inst, table_id, items_4,
sizeof(items_4)/sizeof(unsigned long long), sizeof(items_4)/sizeof(unsigned long long),
(void**)result, 6); (void**)result, 6);
if (ret == 1) { if (ret == 1) {
@@ -1116,19 +1116,19 @@ void* perf_bool_plugin_scan_thread(void *arg)
void *perf_bool_plugin_update_thread(void *arg) void *perf_bool_plugin_update_thread(void *arg)
{ {
struct thread_param *param = (struct thread_param *)arg; struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance; struct maat *maat_inst = param->maat_inst;
const int CMD_EXPR_NUM = 20; const int CMD_EXPR_NUM = 20;
int i = 0; int i = 0;
struct maat_cmd_line line_rule; struct maat_cmd_line line_rule;
char line_buff[1024]; char line_buff[1024];
for (i = 0; i < CMD_EXPR_NUM; i++) { for (i = 0; i < CMD_EXPR_NUM; i++) {
line_rule.rule_id = (int)maat_cmd_incrby(maat_instance, "TEST_PLUG_SEQ", 1); line_rule.rule_id = (int)maat_cmd_incrby(maat_inst, "TEST_PLUG_SEQ", 1);
line_rule.table_name = param->table_name; line_rule.table_name = param->table_name;
snprintf(line_buff, 1024, "%lld\t1&%d\ttunnel2\t1", line_rule.rule_id, i); snprintf(line_buff, 1024, "%lld\t1&%d\ttunnel2\t1", line_rule.rule_id, i);
line_rule.table_line = line_buff; line_rule.table_line = line_buff;
line_rule.expire_after = 0; line_rule.expire_after = 0;
maat_cmd_set_line(maat_instance, &line_rule); maat_cmd_set_line(maat_inst, &line_rule);
sleep(WAIT_FOR_EFFECTIVE_S); sleep(WAIT_FOR_EFFECTIVE_S);
} }
@@ -1141,12 +1141,12 @@ void *perf_bool_plugin_update_thread(void *arg)
TEST_F(MaatPerfBoolPluginScan, MultiThread) { TEST_F(MaatPerfBoolPluginScan, MultiThread) {
int ex_data_counter = 0; int ex_data_counter = 0;
const char *table_name = "TEST_BOOL_PLUGIN_WITH_EXDATA"; const char *table_name = "TEST_BOOL_PLUGIN_WITH_EXDATA";
struct maat *maat_instance = MaatPerfBoolPluginScan::_shared_maat_instance; struct maat *maat_inst = MaatPerfBoolPluginScan::_shared_maat_inst;
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0); ASSERT_GT(table_id, 0);
int ret = maat_plugin_table_ex_schema_register(maat_instance, table_name, int ret = maat_plugin_table_ex_schema_register(maat_inst, table_name,
perf_bool_plugin_ex_new_cb, perf_bool_plugin_ex_new_cb,
perf_bool_plugin_ex_free_cb, perf_bool_plugin_ex_free_cb,
perf_bool_plugin_ex_dup_cb, perf_bool_plugin_ex_dup_cb,
@@ -1159,7 +1159,7 @@ TEST_F(MaatPerfBoolPluginScan, MultiThread) {
int *is_all_hit = NULL; int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) { for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_instance = maat_instance; thread_params[i].maat_inst = maat_inst;
thread_params[i].thread_id = i; thread_params[i].thread_id = i;
thread_params[i].table_name = table_name; thread_params[i].table_name = table_name;
thread_params[i].test_count = PERF_SCAN_COUNT; thread_params[i].test_count = PERF_SCAN_COUNT;
@@ -1188,7 +1188,7 @@ TEST_F(MaatPerfBoolPluginScan, MultiThread) {
} }
scan_per_second = scan_count * 1000 / time_elapse_ms; scan_per_second = scan_count * 1000 / time_elapse_ms;
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST, log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST,
"BoolPluginScan match rate on %d-threads speed %lld lookups/s/thread", "BoolPluginScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second); PERF_THREAD_NUM, scan_per_second);
} }
@@ -1208,9 +1208,9 @@ protected:
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info); _shared_maat_inst = maat_new(opts, table_info);
maat_options_free(opts); maat_options_free(opts);
if (NULL == _shared_maat_instance) { if (NULL == _shared_maat_inst) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatPerfFileScan failed.", "[%s:%d] create maat instance in MaatPerfFileScan failed.",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
@@ -1218,15 +1218,15 @@ protected:
} }
static void TearDownTestCase() { static void TearDownTestCase() {
maat_free(_shared_maat_instance); maat_free(_shared_maat_inst);
log_handle_destroy(logger); log_handle_destroy(logger);
} }
static struct log_handle *logger; static struct log_handle *logger;
static struct maat *_shared_maat_instance; static struct maat *_shared_maat_inst;
}; };
struct maat *MaatPerfFileScan::_shared_maat_instance; struct maat *MaatPerfFileScan::_shared_maat_inst;
struct log_handle *MaatPerfFileScan::logger; struct log_handle *MaatPerfFileScan::logger;
struct perf_ip_plugin_ud { struct perf_ip_plugin_ud {
@@ -1274,9 +1274,9 @@ static void *ip_plugin_get_thread(void *arg)
const char *table_name = "TSG_IP_LOCATION_BUILT_IN"; const char *table_name = "TSG_IP_LOCATION_BUILT_IN";
int test_times = 1000*1000, hit_times = 0; int test_times = 1000*1000, hit_times = 0;
int ret = 0, i=0, j=0; int ret = 0, i=0, j=0;
struct maat *maat_instance = (struct maat *)arg; struct maat *maat_inst = (struct maat *)arg;
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
struct timespec start, end; struct timespec start, end;
struct ip_addr ipv4; struct ip_addr ipv4;
@@ -1287,7 +1287,7 @@ static void *ip_plugin_get_thread(void *arg)
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
struct perf_ip_plugin_ud *results[ARRAY_SIZE]; struct perf_ip_plugin_ud *results[ARRAY_SIZE];
for (i = 0; i < test_times; i++) { for (i = 0; i < test_times; i++) {
ret = maat_ip_plugin_table_get_ex_data(maat_instance, table_id, &ipv4, (void**)results, 4); ret = maat_ip_plugin_table_get_ex_data(maat_inst, table_id, &ipv4, (void**)results, 4);
if (ret > 0) { if (ret > 0) {
hit_times++; hit_times++;
} }
@@ -1299,7 +1299,7 @@ static void *ip_plugin_get_thread(void *arg)
clock_gettime(CLOCK_MONOTONIC, &end); clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000; long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST, log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST,
"ip_plugin_get_ex_data time_elapse:%lldms hit_times:%d", "ip_plugin_get_ex_data time_elapse:%lldms hit_times:%d",
time_elapse_ms, hit_times); time_elapse_ms, hit_times);
@@ -1309,15 +1309,15 @@ static void *ip_plugin_get_thread(void *arg)
} }
TEST_F(MaatPerfFileScan, IPPlugin) { TEST_F(MaatPerfFileScan, IPPlugin) {
struct maat *maat_instance = MaatPerfFileScan::_shared_maat_instance; struct maat *maat_inst = MaatPerfFileScan::_shared_maat_inst;
const char* table_name = "TSG_IP_LOCATION_BUILT_IN"; const char* table_name = "TSG_IP_LOCATION_BUILT_IN";
int ip_plugin_ex_data_counter = 0; int ip_plugin_ex_data_counter = 0;
int* is_all_hit = NULL; int* is_all_hit = NULL;
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0); ASSERT_GT(table_id, 0);
int ret = maat_plugin_table_ex_schema_register(maat_instance, table_name, int ret = maat_plugin_table_ex_schema_register(maat_inst, table_name,
perf_ip_plugin_EX_new_cb, perf_ip_plugin_EX_new_cb,
perf_ip_plugin_EX_free_cb, perf_ip_plugin_EX_free_cb,
perf_ip_plugin_EX_dup_cb, perf_ip_plugin_EX_dup_cb,
@@ -1328,7 +1328,7 @@ TEST_F(MaatPerfFileScan, IPPlugin) {
pthread_t threads[PERF_THREAD_NUM]; pthread_t threads[PERF_THREAD_NUM];
for (i = 0; i < PERF_THREAD_NUM; i++) { for (i = 0; i < PERF_THREAD_NUM; i++) {
pthread_create(&(threads[i]), NULL, ip_plugin_get_thread, maat_instance); pthread_create(&(threads[i]), NULL, ip_plugin_get_thread, maat_inst);
} }
for (i = 0; i < PERF_THREAD_NUM; i++) { for (i = 0; i < PERF_THREAD_NUM; i++) {

40
test/maat_input_mode_gtest.cpp
View File

@@ -37,33 +37,33 @@ TEST(json_mode, maat_scan_string) {
maat_options_set_json_file(opts, json_path); maat_options_set_json_file(opts, json_path);
maat_options_set_logger(opts, "./maat_input_mode_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_input_mode_gtest.log", LOG_LEVEL_INFO);
struct maat *maat_instance = maat_new(opts, table_info_path); struct maat *maat_inst = maat_new(opts, table_info_path);
EXPECT_TRUE(maat_instance != NULL); EXPECT_TRUE(maat_inst != NULL);
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
long long results[5] = {0}; long long results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
int thread_id = 0; int thread_id = 0;
struct maat_state *state = maat_state_new(maat_instance, thread_id); struct maat_state *state = maat_state_new(maat_inst, thread_id);
int ret = maat_scan_string(maat_instance, table_id, scan_data, strlen(scan_data), int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, sizeof(results), &n_hit_result, state); results, sizeof(results), &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 182); EXPECT_EQ(results[0], 182);
long long clause_id = maat_runtime_get_sequence(maat_instance->maat_rt, "test_seq"); long long clause_id = maat_runtime_get_sequence(maat_inst->maat_rt, "test_seq");
EXPECT_EQ(clause_id, 0); EXPECT_EQ(clause_id, 0);
clause_id = maat_runtime_get_sequence(maat_instance->maat_rt, "test_seq"); clause_id = maat_runtime_get_sequence(maat_inst->maat_rt, "test_seq");
EXPECT_EQ(clause_id, 1); EXPECT_EQ(clause_id, 1);
clause_id = maat_runtime_get_sequence(maat_instance->maat_rt, "test_seq"); clause_id = maat_runtime_get_sequence(maat_inst->maat_rt, "test_seq");
EXPECT_EQ(clause_id, 2); EXPECT_EQ(clause_id, 2);
maat_options_free(opts); maat_options_free(opts);
maat_state_free(state); maat_state_free(state);
state = NULL; state = NULL;
maat_free(maat_instance); maat_free(maat_inst);
} }
TEST(iris_mode, maat_scan_string) { TEST(iris_mode, maat_scan_string) {
@@ -93,17 +93,17 @@ TEST(iris_mode, maat_scan_string) {
maat_options_set_iris(opts, tmp_iris_full_idx_path, tmp_iris_inc_idx_path); maat_options_set_iris(opts, tmp_iris_full_idx_path, tmp_iris_inc_idx_path);
maat_options_set_logger(opts, "./maat_input_mode_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_input_mode_gtest.log", LOG_LEVEL_INFO);
struct maat *maat_instance = maat_new(opts, table_info_path); struct maat *maat_inst = maat_new(opts, table_info_path);
EXPECT_TRUE(maat_instance != NULL); EXPECT_TRUE(maat_inst != NULL);
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
long long results[5] = {0}; long long results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
int thread_id = 0; int thread_id = 0;
struct maat_state *state = maat_state_new(maat_instance, thread_id); struct maat_state *state = maat_state_new(maat_inst, thread_id);
int ret = maat_scan_string(maat_instance, table_id, scan_data, strlen(scan_data), int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, sizeof(results), &n_hit_result, state); results, sizeof(results), &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
@@ -112,7 +112,7 @@ TEST(iris_mode, maat_scan_string) {
maat_options_free(opts); maat_options_free(opts);
maat_state_free(state); maat_state_free(state);
state = NULL; state = NULL;
maat_free(maat_instance); maat_free(maat_inst);
} }
int count_line_num_cb(const char *table_name, const char *line, void *u_para) int count_line_num_cb(const char *table_name, const char *line, void *u_para)
@@ -217,15 +217,15 @@ TEST(redis_mode, maat_scan_string) {
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_input_mode_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_input_mode_gtest.log", LOG_LEVEL_INFO);
struct maat *maat_instance = maat_new(opts, table_info_path); struct maat *maat_inst = maat_new(opts, table_info_path);
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
long long results[5] = {0}; long long results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
int thread_id = 0; int thread_id = 0;
struct maat_state *state = maat_state_new(maat_instance, thread_id); struct maat_state *state = maat_state_new(maat_inst, thread_id);
int ret = maat_scan_string(maat_instance, table_id, scan_data, strlen(scan_data), int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, sizeof(results), &n_hit_result, state); results, sizeof(results), &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
@@ -234,7 +234,7 @@ TEST(redis_mode, maat_scan_string) {
maat_options_free(opts); maat_options_free(opts);
maat_state_free(state); maat_state_free(state);
state = NULL; state = NULL;
maat_free(maat_instance); maat_free(maat_inst);
} }
int main(int argc, char ** argv) int main(int argc, char ** argv)