gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reduce unnecessary memory usage & add maat_stat info

Browse Source
This commit is contained in:
刘文坛
2023-05-31 09:13:14 +00:00
parent a006c22944
commit 64ac2dd7da
11 changed files with 286 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/inc_internal/maat_group.h
View File

@@ -47,6 +47,8 @@ size_t group2group_runtime_get_super_groups(void *g2g_runtime, long long *group_
long long group2group_runtime_rule_count(void *g2g_runtime);
long long group2group_runtime_exclude_rule_count(void *g2g_runtime);
long long group2group_runtime_update_err_count(void *g2g_runtime);
#ifdef __cplusplus

18
src/inc_internal/maat_rule.h
View File

@@ -53,13 +53,6 @@ enum tag_match {
TAG_MATCH_MATCHED
};
struct maat_item_inner {
long long magic_num;
long long item_id;
long long group_id;
long long district_id;
};
struct maat_item {
long long item_id;
long long group_id;
@@ -161,10 +154,8 @@ struct maat_stat {
long long *thread_call_cnt;
long long *hit_cnt;
long long *not_grp_hit_cnt;
long long *maat_state_cnt;
long long scan_bytes;
long long scan_cnt;
long long update_err_cnt;
long long scan_err_cnt;
long long zombie_rs_stream;
long long line_cmd_acc_num;
@@ -222,10 +213,11 @@ int my_scandir(const char *dir, struct dirent ***namelist,
int(*filter)(const struct dirent *),
int(*compar)(const void *, const void *));
struct maat_item_inner *maat_item_inner_new(long long item_id, long long group_id,
long long district_id);
struct item_district *item_district_new(long long district_id);
void maat_item_inner_free(void *item_inner);
void item_district_free(void *item_district);
long long item_district_id(struct item_district *item_dist);
void *rule_monitor_loop(void *arg);

36
src/maat_api.c
View File

@@ -932,8 +932,8 @@ int interval_scan(struct table_manager *tbl_mgr, int thread_id, long long intege
{
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
if (table_type == TABLE_TYPE_INTERVAL_PLUS && DISTRICT_FLAG_UNSET == state->is_set_district) {
// maat_instance->scan_err_cnt++;
if (table_type == TABLE_TYPE_INTERVAL_PLUS &&
DISTRICT_FLAG_UNSET == state->is_set_district) {
return -1;
}
@@ -1014,8 +1014,8 @@ int string_scan(struct table_manager *tbl_mgr, int thread_id, const char *data,
int physical_table_id, int vtable_id, struct maat_state *state)
{
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
if (table_type == TABLE_TYPE_EXPR_PLUS && DISTRICT_FLAG_UNSET == state->is_set_district) {
// maat_instance->scan_err_cnt++;
if (table_type == TABLE_TYPE_EXPR_PLUS &&
DISTRICT_FLAG_UNSET == state->is_set_district) {
return -1;
}
@@ -1051,7 +1051,6 @@ int expr_stream_scan(struct maat_stream *stream, const char *data, size_t data_l
enum table_type table_type = table_manager_get_table_type(tbl_mgr, stream->physical_table_id);
if (table_type == TABLE_TYPE_EXPR_PLUS &&
DISTRICT_FLAG_UNSET == state->is_set_district) {
// maat_instance->scan_err_cnt++;
return -1;
}
@@ -1133,11 +1132,13 @@ int maat_scan_flag(struct maat *maat_instance, int table_id,
}
if (physical_table_id < 0) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
@@ -1147,6 +1148,7 @@ int maat_scan_flag(struct maat *maat_instance, int table_id,
int hit_group_cnt = flag_scan(maat_instance->tbl_mgr, state->thread_id, flag,
physical_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
@@ -1217,11 +1219,13 @@ int maat_scan_integer(struct maat *maat_instance, int table_id,
}
if (physical_table_id < 0) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
@@ -1231,6 +1235,7 @@ int maat_scan_integer(struct maat *maat_instance, int table_id,
int hit_group_cnt = interval_scan(maat_instance->tbl_mgr, state->thread_id, integer,
physical_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
@@ -1302,11 +1307,13 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id,
}
if (physical_table_id < 0) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (table_type != TABLE_TYPE_IP_PLUS) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
@@ -1316,6 +1323,7 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id,
int hit_group_cnt = ipv4_scan(maat_instance->tbl_mgr, state->thread_id, ip_addr, port, protocol,
physical_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
@@ -1387,11 +1395,13 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id,
}
if (physical_table_id < 0) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (table_type != TABLE_TYPE_IP_PLUS) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
@@ -1401,6 +1411,7 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id,
int hit_group_cnt = ipv6_scan(maat_instance->tbl_mgr, state->thread_id, ip_addr, port, protocol,
physical_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
@@ -1471,11 +1482,13 @@ int maat_scan_string(struct maat *maat_instance, int table_id, const char *data,
}
if (physical_table_id < 0) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
@@ -1485,10 +1498,11 @@ int maat_scan_string(struct maat *maat_instance, int table_id, const char *data,
int hit_group_cnt = string_scan(maat_instance->tbl_mgr, state->thread_id, data, data_len,
physical_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
size_t sum_hit_compile_cnt = 0;
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_instance, results, n_result, state);
*n_hit_result = sum_hit_compile_cnt;
@@ -1607,6 +1621,7 @@ int maat_stream_scan(struct maat_stream *maat_stream, const char *data, int data
int hit_group_cnt = expr_stream_scan(maat_stream, data, data_len, state);
if (hit_group_cnt < 0) {
maat_instance->stat->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
@@ -1669,6 +1684,8 @@ struct maat_state *maat_state_new(struct maat *maat_instance, int thread_id)
state->thread_id = thread_id;
state->compile_state = maat_compile_state_new(thread_id);
alignment_int64_array_add(maat_instance->stat->maat_state_cnt, thread_id, 1);
return state;
}
@@ -1694,6 +1711,11 @@ void maat_state_free(struct maat_state *state)
return;
}
assert(state->maat_instance != NULL);
struct maat *maat_instance = state->maat_instance;
long long thread_id = state->thread_id;
if (state->compile_state != NULL) {
maat_compile_state_free(state->compile_state);
state->compile_state = NULL;
@@ -1701,6 +1723,8 @@ void maat_state_free(struct maat_state *state)
state->maat_instance = NULL;
FREE(state);
alignment_int64_array_add(maat_instance->stat->maat_state_cnt, thread_id, -1);
}
int maat_state_set_scan_district(struct maat_state *state, int vtable_id,

39
src/maat_expr.c
View File

@@ -797,11 +797,9 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
return -1;
}
struct maat_item_inner *inner_item = maat_item_inner_new(expr_item->item_id,
expr_item->group_id,
expr_item->district_id);
expr_item->user_data = inner_item;
expr_item->user_data_free = maat_item_inner_free;
struct item_district *item_dist = item_district_new(expr_item->district_id);
expr_item->user_data = item_dist;
expr_item->user_data_free = item_district_free;
}
int ret = expr_runtime_update_row(expr_rt, (char *)&item_id, sizeof(long long),
@@ -848,6 +846,7 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name,
int ret = 0;
size_t i = 0;
size_t real_rule_cnt = 0;
size_t real_regex_rule_cnt = 0;
struct expr_rule *rules = NULL;
void **ex_data_array = NULL;
@@ -856,14 +855,17 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name,
rules = ALLOC(struct expr_rule, rule_cnt);
for (i = 0; i < rule_cnt; i++) {
struct expr_item *expr_item = (struct expr_item *)ex_data_array[i];
struct expr_rule tmp_rule;
struct expr_rule tmp_rule = {0};
ret = expr_item_to_expr_rule(expr_item, &tmp_rule, expr_rt->logger);
if (ret < 0) {
continue;
}
rules[i] = tmp_rule;
real_rule_cnt++;
rules[real_rule_cnt++] = tmp_rule;
if (expr_item->expr_type == EXPR_TYPE_REGEX) {
real_regex_rule_cnt++;
}
}
}
@@ -871,12 +873,12 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name,
struct adapter_hs *old_adapter_hs = NULL;
if (rule_cnt > 0) {
new_adapter_hs = adapter_hs_new(expr_rt->n_worker_thread, rules, rule_cnt,
new_adapter_hs = adapter_hs_new(expr_rt->n_worker_thread, rules, real_rule_cnt,
expr_rt->logger);
if (NULL == new_adapter_hs) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] table[%s] rebuild adapter_hs engine failed when update"
" %zu expr rules", __FUNCTION__, __LINE__, table_name, rule_cnt);
" %zu expr rules", __FUNCTION__, __LINE__, table_name, real_rule_cnt);
ret = -1;
}
}
@@ -890,13 +892,14 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name,
garbage_adapter_hs_free);
}
expr_rt->rule_num = rule_cnt;
expr_rt->rule_num = real_rule_cnt;
expr_rt->regex_rule_num = real_regex_rule_cnt;
expr_rt->version = maat_rt_version;
log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] has %zu rules, commit %zu expr rules and rebuild adapter_hs"
" completed, version:%lld", table_name, rule_cnt, real_rule_cnt,
expr_rt->version);
"table[%s] has %zu rules, commit %zu expr rules(regex rules:%zu) "
"and rebuild adapter_hs completed, version:%lld", table_name, rule_cnt,
real_rule_cnt, real_regex_rule_cnt, expr_rt->version);
if (rules != NULL) {
for (i = 0; i < rule_cnt; i++) {
@@ -973,15 +976,15 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
}
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *inner_item = NULL;
struct item_district *item_dist = NULL;
struct expr_item *expr_item = NULL;
size_t real_hit_item_cnt = 0;
long long district_id = state->district_id;
for (size_t i = 0; i < n_hit_item; i++) {
inner_item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (inner_item->district_id == district_id ||
inner_item->district_id == DISTRICT_ANY) {
item_dist = (struct item_district *)(hit_results[i].user_tag);
long long tag_district_id = item_district_id(item_dist);
if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id;
expr_item = (struct expr_item *)rcu_hash_find(expr_rt->item_hash,
(char *)&item_id,

16
src/maat_flag.c
View File

@@ -425,11 +425,9 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table
return -1;
}
struct maat_item_inner *inner_item = maat_item_inner_new(flag_item->item_id,
flag_item->group_id,
flag_item->district_id);
flag_item->user_data = inner_item;
flag_item->user_data_free = maat_item_inner_free;
struct item_district *item_dist = item_district_new(flag_item->district_id);
flag_item->user_data = item_dist;
flag_item->user_data_free = item_district_free;
}
int ret = flag_runtime_update_row(flag_rt, (char *)&item_id, sizeof(long long),
@@ -558,15 +556,15 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
}
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *inner_item = NULL;
struct item_district *item_dist = NULL;
struct flag_item *flag_item = NULL;
size_t real_hit_item_cnt = 0;
long long district_id = state->district_id;
for (int i = 0; i < n_hit_item; i++) {
inner_item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (inner_item->district_id == district_id ||
inner_item->district_id == DISTRICT_ANY) {
item_dist = (struct item_district *)(hit_results[i].user_tag);
long long tag_district_id = item_district_id(item_dist);
if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id;
flag_item = (struct flag_item *)rcu_hash_find(flag_rt->item_hash,
(char *)&item_id,

17
src/maat_group.c
View File

@@ -64,6 +64,7 @@ struct group2group_runtime {
struct maat_group_topology *updating_group_topo;
long long version;
long long rule_num;
long long excl_rule_num; //exclude g2g rule num
long long update_err_cnt;
int updating_flag;
@@ -714,6 +715,9 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
g2g_item->is_exclude);
if (0 == ret) {
g2g_rt->rule_num--;
if (1 == g2g_item->is_exclude) {
g2g_rt->excl_rule_num--;
}
} else {
g2g_rt->update_err_cnt++;
}
@@ -725,6 +729,9 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
g2g_item->is_exclude);
if (0 == ret) {
g2g_rt->rule_num++;
if (1 == g2g_item->is_exclude) {
g2g_rt->excl_rule_num++;
}
} else {
g2g_rt->update_err_cnt++;
}
@@ -1088,6 +1095,16 @@ long long group2group_runtime_rule_count(void *g2g_runtime)
return g2g_rt->rule_num;
}
long long group2group_runtime_exclude_rule_count(void *g2g_runtime)
{
if (NULL == g2g_runtime) {
return 0;
}
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
return g2g_rt->excl_rule_num;
}
long long group2group_runtime_update_err_count(void *g2g_runtime)
{
if (NULL == g2g_runtime) {

16
src/maat_interval.c
View File

@@ -429,11 +429,9 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
return -1;
}
struct maat_item_inner *inner_item = maat_item_inner_new(interval_item->item_id,
interval_item->group_id,
interval_item->district_id);
interval_item->user_data = inner_item;
interval_item->user_data_free = maat_item_inner_free;
struct item_district *item_dist = item_district_new(interval_item->district_id);
interval_item->user_data = item_dist;
interval_item->user_data_free = item_district_free;
}
int ret = interval_runtime_update_row(interval_rt, (char *)&item_id, sizeof(long long),
@@ -562,15 +560,15 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
}
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *inner_item = NULL;
struct item_district *item_dist = NULL;
struct interval_item *int_item = NULL;
size_t real_hit_item_cnt = 0;
long long district_id = state->district_id;
for (int i = 0; i < n_hit_item; i++) {
inner_item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (inner_item->district_id == district_id ||
inner_item->district_id == DISTRICT_ANY) {
item_dist = (struct item_district *)(hit_results[i].user_tag);
long long tag_district_id = item_district_id(item_dist);
if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id;
int_item = (struct interval_item *)rcu_hash_find(interval_rt->item_hash,
(char *)&item_id,

15
src/maat_ip.c
View File

@@ -62,8 +62,6 @@ struct ip_item {
uint16_t min_port;
uint16_t max_port;
int proto;
void *user_data;
void (*user_data_free)(void *data);
};
struct ip_runtime {
@@ -386,12 +384,7 @@ void ip_item_free(struct ip_item *item)
if (NULL == item) {
return;
}
if (item->user_data != NULL && item->user_data_free != NULL) {
item->user_data_free(item->user_data);
item->user_data = NULL;
}
FREE(item);
}
@@ -547,12 +540,6 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
ip_rt->update_err_cnt++;
return -1;
}
struct maat_item_inner *inner_item = maat_item_inner_new(ip_item->item_id,
ip_item->group_id,
0);
ip_item->user_data = inner_item;
ip_item->user_data_free = maat_item_inner_free;
}
int ret = ip_runtime_update_row(ip_rt, (char *)&item_id, sizeof(long long),

49
src/maat_rule.c
View File

@@ -32,30 +32,38 @@
#define MODULE_MAAT_RULE module_name_str("maat.rule")
#define ITEM_RULE_MAGIC 0x4d3c2b1a
struct maat_item_inner *maat_item_inner_new(long long item_id, long long group_id,
long long district_id)
{
struct maat_item_inner *item = ALLOC(struct maat_item_inner, 1);
item->magic_num = ITEM_RULE_MAGIC;
item->item_id = item_id;
item->group_id = group_id;
item->district_id = district_id;
struct item_district {
long long magic_num;
long long district_id;
};
return item;
#define ITEM_RULE_MAGIC 0x4d3c2b1a
struct item_district *
item_district_new(long long district_id)
{
struct item_district *item_dist = ALLOC(struct item_district, 1);
item_dist->magic_num = ITEM_RULE_MAGIC;
item_dist->district_id = district_id;
return item_dist;
}
void maat_item_inner_free(void *item_inner)
void item_district_free(void *item_district)
{
if (NULL == item_inner) {
if (NULL == item_district) {
return;
}
struct maat_item_inner *item = (struct maat_item_inner *)item_inner;
assert(item->magic_num == ITEM_RULE_MAGIC);
item->magic_num = 0;
struct item_district *item_dist = (struct item_district *)item_district;
assert(item_dist->magic_num == ITEM_RULE_MAGIC);
item_dist->magic_num = 0;
FREE(item);
FREE(item_dist);
}
long long item_district_id(struct item_district *item_dist)
{
return item_dist->district_id;
}
struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance)
@@ -63,7 +71,8 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in
struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1);
maat_rt->version = version;
int ret = table_manager_runtime_create(maat_instance->tbl_mgr, maat_instance->opts.nr_worker_thread,
int ret = table_manager_runtime_create(maat_instance->tbl_mgr,
maat_instance->opts.nr_worker_thread,
maat_instance->garbage_bin);
if (ret < 0) {
FREE(maat_rt);
@@ -376,7 +385,11 @@ void *rule_monitor_loop(void *arg)
char err_str[NAME_MAX] = {0};
struct stat attrib;
while (maat_instance->is_running) {
log_info(maat_instance->logger, MODULE_MAAT_RULE, "%s:%d.............", __FUNCTION__, __LINE__);
if (time(NULL) % 10 == 0) {
log_info(maat_instance->logger, MODULE_MAAT_RULE,
"%s thread still alive.........", __FUNCTION__);
}
usleep(maat_instance->opts.rule_update_checking_interval_ms * 1000);
if (0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
switch (maat_instance->opts.input_mode) {

223
src/maat_stat.c
View File

@@ -30,41 +30,46 @@ enum MAAT_FS_STATUS {
STATUS_PLUGIN_ACC_NUM,
STATUS_GROUP_REF_NUM,
STATUS_GROUP_REF_NOT_NUM,
STATUS_GROUP_REF_EXCL_NUM, //group reference exclude group num
STATUS_COMPILE_RULE_NUM,
STATUS_MAAT_STATE_NUM,
STATUS_GARBAGE_QSIZE,
STATUS_TOTAL_SCAN_LEN,
STATUS_TOTAL_SCAN_BYTES,
STATUS_TOTAL_SCAN_CNT,
STATUS_UPDATE_ERR_CNT,
STATUS_ICONV_ERR_CNT,
STATUS_SCAN_ERR_CNT,
STATUS_ZOMBIE_RS_STREAM,
STATUS_NOT_GROUP_HIT,
STATUS_CMD_NUM,
STATUS_CMD_Q_SIZE,
STATUS_CMD_LINE_NUM
};
enum MAAT_FS_COLUMN {
COLUMN_TABLE_RULE_NUM = 0,
COLUMN_TABLE_REGEX_NUM,
COLUMN_TABLE_IPV6_NUM,
COLUMN_TABLE_STREAM_NUM,
COLUMN_TABLE_SCAN_CNT,
COLUMN_TABLE_SCAN_BYTES,
COLUMN_TABLE_CPU_TIME,//microseconds
COLUMN_TABLE_HIT_CNT,
COLUMN_RULE_NUM = 0,
COLUMN_REGEX_NUM,
COLUMN_STREAM_NUM,
COLUMN_SCAN_CNT,
COLUMN_SCAN_BYTES,
COLUMN_CPU_TIME, //microseconds
COLUMN_HIT_CNT,
COLUMN_SCAN_BPS, //scan bytes per second
COLUMN_SCAN_TPS, //scan times per second
COLUMN_HIT_RATE //scan hit rate(hit_cnt/scan_cnt)
};
const char *common_column_name[] = {"rule", "reg", "v6", "stream", "scan_cnt", "scan_bytes", "scan_cpu_time", "hit_cnt"};
const char *common_column_name[] = {"rule", "reg/v6", "stream", "scan_cnt", "scan_bytes",
"scan_cpu_time", "hit_cnt", "IN_Bps", "IN_Tps", "hit_rate"};
enum field_type common_column_type[] = {
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE,
FIELD_TYPE_GAUGE
};
@@ -85,6 +90,8 @@ void maat_fieldstat_register(struct maat_stat *stat)
"group", NULL, 0);
stat->fs_status_id[STATUS_GROUP_REF_NOT_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"not_grp", NULL, 0);
stat->fs_status_id[STATUS_GROUP_REF_EXCL_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"excl_grp", NULL, 0);
stat->fs_status_id[STATUS_COMPILE_RULE_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"compile", NULL, 0);
stat->fs_status_id[STATUS_GARBAGE_QSIZE] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
@@ -95,18 +102,14 @@ void maat_fieldstat_register(struct maat_stat *stat)
"z_stream", NULL, 0);
stat->fs_status_id[STATUS_NOT_GROUP_HIT] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"nt_grp_hit", NULL, 0);
stat->fs_status_id[STATUS_TOTAL_SCAN_LEN] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
stat->fs_status_id[STATUS_TOTAL_SCAN_BYTES] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"scan_bytes", NULL, 0);
stat->fs_status_id[STATUS_TOTAL_SCAN_CNT] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"scan_times", NULL, 0);
stat->fs_status_id[STATUS_UPDATE_ERR_CNT] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"update_err", NULL, 0);
"update_error", NULL, 0);
stat->fs_status_id[STATUS_SCAN_ERR_CNT] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"scan_error", NULL, 0);
stat->fs_status_id[STATUS_CMD_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"cmd_commit", NULL, 0);
stat->fs_status_id[STATUS_CMD_Q_SIZE] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"cmd_in_q", NULL, 0);
stat->fs_status_id[STATUS_CMD_LINE_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"line_cmd/s", NULL, 0);
}
@@ -122,6 +125,30 @@ int maat_fieldstat_table_row_register(struct maat_stat *stat, struct table_manag
return -1;
}
ret = fieldstat_set_metric_ratio_para(stat->fs_handle, stat->total_stat_id[COLUMN_SCAN_BPS],
stat->total_stat_id[COLUMN_SCAN_BYTES],
stat->total_stat_id[COLUMN_CPU_TIME], 1000000000);
if (ret < 0) {
log_error(stat->logger, MODULE_MAAT_STAT, "fieldstat set IN_Bps for Sum failed.");
return -1;
}
ret = fieldstat_set_metric_ratio_para(stat->fs_handle, stat->total_stat_id[COLUMN_SCAN_TPS],
stat->total_stat_id[COLUMN_SCAN_CNT],
stat->total_stat_id[COLUMN_CPU_TIME], 1000000000);
if (ret < 0) {
log_error(stat->logger, MODULE_MAAT_STAT, "fieldstat set IN_Tps for Sum failed.");
return -1;
}
ret = fieldstat_set_metric_ratio_para(stat->fs_handle, stat->total_stat_id[COLUMN_HIT_RATE],
stat->total_stat_id[COLUMN_HIT_CNT],
stat->total_stat_id[COLUMN_SCAN_CNT], 1000000000);
if (ret < 0) {
log_error(stat->logger, MODULE_MAAT_STAT, "fieldstat set hit_rate for Sum failed.");
return -1;
}
for (size_t i = 0; i < max_table_cnt; i++) {
void *schema = table_manager_get_schema(tbl_mgr, i);
if (NULL == schema) {
@@ -138,12 +165,40 @@ int maat_fieldstat_table_row_register(struct maat_stat *stat, struct table_manag
"fieldstat_register_table_row %s failed.", table_name);
return -1;
}
ret = fieldstat_set_metric_ratio_para(stat->fs_handle, stat->fs_column_id[i][COLUMN_SCAN_BPS],
stat->fs_column_id[i][COLUMN_SCAN_BYTES],
stat->fs_column_id[i][COLUMN_CPU_TIME], 1000000000);
if (ret < 0) {
log_error(stat->logger, MODULE_MAAT_STAT,
"fieldstat set IN_Bps for %s failed.", table_name);
return -1;
}
ret = fieldstat_set_metric_ratio_para(stat->fs_handle, stat->fs_column_id[i][COLUMN_SCAN_TPS],
stat->fs_column_id[i][COLUMN_SCAN_CNT],
stat->fs_column_id[i][COLUMN_CPU_TIME], 1000000000);
if (ret < 0) {
log_error(stat->logger, MODULE_MAAT_STAT,
"fieldstat set IN_Tps for %s failed.", table_name);
return -1;
}
ret = fieldstat_set_metric_ratio_para(stat->fs_handle, stat->fs_column_id[i][COLUMN_HIT_RATE],
stat->fs_column_id[i][COLUMN_HIT_CNT],
stat->fs_column_id[i][COLUMN_SCAN_CNT], 1000000000);
if (ret < 0) {
log_error(stat->logger, MODULE_MAAT_STAT,
"fieldstat set hit_rate for %s failed.", table_name);
return -1;
}
}
return 0;
}
struct maat_stat *maat_stat_new(const char *stat_file, size_t max_thread_num, struct log_handle *logger)
struct maat_stat *maat_stat_new(const char *stat_file, size_t max_thread_num,
struct log_handle *logger)
{
struct maat_stat *stat = ALLOC(struct maat_stat, 1);
@@ -155,6 +210,7 @@ struct maat_stat *maat_stat_new(const char *stat_file, size_t max_thread_num, st
stat->thread_call_cnt = alignment_int64_array_alloc(max_thread_num);
stat->hit_cnt = alignment_int64_array_alloc(max_thread_num);
stat->not_grp_hit_cnt = alignment_int64_array_alloc(max_thread_num);
stat->maat_state_cnt = alignment_int64_array_alloc(max_thread_num);
return stat;
}
@@ -180,6 +236,11 @@ void maat_stat_free(struct maat_stat *stat)
stat->not_grp_hit_cnt = NULL;
}
if (stat->maat_state_cnt != NULL) {
alignment_int64_array_free(stat->maat_state_cnt);
stat->maat_state_cnt = NULL;
}
if (stat->fs_handle != NULL) {
fieldstat_instance_free(stat->fs_handle);
stat->fs_handle = NULL;
@@ -210,7 +271,8 @@ int maat_stat_init(struct maat_stat *stat, struct table_manager *tbl_mgr,
ret = fieldstat_disable_background_thread(stat->fs_handle);
if (ret < 0) {
log_error(stat->logger, MODULE_MAAT_STAT, "fieldstat_disable_background_thread failed.");
log_error(stat->logger, MODULE_MAAT_STAT,
"fieldstat_disable_background_thread failed.");
return -1;
}
@@ -239,11 +301,12 @@ int maat_stat_init(struct maat_stat *stat, struct table_manager *tbl_mgr,
void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on)
{
long long plugin_cache_num = 0, plugin_rule_num = 0, not_group_num = 0;
long long plugin_cache_num = 0, plugin_rule_num = 0, g2c_not_group_num = 0;
long long compile_rule_num = 0, g2c_rule_num = 0, g2g_rule_num = 0;
long long total_rule_num = 0, total_stream_num = 0, total_input_bytes = 0;
long long total_scan_cnt = 0, total_scan_cpu_time = 0, total_regex_num = 0;
long long total_ipv6_num = 0, total_hit_cnt = 0, total_update_err = 0;
long long g2g_excl_rule_num = 0;
size_t max_table_count = table_manager_table_size(stat->ref_tbl_mgr);
for (size_t i = 0; i < max_table_count; i++) {
@@ -269,15 +332,14 @@ void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on)
break;
case TABLE_TYPE_GROUP2COMPILE:
g2c_rule_num += group2compile_runtime_rule_count(runtime);
not_group_num += group2compile_runtime_not_group_count(runtime);
g2c_not_group_num += group2compile_runtime_not_group_count(runtime);
break;
case TABLE_TYPE_GROUP2GROUP:
g2g_rule_num += group2group_runtime_rule_count(runtime);
//TODO exclude group num
g2g_excl_rule_num += group2group_runtime_exclude_rule_count(runtime);
break;
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
//regex rule cnt
regex_rule_num = expr_runtime_regex_rule_count(runtime);
break;
case TABLE_TYPE_IP_PLUS:
@@ -287,7 +349,7 @@ void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on)
break;
}
if (table_type == TABLE_TYPE_PLUGIN ) {
if (table_type == TABLE_TYPE_PLUGIN) {
continue;
}
@@ -295,66 +357,79 @@ void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on)
total_ipv6_num += ipv6_rule_num;
long long rule_num = table_manager_runtime_rule_count(stat->ref_tbl_mgr, i);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_RULE_NUM], rule_num);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_RULE_NUM], rule_num);
total_rule_num += rule_num;
if (table_type == TABLE_TYPE_EXPR || table_type == TABLE_TYPE_EXPR_PLUS) {
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_REGEX_NUM], regex_rule_num);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_REGEX_NUM], regex_rule_num);
long long stream_num = expr_runtime_stream_num(runtime);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_STREAM_NUM], stream_num);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_STREAM_NUM], stream_num);
total_stream_num += stream_num;
long long input_bytes = expr_runtime_scan_bytes(runtime);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_SCAN_BYTES], input_bytes);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_SCAN_BYTES], input_bytes);
total_input_bytes += input_bytes;
}
if (table_type == TABLE_TYPE_IP_PLUS) {
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_IPV6_NUM], ipv6_rule_num);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_REGEX_NUM], ipv6_rule_num);
}
if (1 == perf_on) {
long long scan_cpu_time = table_manager_runtime_scan_cpu_time(stat->ref_tbl_mgr, i);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_CPU_TIME], scan_cpu_time);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_CPU_TIME], scan_cpu_time);
total_scan_cpu_time += scan_cpu_time;
}
long long scan_cnt = table_manager_runtime_scan_count(stat->ref_tbl_mgr, i);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_SCAN_CNT], scan_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_SCAN_CNT], scan_cnt);
total_scan_cnt += scan_cnt;
long long hit_cnt = table_manager_runtime_hit_count(stat->ref_tbl_mgr, i);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_HIT_CNT], hit_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_HIT_CNT], hit_cnt);
total_hit_cnt += hit_cnt;
total_update_err += table_manager_runtime_update_err_count(stat->ref_tbl_mgr, i);
}
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_RULE_NUM], total_rule_num);
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_REGEX_NUM], total_regex_num);
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_IPV6_NUM], total_ipv6_num);
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_STREAM_NUM], total_stream_num);
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_SCAN_CNT], total_scan_cnt);
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_SCAN_BYTES], total_input_bytes);
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_HIT_CNT], total_hit_cnt);
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_RULE_NUM],
total_rule_num);
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_REGEX_NUM],
total_regex_num);
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_STREAM_NUM],
total_stream_num);
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_SCAN_CNT],
total_scan_cnt);
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_SCAN_BYTES],
total_input_bytes);
if (1 == perf_on) {
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_CPU_TIME], total_scan_cpu_time);
fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_CPU_TIME],
total_scan_cpu_time);
}
stat->scan_cnt += total_scan_cnt;
stat->scan_bytes += total_input_bytes;
stat->update_err_cnt += total_update_err;
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TOTAL_SCAN_LEN], stat->scan_bytes);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TOTAL_SCAN_CNT], stat->scan_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_UPDATE_ERR_CNT], stat->update_err_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_SCAN_ERR_CNT], stat->scan_err_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_ZOMBIE_RS_STREAM], stat->zombie_rs_stream);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_PLUGIN_CACHE_NUM], plugin_cache_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_PLUGIN_ACC_NUM], plugin_rule_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_NUM], g2c_rule_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_NOT_NUM], not_group_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_COMPILE_RULE_NUM], compile_rule_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TOTAL_SCAN_BYTES],
total_input_bytes);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TOTAL_SCAN_CNT],
total_scan_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_UPDATE_ERR_CNT],
total_update_err);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_SCAN_ERR_CNT],
stat->scan_err_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_ZOMBIE_RS_STREAM],
stat->zombie_rs_stream);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_PLUGIN_CACHE_NUM],
plugin_cache_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_PLUGIN_ACC_NUM],
plugin_rule_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_NUM],
g2c_rule_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_NOT_NUM],
g2c_not_group_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_EXCL_NUM],
g2g_excl_rule_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_COMPILE_RULE_NUM],
compile_rule_num);
}
void maat_stat_output(struct maat_stat *stat, long long maat_version, int perf_on)
@@ -365,19 +440,27 @@ void maat_stat_output(struct maat_stat *stat, long long maat_version, int perf_o
long long active_thread_num = alignment_int64_array_cnt(stat->thread_call_cnt,
stat->nr_worker_thread);
//TODO: maat_state_cnt
long long maat_state_cnt = alignment_int64_array_sum(stat->maat_state_cnt,
stat->nr_worker_thread);
long long not_grp_hit_cnt = alignment_int64_array_sum(stat->not_grp_hit_cnt,
stat->nr_worker_thread);
size_t table_cnt = table_manager_table_count(stat->ref_tbl_mgr);
size_t garbage_q_len = maat_garbage_bin_get_size(stat->ref_garbage_bin);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_VERSION], maat_version);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_THRED_NUM], active_thread_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TABLE_NUM], table_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_NOT_GROUP_HIT], not_grp_hit_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_CMD_Q_SIZE], stat->cmd_q_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_CMD_LINE_NUM], stat->line_cmd_acc_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GARBAGE_QSIZE], garbage_q_len);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_VERSION],
maat_version);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_THRED_NUM],
active_thread_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TABLE_NUM],
table_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_MAAT_STATE_NUM],
maat_state_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_NOT_GROUP_HIT],
not_grp_hit_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_CMD_LINE_NUM],
stat->line_cmd_acc_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GARBAGE_QSIZE],
garbage_q_len);
maat_fieldstat_table_row_output(stat, perf_on);

12
test/maat_framework_perf_gtest.cpp
View File

@@ -308,6 +308,8 @@ protected:
}
struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
@@ -522,6 +524,8 @@ protected:
}
struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
@@ -674,6 +678,8 @@ protected:
}
struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
@@ -766,6 +772,8 @@ protected:
}
struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
@@ -994,6 +1002,8 @@ protected:
}
struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
@@ -1191,6 +1201,8 @@ protected:
const char *table_info = "./tsg_table_info.conf";
struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts);
maat_options_set_iris(opts, rule_folder, rule_folder);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_caller_thread_number(opts, 5);