diff --git a/src/inc_internal/maat_group.h b/src/inc_internal/maat_group.h index a39e39b..f03a23e 100644 --- a/src/inc_internal/maat_group.h +++ b/src/inc_internal/maat_group.h @@ -47,6 +47,8 @@ size_t group2group_runtime_get_super_groups(void *g2g_runtime, long long *group_ long long group2group_runtime_rule_count(void *g2g_runtime); +long long group2group_runtime_exclude_rule_count(void *g2g_runtime); + long long group2group_runtime_update_err_count(void *g2g_runtime); #ifdef __cplusplus diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index 0eabb23..9fa5965 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -53,13 +53,6 @@ enum tag_match { TAG_MATCH_MATCHED }; -struct maat_item_inner { - long long magic_num; - long long item_id; - long long group_id; - long long district_id; -}; - struct maat_item { long long item_id; long long group_id; @@ -161,10 +154,8 @@ struct maat_stat { long long *thread_call_cnt; long long *hit_cnt; long long *not_grp_hit_cnt; + long long *maat_state_cnt; - long long scan_bytes; - long long scan_cnt; - long long update_err_cnt; long long scan_err_cnt; long long zombie_rs_stream; long long line_cmd_acc_num; @@ -222,10 +213,11 @@ int my_scandir(const char *dir, struct dirent ***namelist, int(*filter)(const struct dirent *), int(*compar)(const void *, const void *)); -struct maat_item_inner *maat_item_inner_new(long long item_id, long long group_id, - long long district_id); +struct item_district *item_district_new(long long district_id); -void maat_item_inner_free(void *item_inner); +void item_district_free(void *item_district); + +long long item_district_id(struct item_district *item_dist); void *rule_monitor_loop(void *arg); diff --git a/src/maat_api.c b/src/maat_api.c index e6aaebc..7698891 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -932,8 +932,8 @@ int interval_scan(struct table_manager *tbl_mgr, int thread_id, long long intege { enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id); - if (table_type == TABLE_TYPE_INTERVAL_PLUS && DISTRICT_FLAG_UNSET == state->is_set_district) { - // maat_instance->scan_err_cnt++; + if (table_type == TABLE_TYPE_INTERVAL_PLUS && + DISTRICT_FLAG_UNSET == state->is_set_district) { return -1; } @@ -1014,8 +1014,8 @@ int string_scan(struct table_manager *tbl_mgr, int thread_id, const char *data, int physical_table_id, int vtable_id, struct maat_state *state) { enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id); - if (table_type == TABLE_TYPE_EXPR_PLUS && DISTRICT_FLAG_UNSET == state->is_set_district) { - // maat_instance->scan_err_cnt++; + if (table_type == TABLE_TYPE_EXPR_PLUS && + DISTRICT_FLAG_UNSET == state->is_set_district) { return -1; } @@ -1051,7 +1051,6 @@ int expr_stream_scan(struct maat_stream *stream, const char *data, size_t data_l enum table_type table_type = table_manager_get_table_type(tbl_mgr, stream->physical_table_id); if (table_type == TABLE_TYPE_EXPR_PLUS && DISTRICT_FLAG_UNSET == state->is_set_district) { - // maat_instance->scan_err_cnt++; return -1; } @@ -1133,11 +1132,13 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, } if (physical_table_id < 0) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } @@ -1147,6 +1148,7 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int hit_group_cnt = flag_scan(maat_instance->tbl_mgr, state->thread_id, flag, physical_table_id, vtable_id, state); if (hit_group_cnt < 0) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } @@ -1217,11 +1219,13 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, } if (physical_table_id < 0) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } @@ -1231,6 +1235,7 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int hit_group_cnt = interval_scan(maat_instance->tbl_mgr, state->thread_id, integer, physical_table_id, vtable_id, state); if (hit_group_cnt < 0) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } @@ -1302,11 +1307,13 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, } if (physical_table_id < 0) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); if (table_type != TABLE_TYPE_IP_PLUS) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } @@ -1316,6 +1323,7 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int hit_group_cnt = ipv4_scan(maat_instance->tbl_mgr, state->thread_id, ip_addr, port, protocol, physical_table_id, vtable_id, state); if (hit_group_cnt < 0) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } @@ -1387,11 +1395,13 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, } if (physical_table_id < 0) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); if (table_type != TABLE_TYPE_IP_PLUS) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } @@ -1401,6 +1411,7 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int hit_group_cnt = ipv6_scan(maat_instance->tbl_mgr, state->thread_id, ip_addr, port, protocol, physical_table_id, vtable_id, state); if (hit_group_cnt < 0) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } @@ -1471,11 +1482,13 @@ int maat_scan_string(struct maat *maat_instance, int table_id, const char *data, } if (physical_table_id < 0) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } @@ -1485,10 +1498,11 @@ int maat_scan_string(struct maat *maat_instance, int table_id, const char *data, int hit_group_cnt = string_scan(maat_instance->tbl_mgr, state->thread_id, data, data_len, physical_table_id, vtable_id, state); if (hit_group_cnt < 0) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } - size_t sum_hit_compile_cnt = 0; + size_t sum_hit_compile_cnt = 0; if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { sum_hit_compile_cnt = group_to_compile(maat_instance, results, n_result, state); *n_hit_result = sum_hit_compile_cnt; @@ -1607,6 +1621,7 @@ int maat_stream_scan(struct maat_stream *maat_stream, const char *data, int data int hit_group_cnt = expr_stream_scan(maat_stream, data, data_len, state); if (hit_group_cnt < 0) { + maat_instance->stat->scan_err_cnt++; return MAAT_SCAN_ERR; } @@ -1669,6 +1684,8 @@ struct maat_state *maat_state_new(struct maat *maat_instance, int thread_id) state->thread_id = thread_id; state->compile_state = maat_compile_state_new(thread_id); + alignment_int64_array_add(maat_instance->stat->maat_state_cnt, thread_id, 1); + return state; } @@ -1694,6 +1711,11 @@ void maat_state_free(struct maat_state *state) return; } + assert(state->maat_instance != NULL); + + struct maat *maat_instance = state->maat_instance; + long long thread_id = state->thread_id; + if (state->compile_state != NULL) { maat_compile_state_free(state->compile_state); state->compile_state = NULL; @@ -1701,6 +1723,8 @@ void maat_state_free(struct maat_state *state) state->maat_instance = NULL; FREE(state); + + alignment_int64_array_add(maat_instance->stat->maat_state_cnt, thread_id, -1); } int maat_state_set_scan_district(struct maat_state *state, int vtable_id, diff --git a/src/maat_expr.c b/src/maat_expr.c index e07ef78..d405502 100644 --- a/src/maat_expr.c +++ b/src/maat_expr.c @@ -797,11 +797,9 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, return -1; } - struct maat_item_inner *inner_item = maat_item_inner_new(expr_item->item_id, - expr_item->group_id, - expr_item->district_id); - expr_item->user_data = inner_item; - expr_item->user_data_free = maat_item_inner_free; + struct item_district *item_dist = item_district_new(expr_item->district_id); + expr_item->user_data = item_dist; + expr_item->user_data_free = item_district_free; } int ret = expr_runtime_update_row(expr_rt, (char *)&item_id, sizeof(long long), @@ -848,6 +846,7 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, int ret = 0; size_t i = 0; size_t real_rule_cnt = 0; + size_t real_regex_rule_cnt = 0; struct expr_rule *rules = NULL; void **ex_data_array = NULL; @@ -856,14 +855,17 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, rules = ALLOC(struct expr_rule, rule_cnt); for (i = 0; i < rule_cnt; i++) { struct expr_item *expr_item = (struct expr_item *)ex_data_array[i]; - struct expr_rule tmp_rule; + struct expr_rule tmp_rule = {0}; ret = expr_item_to_expr_rule(expr_item, &tmp_rule, expr_rt->logger); if (ret < 0) { continue; } - rules[i] = tmp_rule; - real_rule_cnt++; + rules[real_rule_cnt++] = tmp_rule; + + if (expr_item->expr_type == EXPR_TYPE_REGEX) { + real_regex_rule_cnt++; + } } } @@ -871,12 +873,12 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, struct adapter_hs *old_adapter_hs = NULL; if (rule_cnt > 0) { - new_adapter_hs = adapter_hs_new(expr_rt->n_worker_thread, rules, rule_cnt, + new_adapter_hs = adapter_hs_new(expr_rt->n_worker_thread, rules, real_rule_cnt, expr_rt->logger); if (NULL == new_adapter_hs) { log_error(expr_rt->logger, MODULE_EXPR, "[%s:%d] table[%s] rebuild adapter_hs engine failed when update" - " %zu expr rules", __FUNCTION__, __LINE__, table_name, rule_cnt); + " %zu expr rules", __FUNCTION__, __LINE__, table_name, real_rule_cnt); ret = -1; } } @@ -890,13 +892,14 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, garbage_adapter_hs_free); } - expr_rt->rule_num = rule_cnt; + expr_rt->rule_num = real_rule_cnt; + expr_rt->regex_rule_num = real_regex_rule_cnt; expr_rt->version = maat_rt_version; log_info(expr_rt->logger, MODULE_EXPR, - "table[%s] has %zu rules, commit %zu expr rules and rebuild adapter_hs" - " completed, version:%lld", table_name, rule_cnt, real_rule_cnt, - expr_rt->version); + "table[%s] has %zu rules, commit %zu expr rules(regex rules:%zu) " + "and rebuild adapter_hs completed, version:%lld", table_name, rule_cnt, + real_rule_cnt, real_regex_rule_cnt, expr_rt->version); if (rules != NULL) { for (i = 0; i < rule_cnt; i++) { @@ -973,15 +976,15 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, } struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM]; - struct maat_item_inner *inner_item = NULL; + struct item_district *item_dist = NULL; struct expr_item *expr_item = NULL; size_t real_hit_item_cnt = 0; long long district_id = state->district_id; for (size_t i = 0; i < n_hit_item; i++) { - inner_item = (struct maat_item_inner *)(hit_results[i].user_tag); - if (inner_item->district_id == district_id || - inner_item->district_id == DISTRICT_ANY) { + item_dist = (struct item_district *)(hit_results[i].user_tag); + long long tag_district_id = item_district_id(item_dist); + if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) { long long item_id = hit_results[i].rule_id; expr_item = (struct expr_item *)rcu_hash_find(expr_rt->item_hash, (char *)&item_id, diff --git a/src/maat_flag.c b/src/maat_flag.c index 55d754a..2d96139 100644 --- a/src/maat_flag.c +++ b/src/maat_flag.c @@ -425,11 +425,9 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table return -1; } - struct maat_item_inner *inner_item = maat_item_inner_new(flag_item->item_id, - flag_item->group_id, - flag_item->district_id); - flag_item->user_data = inner_item; - flag_item->user_data_free = maat_item_inner_free; + struct item_district *item_dist = item_district_new(flag_item->district_id); + flag_item->user_data = item_dist; + flag_item->user_data_free = item_district_free; } int ret = flag_runtime_update_row(flag_rt, (char *)&item_id, sizeof(long long), @@ -558,15 +556,15 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, } struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM]; - struct maat_item_inner *inner_item = NULL; + struct item_district *item_dist = NULL; struct flag_item *flag_item = NULL; size_t real_hit_item_cnt = 0; long long district_id = state->district_id; for (int i = 0; i < n_hit_item; i++) { - inner_item = (struct maat_item_inner *)(hit_results[i].user_tag); - if (inner_item->district_id == district_id || - inner_item->district_id == DISTRICT_ANY) { + item_dist = (struct item_district *)(hit_results[i].user_tag); + long long tag_district_id = item_district_id(item_dist); + if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) { long long item_id = hit_results[i].rule_id; flag_item = (struct flag_item *)rcu_hash_find(flag_rt->item_hash, (char *)&item_id, diff --git a/src/maat_group.c b/src/maat_group.c index b9a2922..033eba1 100644 --- a/src/maat_group.c +++ b/src/maat_group.c @@ -64,6 +64,7 @@ struct group2group_runtime { struct maat_group_topology *updating_group_topo; long long version; long long rule_num; + long long excl_rule_num; //exclude g2g rule num long long update_err_cnt; int updating_flag; @@ -714,6 +715,9 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, g2g_item->is_exclude); if (0 == ret) { g2g_rt->rule_num--; + if (1 == g2g_item->is_exclude) { + g2g_rt->excl_rule_num--; + } } else { g2g_rt->update_err_cnt++; } @@ -725,6 +729,9 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, g2g_item->is_exclude); if (0 == ret) { g2g_rt->rule_num++; + if (1 == g2g_item->is_exclude) { + g2g_rt->excl_rule_num++; + } } else { g2g_rt->update_err_cnt++; } @@ -1088,6 +1095,16 @@ long long group2group_runtime_rule_count(void *g2g_runtime) return g2g_rt->rule_num; } +long long group2group_runtime_exclude_rule_count(void *g2g_runtime) +{ + if (NULL == g2g_runtime) { + return 0; + } + + struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + return g2g_rt->excl_rule_num; +} + long long group2group_runtime_update_err_count(void *g2g_runtime) { if (NULL == g2g_runtime) { diff --git a/src/maat_interval.c b/src/maat_interval.c index 551eafe..af62846 100644 --- a/src/maat_interval.c +++ b/src/maat_interval.c @@ -429,11 +429,9 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema, return -1; } - struct maat_item_inner *inner_item = maat_item_inner_new(interval_item->item_id, - interval_item->group_id, - interval_item->district_id); - interval_item->user_data = inner_item; - interval_item->user_data_free = maat_item_inner_free; + struct item_district *item_dist = item_district_new(interval_item->district_id); + interval_item->user_data = item_dist; + interval_item->user_data_free = item_district_free; } int ret = interval_runtime_update_row(interval_rt, (char *)&item_id, sizeof(long long), @@ -562,15 +560,15 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, } struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM]; - struct maat_item_inner *inner_item = NULL; + struct item_district *item_dist = NULL; struct interval_item *int_item = NULL; size_t real_hit_item_cnt = 0; long long district_id = state->district_id; for (int i = 0; i < n_hit_item; i++) { - inner_item = (struct maat_item_inner *)(hit_results[i].user_tag); - if (inner_item->district_id == district_id || - inner_item->district_id == DISTRICT_ANY) { + item_dist = (struct item_district *)(hit_results[i].user_tag); + long long tag_district_id = item_district_id(item_dist); + if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) { long long item_id = hit_results[i].rule_id; int_item = (struct interval_item *)rcu_hash_find(interval_rt->item_hash, (char *)&item_id, diff --git a/src/maat_ip.c b/src/maat_ip.c index 871881e..a3e468e 100644 --- a/src/maat_ip.c +++ b/src/maat_ip.c @@ -62,8 +62,6 @@ struct ip_item { uint16_t min_port; uint16_t max_port; int proto; - void *user_data; - void (*user_data_free)(void *data); }; struct ip_runtime { @@ -386,12 +384,7 @@ void ip_item_free(struct ip_item *item) if (NULL == item) { return; } - - if (item->user_data != NULL && item->user_data_free != NULL) { - item->user_data_free(item->user_data); - item->user_data = NULL; - } - + FREE(item); } @@ -547,12 +540,6 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema, ip_rt->update_err_cnt++; return -1; } - - struct maat_item_inner *inner_item = maat_item_inner_new(ip_item->item_id, - ip_item->group_id, - 0); - ip_item->user_data = inner_item; - ip_item->user_data_free = maat_item_inner_free; } int ret = ip_runtime_update_row(ip_rt, (char *)&item_id, sizeof(long long), diff --git a/src/maat_rule.c b/src/maat_rule.c index a751888..3777af0 100644 --- a/src/maat_rule.c +++ b/src/maat_rule.c @@ -32,30 +32,38 @@ #define MODULE_MAAT_RULE module_name_str("maat.rule") -#define ITEM_RULE_MAGIC 0x4d3c2b1a -struct maat_item_inner *maat_item_inner_new(long long item_id, long long group_id, - long long district_id) -{ - struct maat_item_inner *item = ALLOC(struct maat_item_inner, 1); - item->magic_num = ITEM_RULE_MAGIC; - item->item_id = item_id; - item->group_id = group_id; - item->district_id = district_id; +struct item_district { + long long magic_num; + long long district_id; +}; - return item; +#define ITEM_RULE_MAGIC 0x4d3c2b1a +struct item_district * +item_district_new(long long district_id) +{ + struct item_district *item_dist = ALLOC(struct item_district, 1); + item_dist->magic_num = ITEM_RULE_MAGIC; + item_dist->district_id = district_id; + + return item_dist; } -void maat_item_inner_free(void *item_inner) +void item_district_free(void *item_district) { - if (NULL == item_inner) { + if (NULL == item_district) { return; } - struct maat_item_inner *item = (struct maat_item_inner *)item_inner; - assert(item->magic_num == ITEM_RULE_MAGIC); - item->magic_num = 0; + struct item_district *item_dist = (struct item_district *)item_district; + assert(item_dist->magic_num == ITEM_RULE_MAGIC); + item_dist->magic_num = 0; - FREE(item); + FREE(item_dist); +} + +long long item_district_id(struct item_district *item_dist) +{ + return item_dist->district_id; } struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance) @@ -63,7 +71,8 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1); maat_rt->version = version; - int ret = table_manager_runtime_create(maat_instance->tbl_mgr, maat_instance->opts.nr_worker_thread, + int ret = table_manager_runtime_create(maat_instance->tbl_mgr, + maat_instance->opts.nr_worker_thread, maat_instance->garbage_bin); if (ret < 0) { FREE(maat_rt); @@ -376,7 +385,11 @@ void *rule_monitor_loop(void *arg) char err_str[NAME_MAX] = {0}; struct stat attrib; while (maat_instance->is_running) { - log_info(maat_instance->logger, MODULE_MAAT_RULE, "%s:%d.............", __FUNCTION__, __LINE__); + if (time(NULL) % 10 == 0) { + log_info(maat_instance->logger, MODULE_MAAT_RULE, + "%s thread still alive.........", __FUNCTION__); + } + usleep(maat_instance->opts.rule_update_checking_interval_ms * 1000); if (0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) { switch (maat_instance->opts.input_mode) { diff --git a/src/maat_stat.c b/src/maat_stat.c index cb12318..7e2956a 100644 --- a/src/maat_stat.c +++ b/src/maat_stat.c @@ -30,41 +30,46 @@ enum MAAT_FS_STATUS { STATUS_PLUGIN_ACC_NUM, STATUS_GROUP_REF_NUM, STATUS_GROUP_REF_NOT_NUM, + STATUS_GROUP_REF_EXCL_NUM, //group reference exclude group num STATUS_COMPILE_RULE_NUM, STATUS_MAAT_STATE_NUM, STATUS_GARBAGE_QSIZE, - STATUS_TOTAL_SCAN_LEN, + STATUS_TOTAL_SCAN_BYTES, STATUS_TOTAL_SCAN_CNT, STATUS_UPDATE_ERR_CNT, STATUS_ICONV_ERR_CNT, STATUS_SCAN_ERR_CNT, STATUS_ZOMBIE_RS_STREAM, STATUS_NOT_GROUP_HIT, - STATUS_CMD_NUM, - STATUS_CMD_Q_SIZE, STATUS_CMD_LINE_NUM }; enum MAAT_FS_COLUMN { - COLUMN_TABLE_RULE_NUM = 0, - COLUMN_TABLE_REGEX_NUM, - COLUMN_TABLE_IPV6_NUM, - COLUMN_TABLE_STREAM_NUM, - COLUMN_TABLE_SCAN_CNT, - COLUMN_TABLE_SCAN_BYTES, - COLUMN_TABLE_CPU_TIME,//microseconds - COLUMN_TABLE_HIT_CNT, + COLUMN_RULE_NUM = 0, + COLUMN_REGEX_NUM, + COLUMN_STREAM_NUM, + COLUMN_SCAN_CNT, + COLUMN_SCAN_BYTES, + COLUMN_CPU_TIME, //microseconds + COLUMN_HIT_CNT, + COLUMN_SCAN_BPS, //scan bytes per second + COLUMN_SCAN_TPS, //scan times per second + COLUMN_HIT_RATE //scan hit rate(hit_cnt/scan_cnt) }; -const char *common_column_name[] = {"rule", "reg", "v6", "stream", "scan_cnt", "scan_bytes", "scan_cpu_time", "hit_cnt"}; +const char *common_column_name[] = {"rule", "reg/v6", "stream", "scan_cnt", "scan_bytes", + "scan_cpu_time", "hit_cnt", "IN_Bps", "IN_Tps", "hit_rate"}; + enum field_type common_column_type[] = { - FIELD_TYPE_GAUGE, - FIELD_TYPE_GAUGE, - FIELD_TYPE_GAUGE, - FIELD_TYPE_GAUGE, - FIELD_TYPE_GAUGE, - FIELD_TYPE_GAUGE, - FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, FIELD_TYPE_GAUGE }; @@ -85,6 +90,8 @@ void maat_fieldstat_register(struct maat_stat *stat) "group", NULL, 0); stat->fs_status_id[STATUS_GROUP_REF_NOT_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, "not_grp", NULL, 0); + stat->fs_status_id[STATUS_GROUP_REF_EXCL_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "excl_grp", NULL, 0); stat->fs_status_id[STATUS_COMPILE_RULE_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, "compile", NULL, 0); stat->fs_status_id[STATUS_GARBAGE_QSIZE] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, @@ -95,18 +102,14 @@ void maat_fieldstat_register(struct maat_stat *stat) "z_stream", NULL, 0); stat->fs_status_id[STATUS_NOT_GROUP_HIT] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, "nt_grp_hit", NULL, 0); - stat->fs_status_id[STATUS_TOTAL_SCAN_LEN] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + stat->fs_status_id[STATUS_TOTAL_SCAN_BYTES] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, "scan_bytes", NULL, 0); stat->fs_status_id[STATUS_TOTAL_SCAN_CNT] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, "scan_times", NULL, 0); stat->fs_status_id[STATUS_UPDATE_ERR_CNT] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, - "update_err", NULL, 0); + "update_error", NULL, 0); stat->fs_status_id[STATUS_SCAN_ERR_CNT] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, "scan_error", NULL, 0); - stat->fs_status_id[STATUS_CMD_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, - "cmd_commit", NULL, 0); - stat->fs_status_id[STATUS_CMD_Q_SIZE] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, - "cmd_in_q", NULL, 0); stat->fs_status_id[STATUS_CMD_LINE_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, "line_cmd/s", NULL, 0); } @@ -122,6 +125,30 @@ int maat_fieldstat_table_row_register(struct maat_stat *stat, struct table_manag return -1; } + ret = fieldstat_set_metric_ratio_para(stat->fs_handle, stat->total_stat_id[COLUMN_SCAN_BPS], + stat->total_stat_id[COLUMN_SCAN_BYTES], + stat->total_stat_id[COLUMN_CPU_TIME], 1000000000); + if (ret < 0) { + log_error(stat->logger, MODULE_MAAT_STAT, "fieldstat set IN_Bps for Sum failed."); + return -1; + } + + ret = fieldstat_set_metric_ratio_para(stat->fs_handle, stat->total_stat_id[COLUMN_SCAN_TPS], + stat->total_stat_id[COLUMN_SCAN_CNT], + stat->total_stat_id[COLUMN_CPU_TIME], 1000000000); + if (ret < 0) { + log_error(stat->logger, MODULE_MAAT_STAT, "fieldstat set IN_Tps for Sum failed."); + return -1; + } + + ret = fieldstat_set_metric_ratio_para(stat->fs_handle, stat->total_stat_id[COLUMN_HIT_RATE], + stat->total_stat_id[COLUMN_HIT_CNT], + stat->total_stat_id[COLUMN_SCAN_CNT], 1000000000); + if (ret < 0) { + log_error(stat->logger, MODULE_MAAT_STAT, "fieldstat set hit_rate for Sum failed."); + return -1; + } + for (size_t i = 0; i < max_table_cnt; i++) { void *schema = table_manager_get_schema(tbl_mgr, i); if (NULL == schema) { @@ -138,12 +165,40 @@ int maat_fieldstat_table_row_register(struct maat_stat *stat, struct table_manag "fieldstat_register_table_row %s failed.", table_name); return -1; } + + ret = fieldstat_set_metric_ratio_para(stat->fs_handle, stat->fs_column_id[i][COLUMN_SCAN_BPS], + stat->fs_column_id[i][COLUMN_SCAN_BYTES], + stat->fs_column_id[i][COLUMN_CPU_TIME], 1000000000); + if (ret < 0) { + log_error(stat->logger, MODULE_MAAT_STAT, + "fieldstat set IN_Bps for %s failed.", table_name); + return -1; + } + + ret = fieldstat_set_metric_ratio_para(stat->fs_handle, stat->fs_column_id[i][COLUMN_SCAN_TPS], + stat->fs_column_id[i][COLUMN_SCAN_CNT], + stat->fs_column_id[i][COLUMN_CPU_TIME], 1000000000); + if (ret < 0) { + log_error(stat->logger, MODULE_MAAT_STAT, + "fieldstat set IN_Tps for %s failed.", table_name); + return -1; + } + + ret = fieldstat_set_metric_ratio_para(stat->fs_handle, stat->fs_column_id[i][COLUMN_HIT_RATE], + stat->fs_column_id[i][COLUMN_HIT_CNT], + stat->fs_column_id[i][COLUMN_SCAN_CNT], 1000000000); + if (ret < 0) { + log_error(stat->logger, MODULE_MAAT_STAT, + "fieldstat set hit_rate for %s failed.", table_name); + return -1; + } } return 0; } -struct maat_stat *maat_stat_new(const char *stat_file, size_t max_thread_num, struct log_handle *logger) +struct maat_stat *maat_stat_new(const char *stat_file, size_t max_thread_num, + struct log_handle *logger) { struct maat_stat *stat = ALLOC(struct maat_stat, 1); @@ -155,6 +210,7 @@ struct maat_stat *maat_stat_new(const char *stat_file, size_t max_thread_num, st stat->thread_call_cnt = alignment_int64_array_alloc(max_thread_num); stat->hit_cnt = alignment_int64_array_alloc(max_thread_num); stat->not_grp_hit_cnt = alignment_int64_array_alloc(max_thread_num); + stat->maat_state_cnt = alignment_int64_array_alloc(max_thread_num); return stat; } @@ -180,6 +236,11 @@ void maat_stat_free(struct maat_stat *stat) stat->not_grp_hit_cnt = NULL; } + if (stat->maat_state_cnt != NULL) { + alignment_int64_array_free(stat->maat_state_cnt); + stat->maat_state_cnt = NULL; + } + if (stat->fs_handle != NULL) { fieldstat_instance_free(stat->fs_handle); stat->fs_handle = NULL; @@ -210,7 +271,8 @@ int maat_stat_init(struct maat_stat *stat, struct table_manager *tbl_mgr, ret = fieldstat_disable_background_thread(stat->fs_handle); if (ret < 0) { - log_error(stat->logger, MODULE_MAAT_STAT, "fieldstat_disable_background_thread failed."); + log_error(stat->logger, MODULE_MAAT_STAT, + "fieldstat_disable_background_thread failed."); return -1; } @@ -239,11 +301,12 @@ int maat_stat_init(struct maat_stat *stat, struct table_manager *tbl_mgr, void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on) { - long long plugin_cache_num = 0, plugin_rule_num = 0, not_group_num = 0; + long long plugin_cache_num = 0, plugin_rule_num = 0, g2c_not_group_num = 0; long long compile_rule_num = 0, g2c_rule_num = 0, g2g_rule_num = 0; long long total_rule_num = 0, total_stream_num = 0, total_input_bytes = 0; long long total_scan_cnt = 0, total_scan_cpu_time = 0, total_regex_num = 0; long long total_ipv6_num = 0, total_hit_cnt = 0, total_update_err = 0; + long long g2g_excl_rule_num = 0; size_t max_table_count = table_manager_table_size(stat->ref_tbl_mgr); for (size_t i = 0; i < max_table_count; i++) { @@ -269,15 +332,14 @@ void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on) break; case TABLE_TYPE_GROUP2COMPILE: g2c_rule_num += group2compile_runtime_rule_count(runtime); - not_group_num += group2compile_runtime_not_group_count(runtime); + g2c_not_group_num += group2compile_runtime_not_group_count(runtime); break; case TABLE_TYPE_GROUP2GROUP: g2g_rule_num += group2group_runtime_rule_count(runtime); - //TODO exclude group num + g2g_excl_rule_num += group2group_runtime_exclude_rule_count(runtime); break; case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - //regex rule cnt regex_rule_num = expr_runtime_regex_rule_count(runtime); break; case TABLE_TYPE_IP_PLUS: @@ -287,7 +349,7 @@ void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on) break; } - if (table_type == TABLE_TYPE_PLUGIN ) { + if (table_type == TABLE_TYPE_PLUGIN) { continue; } @@ -295,66 +357,79 @@ void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on) total_ipv6_num += ipv6_rule_num; long long rule_num = table_manager_runtime_rule_count(stat->ref_tbl_mgr, i); - fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_RULE_NUM], rule_num); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_RULE_NUM], rule_num); total_rule_num += rule_num; if (table_type == TABLE_TYPE_EXPR || table_type == TABLE_TYPE_EXPR_PLUS) { - fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_REGEX_NUM], regex_rule_num); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_REGEX_NUM], regex_rule_num); long long stream_num = expr_runtime_stream_num(runtime); - fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_STREAM_NUM], stream_num); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_STREAM_NUM], stream_num); total_stream_num += stream_num; long long input_bytes = expr_runtime_scan_bytes(runtime); - fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_SCAN_BYTES], input_bytes); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_SCAN_BYTES], input_bytes); total_input_bytes += input_bytes; } if (table_type == TABLE_TYPE_IP_PLUS) { - fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_IPV6_NUM], ipv6_rule_num); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_REGEX_NUM], ipv6_rule_num); } if (1 == perf_on) { long long scan_cpu_time = table_manager_runtime_scan_cpu_time(stat->ref_tbl_mgr, i); - fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_CPU_TIME], scan_cpu_time); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_CPU_TIME], scan_cpu_time); total_scan_cpu_time += scan_cpu_time; } long long scan_cnt = table_manager_runtime_scan_count(stat->ref_tbl_mgr, i); - fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_SCAN_CNT], scan_cnt); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_SCAN_CNT], scan_cnt); total_scan_cnt += scan_cnt; long long hit_cnt = table_manager_runtime_hit_count(stat->ref_tbl_mgr, i); - fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_HIT_CNT], hit_cnt); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_HIT_CNT], hit_cnt); total_hit_cnt += hit_cnt; total_update_err += table_manager_runtime_update_err_count(stat->ref_tbl_mgr, i); } - fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_RULE_NUM], total_rule_num); - fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_REGEX_NUM], total_regex_num); - fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_IPV6_NUM], total_ipv6_num); - fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_STREAM_NUM], total_stream_num); - fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_SCAN_CNT], total_scan_cnt); - fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_SCAN_BYTES], total_input_bytes); - fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_HIT_CNT], total_hit_cnt); + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_RULE_NUM], + total_rule_num); + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_REGEX_NUM], + total_regex_num); + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_STREAM_NUM], + total_stream_num); + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_SCAN_CNT], + total_scan_cnt); + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_SCAN_BYTES], + total_input_bytes); + if (1 == perf_on) { - fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_CPU_TIME], total_scan_cpu_time); + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_CPU_TIME], + total_scan_cpu_time); } - stat->scan_cnt += total_scan_cnt; - stat->scan_bytes += total_input_bytes; - stat->update_err_cnt += total_update_err; - - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TOTAL_SCAN_LEN], stat->scan_bytes); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TOTAL_SCAN_CNT], stat->scan_cnt); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_UPDATE_ERR_CNT], stat->update_err_cnt); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_SCAN_ERR_CNT], stat->scan_err_cnt); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_ZOMBIE_RS_STREAM], stat->zombie_rs_stream); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_PLUGIN_CACHE_NUM], plugin_cache_num); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_PLUGIN_ACC_NUM], plugin_rule_num); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_NUM], g2c_rule_num); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_NOT_NUM], not_group_num); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_COMPILE_RULE_NUM], compile_rule_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TOTAL_SCAN_BYTES], + total_input_bytes); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TOTAL_SCAN_CNT], + total_scan_cnt); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_UPDATE_ERR_CNT], + total_update_err); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_SCAN_ERR_CNT], + stat->scan_err_cnt); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_ZOMBIE_RS_STREAM], + stat->zombie_rs_stream); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_PLUGIN_CACHE_NUM], + plugin_cache_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_PLUGIN_ACC_NUM], + plugin_rule_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_NUM], + g2c_rule_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_NOT_NUM], + g2c_not_group_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_EXCL_NUM], + g2g_excl_rule_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_COMPILE_RULE_NUM], + compile_rule_num); } void maat_stat_output(struct maat_stat *stat, long long maat_version, int perf_on) @@ -365,19 +440,27 @@ void maat_stat_output(struct maat_stat *stat, long long maat_version, int perf_o long long active_thread_num = alignment_int64_array_cnt(stat->thread_call_cnt, stat->nr_worker_thread); - //TODO: maat_state_cnt + long long maat_state_cnt = alignment_int64_array_sum(stat->maat_state_cnt, + stat->nr_worker_thread); long long not_grp_hit_cnt = alignment_int64_array_sum(stat->not_grp_hit_cnt, stat->nr_worker_thread); size_t table_cnt = table_manager_table_count(stat->ref_tbl_mgr); size_t garbage_q_len = maat_garbage_bin_get_size(stat->ref_garbage_bin); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_VERSION], maat_version); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_THRED_NUM], active_thread_num); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TABLE_NUM], table_cnt); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_NOT_GROUP_HIT], not_grp_hit_cnt); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_CMD_Q_SIZE], stat->cmd_q_cnt); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_CMD_LINE_NUM], stat->line_cmd_acc_num); - fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GARBAGE_QSIZE], garbage_q_len); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_VERSION], + maat_version); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_THRED_NUM], + active_thread_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TABLE_NUM], + table_cnt); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_MAAT_STATE_NUM], + maat_state_cnt); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_NOT_GROUP_HIT], + not_grp_hit_cnt); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_CMD_LINE_NUM], + stat->line_cmd_acc_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GARBAGE_QSIZE], + garbage_q_len); maat_fieldstat_table_row_output(stat, perf_on); diff --git a/test/maat_framework_perf_gtest.cpp b/test/maat_framework_perf_gtest.cpp index 36cfaf0..90b959a 100644 --- a/test/maat_framework_perf_gtest.cpp +++ b/test/maat_framework_perf_gtest.cpp @@ -308,6 +308,8 @@ protected: } struct maat_options *opts = maat_options_new(); + maat_options_set_stat_file(opts, "./stat.log"); + maat_options_set_perf_on(opts); maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); @@ -522,6 +524,8 @@ protected: } struct maat_options *opts = maat_options_new(); + maat_options_set_stat_file(opts, "./stat.log"); + maat_options_set_perf_on(opts); maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); @@ -674,6 +678,8 @@ protected: } struct maat_options *opts = maat_options_new(); + maat_options_set_stat_file(opts, "./stat.log"); + maat_options_set_perf_on(opts); maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); @@ -766,6 +772,8 @@ protected: } struct maat_options *opts = maat_options_new(); + maat_options_set_stat_file(opts, "./stat.log"); + maat_options_set_perf_on(opts); maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); @@ -994,6 +1002,8 @@ protected: } struct maat_options *opts = maat_options_new(); + maat_options_set_stat_file(opts, "./stat.log"); + maat_options_set_perf_on(opts); maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); @@ -1191,6 +1201,8 @@ protected: const char *table_info = "./tsg_table_info.conf"; struct maat_options *opts = maat_options_new(); + maat_options_set_stat_file(opts, "./stat.log"); + maat_options_set_perf_on(opts); maat_options_set_iris(opts, rule_folder, rule_folder); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_caller_thread_number(opts, 5);