From af6df5951a5726f9a49ba24d3f09336c5a60d488 Mon Sep 17 00:00:00 2001 From: liuwentan Date: Thu, 20 Apr 2023 15:34:56 +0800 Subject: [PATCH] support maat stat --- CMakeLists.txt | 2 +- include/maat.h | 28 +-- scanner/adapter_hs/adapter_hs.cpp | 3 - src/CMakeLists.txt | 2 +- src/inc_internal/maat_bool_plugin.h | 14 +- src/inc_internal/maat_common.h | 3 + src/inc_internal/maat_compile.h | 10 +- src/inc_internal/maat_expr.h | 27 +- src/inc_internal/maat_flag.h | 16 +- src/inc_internal/maat_fqdn_plugin.h | 14 +- src/inc_internal/maat_group.h | 4 +- src/inc_internal/maat_interval.h | 16 +- src/inc_internal/maat_ip.h | 18 +- src/inc_internal/maat_ip_plugin.h | 11 +- src/inc_internal/maat_limits.h | 4 +- src/inc_internal/maat_plugin.h | 4 +- src/inc_internal/maat_rule.h | 42 +++- src/inc_internal/maat_stat.h | 35 +++ src/inc_internal/maat_table.h | 15 +- src/maat_api.c | 308 +++++++++++++++-------- src/maat_bool_plugin.c | 85 ++++++- src/maat_command.c | 2 +- src/maat_compile.c | 59 ++++- src/maat_expr.c | 159 +++++++++++- src/maat_flag.c | 93 ++++++- src/maat_fqdn_plugin.c | 82 +++++- src/maat_group.c | 20 +- src/maat_interval.c | 91 ++++++- src/maat_ip.c | 106 +++++++- src/maat_ip_plugin.c | 83 ++++++- src/maat_plugin.c | 24 +- src/maat_rule.c | 11 +- src/maat_stat.c | 373 ++++++++++++++++++++++++++++ src/maat_table.c | 214 +++++++++++++--- test/maat_framework_gtest.cpp | 37 +-- 35 files changed, 1758 insertions(+), 257 deletions(-) create mode 100644 src/inc_internal/maat_stat.h create mode 100644 src/maat_stat.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 48d5185..c19a847 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ include_directories(/opt/MESA/include/) set(CMAKE_C_STANDARD 11) set(CMAKE_C_FLAGS "-fPIC -Wall") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall") -set(MAAT_DEPEND_DYN_LIB pthread m crypto z) +set(MAAT_DEPEND_DYN_LIB pthread m crypto z fieldstat3) include_directories(include) #for ASAN diff --git a/include/maat.h b/include/maat.h index 9b0c935..0ba7da5 100644 --- a/include/maat.h +++ b/include/maat.h @@ -62,22 +62,6 @@ struct ip_addr { }; }; -struct ipv4_tuple { - unsigned int sip; /* network order */ - unsigned int dip; /* network order */ - unsigned short sport; /* network order */ - unsigned short dport; /* network order */ - int protocol; -}; - -struct ipv6_tuple { - unsigned int sip[4] ; /* network order */ - unsigned int dip[4] ; /* network order */ - unsigned short sport; /* network order */ - unsigned short dport; /* network order */ - int protocol; -}; - enum log_level { LOG_LEVEL_TRACE, LOG_LEVEL_DEBUG, @@ -119,6 +103,8 @@ int maat_options_set_json_file(struct maat_options *opts, const char *json_filen int maat_options_set_redis(struct maat_options *opts, const char *redis_ip, uint16_t redis_port, int redis_db); +int maat_options_set_stat_file(struct maat_options *opts, const char *stat_filename); + /* maat_instance API */ struct maat *maat_new(struct maat_options *opts, const char *table_info_path); void maat_free(struct maat *instance); @@ -192,21 +178,11 @@ int maat_scan_ipv4(struct maat *instance, int table_id, long long *results, size_t n_result, size_t *n_hit_result, struct maat_state *state); -int maat_scan_ipv4_tuple4(struct maat *instance, int table_id, - const struct ipv4_tuple *tuple, long long *results, - size_t n_result, size_t *n_hit_result, - struct maat_state *state); - int maat_scan_ipv6(struct maat *instance, int table_id, uint8_t *ip_addr, uint16_t port, int protocol, long long *results, size_t n_result, size_t *n_hit_result, struct maat_state *state); -int maat_scan_ipv6_tuple4(struct maat *instance, int table_id, - const struct ipv6_tuple *tuple, long long *results, - size_t n_result, size_t *n_hit_result, - struct maat_state *state); - int maat_scan_string(struct maat *instance, int table_id, const char *data, size_t data_len, long long *results, size_t n_result, size_t *n_hit_result, diff --git a/scanner/adapter_hs/adapter_hs.cpp b/scanner/adapter_hs/adapter_hs.cpp index b458fef..78c6e03 100644 --- a/scanner/adapter_hs/adapter_hs.cpp +++ b/scanner/adapter_hs/adapter_hs.cpp @@ -121,9 +121,6 @@ int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratches, size_t n_work log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] Unable to query scratch size", __FUNCTION__, __LINE__); return -1; - } else { - log_info(logger, MODULE_ADAPTER_HS, - "[%s:%d] scratch[%d] size:%zu", __FUNCTION__, __LINE__, i, scratch_size); } } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 13dd187..3e51362 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,7 +14,7 @@ add_definitions(-fPIC) set(MAAT_SRC alignment.c json2iris.c maat_api.c rcu_hash.c maat_garbage_collection.c maat_config_monitor.c maat_rule.c maat_kv.c maat_ex_data.c maat_utils.c maat_command.c maat_redis_monitor.c maat_table.c maat_compile.c maat_group.c maat_ip.c maat_flag.c maat_interval.c maat_expr.c maat_plugin.c - maat_ip_plugin.c maat_bool_plugin.c maat_fqdn_plugin.c maat_virtual.c) + maat_ip_plugin.c maat_bool_plugin.c maat_fqdn_plugin.c maat_virtual.c maat_stat.c) set(LIB_SOURCE_FILES ${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c ${PROJECT_SOURCE_DIR}/deps/log/log.c) diff --git a/src/inc_internal/maat_bool_plugin.h b/src/inc_internal/maat_bool_plugin.h index aa360a6..6030e21 100644 --- a/src/inc_internal/maat_bool_plugin.h +++ b/src/inc_internal/maat_bool_plugin.h @@ -37,7 +37,7 @@ int bool_plugin_table_set_ex_container_schema(void *bool_plugin_schema, int tabl struct ex_container_schema *bool_plugin_table_get_ex_container_schema(void *bool_plugin_schema); /* ip plugin runtime API */ -void *bool_plugin_runtime_new(void *bool_plugin_schema, int max_thread_num, +void *bool_plugin_runtime_new(void *bool_plugin_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void bool_plugin_runtime_free(void *bool_plugin_runtime); @@ -45,7 +45,8 @@ void bool_plugin_runtime_free(void *bool_plugin_runtime); int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_schema, const char *table_name, const char *line, int valid_column); -int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name, long long maat_rt_version); +int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name, + long long maat_rt_version); long long bool_plugin_runtime_rule_count(void *bool_plugin_runtime); @@ -54,6 +55,15 @@ struct ex_data_runtime *bool_plugin_runtime_get_ex_data_rt(void *bool_plugin_run int bool_plugin_runtime_get_ex_data(void *bool_plugin_runtime, unsigned long long *item_ids, size_t n_item, void **ex_data_array, size_t n_ex_data); +void bool_plugin_runtime_perf_stat(void *bool_plugin_runtime, struct timespec *start, + struct timespec *end, int thread_id); + +long long bool_plugin_runtime_scan_count(void *bool_plugin_runtime); + +long long bool_plugin_runtime_scan_cpu_time(void *bool_plugin_runtime); + +long long bool_plugin_runtime_update_err_count(void *bool_plugin_runtime); + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_common.h b/src/inc_internal/maat_common.h index 9128ed7..e3cac87 100644 --- a/src/inc_internal/maat_common.h +++ b/src/inc_internal/maat_common.h @@ -26,11 +26,14 @@ struct maat_options { char instance_name[NAME_MAX]; char foreign_cont_dir[NAME_MAX]; char log_path[PATH_MAX]; + char stat_file[PATH_MAX]; size_t nr_worker_threads; char *accept_tags; int rule_effect_interval_ms; int rule_update_checking_interval_ms; int gc_timeout_ms; + int stat_on; + int perf_on; int deferred_load_on; int log_level; enum data_source input_mode; diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h index 19ab918..39239c8 100644 --- a/src/inc_internal/maat_compile.h +++ b/src/inc_internal/maat_compile.h @@ -44,7 +44,7 @@ int compile_table_set_ex_data_schema(struct compile_schema *compile_schema, int long argl, void *argp); /* compile runtime API */ -void *compile_runtime_new(void *compile_schema, int max_thread_num, +void *compile_runtime_new(void *compile_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void compile_runtime_free(void *compile_runtime); @@ -58,6 +58,8 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name, long l long long compile_runtime_rule_count(void *compile_runtime); +long long compile_runtime_update_err_count(void *compile_runtime); + int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile_ids, size_t compile_ids_size, struct maat_state *state); @@ -73,7 +75,7 @@ void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt, struct compile_schema *compile_schema); /* group2compile runtime API */ -void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, +void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, void *g2g_runtime); @@ -82,8 +84,12 @@ void group2compile_runtime_free(void *g2c_runtime); int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *table_name, const char *line, int valid_column); +long long group2compile_runtime_not_group_count(void *g2c_runtime); + long long group2compile_runtime_rule_count(void *g2c_runtime); +long long group2compile_runtime_update_err_count(void *g2c_runtime); + /* maat compile state API */ struct maat_compile_state; struct maat_compile_state *maat_compile_state_new(int thread_id); diff --git a/src/inc_internal/maat_expr.h b/src/inc_internal/maat_expr.h index 1570c55..4463bde 100644 --- a/src/inc_internal/maat_expr.h +++ b/src/inc_internal/maat_expr.h @@ -27,7 +27,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr, void expr_schema_free(void *expr_schema); /* expr runtime API */ -void *expr_runtime_new(void *expr_schema, int max_thread_num, +void *expr_runtime_new(void *expr_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void expr_runtime_free(void *expr_runtime); @@ -39,6 +39,8 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, long long ma long long expr_runtime_rule_count(void *expr_runtime); +long long expr_runtime_regex_rule_count(void *expr_runtime); + long long expr_runtime_get_version(void *expr_runtime); /* expr runtime scan API */ @@ -55,13 +57,30 @@ struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_stream *s_handle, const char *data, size_t data_len, int vtable_id, struct maat_state *state); -void expr_runtime_stream_close(struct adapter_hs_stream *s_handle); + +void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id, + struct adapter_hs_stream *s_handle); int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *district, size_t district_len, long long *district_id); -void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id); -long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread); +void expr_runtime_hit_inc(struct expr_runtime *expr_rt, int thread_id); + +void expr_runtime_perf_stat(struct expr_runtime *flag_rt, size_t scan_len, + struct timespec *start, struct timespec *end, + int thread_id); + +long long expr_runtime_scan_count(void *expr_runtime); + +long long expr_runtime_scan_cpu_time(void *expr_runtime); + +long long expr_runtime_hit_count(void *expr_runtime); + +long long expr_runtime_update_err_count(void *expr_runtime); + +long long expr_runtime_scan_bytes(struct expr_runtime *expr_rt); + +long long expr_runtime_stream_num(struct expr_runtime *expr_rt); #ifdef __cplusplus } diff --git a/src/inc_internal/maat_flag.h b/src/inc_internal/maat_flag.h index c4a49de..66673d3 100644 --- a/src/inc_internal/maat_flag.h +++ b/src/inc_internal/maat_flag.h @@ -29,7 +29,7 @@ void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr, void flag_schema_free(void *flag_schema); /* flag runtime API */ -void *flag_runtime_new(void *flag_schema, int max_thread_num, +void *flag_runtime_new(void *flag_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void flag_runtime_free(void *flag_runtime); @@ -53,8 +53,18 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, long long fla int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *district, size_t district_len, long long *district_id); -void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id); -long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread); +void flag_runtime_hit_inc(struct flag_runtime *flag_rt, int thread_id); + +void flag_runtime_perf_stat(struct flag_runtime *flag_rt, struct timespec *start, + struct timespec *end, int thread_id); + +long long flag_runtime_scan_count(void *flag_runtime); + +long long flag_runtime_scan_cpu_time(void *flag_runtime); + +long long flag_runtime_hit_count(void *flag_runtime); + +long long flag_runtime_update_err_count(void *flag_runtime); #ifdef __cplusplus } diff --git a/src/inc_internal/maat_fqdn_plugin.h b/src/inc_internal/maat_fqdn_plugin.h index e1f881e..781952b 100644 --- a/src/inc_internal/maat_fqdn_plugin.h +++ b/src/inc_internal/maat_fqdn_plugin.h @@ -39,7 +39,7 @@ int fqdn_plugin_table_set_ex_container_schema(void *fqdn_plugin_schema, int tabl struct ex_container_schema *fqdn_plugin_table_get_ex_container_schema(void *fqdn_plugin_schema); /* fqdn plugin runtime API */ -void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, int max_thread_num, +void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void fqdn_plugin_runtime_free(void *fqdn_plugin_runtime); @@ -47,7 +47,8 @@ void fqdn_plugin_runtime_free(void *fqdn_plugin_runtime); int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_schema, const char *table_name, const char *line, int valid_column); -int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name, long long maat_rt_version); +int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name, + long long maat_rt_version); long long fqdn_plugin_runtime_rule_count(void *fqdn_plugin_runtime); @@ -58,6 +59,15 @@ int fqdn_plugin_runtime_get_ex_data(void *fqdn_plugin_runtime, const char *fqdn, void fqdn_rule_free(struct FQDN_rule *fqdn_rule); +void fqdn_plugin_runtime_perf_stat(void *fqdn_plugin_runtime, struct timespec *start, + struct timespec *end, int thread_id); + +long long fqdn_plugin_runtime_scan_count(void *fqdn_plugin_runtime); + +long long fqdn_plugin_runtime_scan_cpu_time(void *fqdn_plugin_runtime); + +long long fqdn_plugin_runtime_update_err_count(void *fqdn_plugin_runtime); + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_group.h b/src/inc_internal/maat_group.h index ae87fd3..d45346f 100644 --- a/src/inc_internal/maat_group.h +++ b/src/inc_internal/maat_group.h @@ -31,7 +31,7 @@ void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr, void group2group_schema_free(void *g2g_schema); /* group2group runtime API */ -void *group2group_runtime_new(void *g2g_schema, int max_thread_num, +void *group2group_runtime_new(void *g2g_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void group2group_runtime_free(void *g2g_runtime); @@ -53,6 +53,8 @@ int group2group_runtime_commit(void *g2g_runtime, const char *table_name, long l long long group2group_runtime_rule_count(void *g2g_runtime); +long long group2group_runtime_update_err_count(void *g2g_runtime); + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_interval.h b/src/inc_internal/maat_interval.h index 9a514f4..48ad052 100644 --- a/src/inc_internal/maat_interval.h +++ b/src/inc_internal/maat_interval.h @@ -28,7 +28,7 @@ void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr, void interval_schema_free(void *interval_schema); /* interval runtime API */ -void *interval_runtime_new(void *interval_schema, int max_thread_num, +void *interval_runtime_new(void *interval_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void interval_runtime_free(void *interval_runtime); @@ -52,8 +52,18 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, int interval_runtime_set_scan_district(struct interval_runtime *interval_rt, const char *district, size_t district_len, long long *district_id); -void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id); -long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread); +void interval_runtime_hit_inc(struct interval_runtime *interval_rt, int thread_id); + +void interval_runtime_perf_stat(struct interval_runtime *interval_rt, struct timespec *start, + struct timespec *end, int thread_id); + +long long interval_runtime_scan_count(void *interval_runtime); + +long long interval_runtime_scan_cpu_time(void *interval_runtime); + +long long interval_runtime_hit_count(void *interval_runtime); + +long long interval_runtime_update_err_cnt(void *interval_runtime); #ifdef __cplusplus } diff --git a/src/inc_internal/maat_ip.h b/src/inc_internal/maat_ip.h index 13bc01b..857d5c8 100644 --- a/src/inc_internal/maat_ip.h +++ b/src/inc_internal/maat_ip.h @@ -26,7 +26,7 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr, void ip_schema_free(void *ip_schema); /* ip runtime API */ -void *ip_runtime_new(void *ip_schema, int max_thread_num, +void *ip_runtime_new(void *ip_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void ip_runtime_free(void *ip_runtime); @@ -38,13 +38,25 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_r long long ip_runtime_rule_count(void *ip_runtime); +long long ip_runtime_ipv6_rule_count(void *ip_runtime); + /* ip runtime scan API */ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, uint8_t *ip_addr, uint16_t port, int proto, int vtable_id, struct maat_state *state); -void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id); -long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread); +void ip_runtime_hit_inc(struct ip_runtime *ip_rt, int thread_id); + +void ip_runtime_perf_stat(struct ip_runtime *ip_rt, struct timespec *start, + struct timespec *end, int thread_id); + +long long ip_runtime_scan_count(void *ip_runtime); + +long long ip_runtime_scan_cpu_time(void *ip_runtime); + +long long ip_runtime_hit_count(void *ip_runtime); + +long long ip_runtime_update_err_count(void *ip_runtime); #ifdef __cplusplus } diff --git a/src/inc_internal/maat_ip_plugin.h b/src/inc_internal/maat_ip_plugin.h index 802944e..af25384 100644 --- a/src/inc_internal/maat_ip_plugin.h +++ b/src/inc_internal/maat_ip_plugin.h @@ -38,7 +38,7 @@ int ip_plugin_table_set_ex_container_schema(void *ip_plugin_schema, int table_id struct ex_container_schema *ip_plugin_table_get_ex_container_schema(void *ip_plugin_schema); /* ip plugin runtime API */ -void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, +void *ip_plugin_runtime_new(void *ip_plugin_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void ip_plugin_runtime_free(void *ip_plugin_runtime); @@ -55,6 +55,15 @@ struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime int ip_plugin_runtime_get_ex_data(void *ip_plugin_runtime, const struct ip_addr *ip_addr, void **ex_data_array, size_t n_ex_data_array); +void ip_plugin_runtime_perf_stat(void *ip_plugin_runtime, struct timespec *start, + struct timespec *end, int thread_id); + +long long ip_plugin_runtime_scan_count(void *ip_plugin_runtime); + +long long ip_plugin_runtime_scan_cpu_time(void *ip_plugin_runtime); + +long long ip_plugin_runtime_update_err_count(void *ip_plugin_runtime); + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_limits.h b/src/inc_internal/maat_limits.h index d09165a..6040f37 100644 --- a/src/inc_internal/maat_limits.h +++ b/src/inc_internal/maat_limits.h @@ -17,7 +17,9 @@ extern "C" #endif #define MAX_KEYWORDS_STR 1024 - + +#define MAX_MAAT_STAT_NUM 64 + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_plugin.h b/src/inc_internal/maat_plugin.h index ec22c02..d1adef4 100644 --- a/src/inc_internal/maat_plugin.h +++ b/src/inc_internal/maat_plugin.h @@ -50,7 +50,7 @@ int plugin_table_set_ex_container_schema(void *plugin_schema, int table_id, struct ex_container_schema *plugin_table_get_ex_container_schema(void *plugin_schema); /* plugin runtime API */ -void *plugin_runtime_new(void *plugin_schema, int max_thread_num, +void *plugin_runtime_new(void *plugin_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void plugin_runtime_free(void *plugin_runtime); @@ -62,6 +62,8 @@ int plugin_runtime_commit(void *plugin_runtime, const char *table_name, long lon long long plugin_runtime_rule_count(void *plugin_runtime); +long long plugin_runtime_update_err_count(void *plugin_runtime); + struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime); size_t plugin_runtime_cached_row_count(void *plugin_runtime); diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index 98f1d7e..691e043 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -27,12 +27,15 @@ extern "C" #include "hiredis/hiredis.h" #include "uthash/uthash.h" +#include "log/log.h" +#include "fieldstat.h" #include "maat_command.h" -//#include "ip_matcher.h" +#include "maat_limits.h" #include "maat.h" #include "maat_kv.h" #include "maat_table.h" #include "maat_virtual.h" +#include "maat_stat.h" #define MAX_TABLE_NUM 1024 #define MAX_COMPILE_TABLE_NUM 16 @@ -168,6 +171,31 @@ struct rule_tag { char *tag_val; }; +struct maat_stat { + char stat_file[PATH_MAX]; + struct fieldstat_instance *fs_handle; + int total_stat_id[MAX_MAAT_STAT_NUM]; + int fs_status_id[MAX_MAAT_STAT_NUM]; + int fs_column_id[MAX_TABLE_NUM][MAX_MAAT_STAT_NUM]; + struct log_handle *logger; + struct table_manager *ref_tbl_mgr; + struct maat_garbage_bin *ref_garbage_bin; + size_t nr_worker_thread; + int cmd_q_cnt; + + + long long *thread_call_cnt; + long long *hit_cnt; + long long *not_grp_hit_cnt; + + long long scan_bytes; + long long scan_cnt; + long long update_err_cnt; + long long scan_err_cnt; + long long zombie_rs_stream; + long long line_cmd_acc_num; +}; + struct maat { char instance_name[NAME_MAX]; @@ -188,7 +216,7 @@ struct maat { int is_running; pthread_mutex_t background_update_mutex; - int nr_worker_thread; + size_t nr_worker_thread; long long maat_version; long long last_full_version; @@ -200,6 +228,8 @@ struct maat { int cumulative_update_off; //Default: cumulative update on + int stat_on; + int perf_on; struct maat_garbage_bin *garbage_bin; int default_compile_table_id; @@ -216,13 +246,7 @@ struct maat { long long new_version; /* statistics */ - long long line_cmd_acc_num; - - long long *thread_call_cnt; - long long *hit_cnt; - long long *not_grp_hit_cnt; - - long long scan_err_cnt; + struct maat_stat *stat; }; enum district_flag { diff --git a/src/inc_internal/maat_stat.h b/src/inc_internal/maat_stat.h new file mode 100644 index 0000000..8da1ebe --- /dev/null +++ b/src/inc_internal/maat_stat.h @@ -0,0 +1,35 @@ +/* +********************************************************************************************** +* File: maat_stat.h +* Description: maat statistics +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_STAT_H_ +#define _MAAT_STAT_H_ + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "maat_rule.h" + +struct maat_stat *maat_stat_new(const char *stat_file, size_t max_thread_num, + struct log_handle *logger); + +void maat_stat_free(struct maat_stat *stat); + +void maat_stat_init(struct maat_stat *stat, struct table_manager *tbl_mgr, + struct maat_garbage_bin *garbage_bin, const char *stat_inst_name); + +void maat_stat_output(struct maat_stat *stat, long long maat_version, int perf_on); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_table.h b/src/inc_internal/maat_table.h index bb745dc..148e2ff 100644 --- a/src/inc_internal/maat_table.h +++ b/src/inc_internal/maat_table.h @@ -37,7 +37,6 @@ enum table_type { TABLE_TYPE_BOOL_PLUGIN, //above are physical table TABLE_TYPE_VIRTUAL, - TABLE_TYPE_COMPOSITION, TABLE_TYPE_COMPILE, TABLE_TYPE_GROUP2GROUP, TABLE_TYPE_GROUP2COMPILE, @@ -49,14 +48,18 @@ struct table_manager; struct table_manager * table_manager_create(const char *table_info_path, const char *accept_tags, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); -int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_num, +int table_manager_runtime_create(struct table_manager *tbl_mgr, size_t max_thread_num, struct maat_garbage_bin *garbage_bin); void table_manager_runtime_destroy(struct table_manager *tbl_mgr); void table_manager_destroy(struct table_manager *tbl_mgr); +size_t table_manager_table_size(struct table_manager *tbl_mgr); size_t table_manager_table_count(struct table_manager *tbl_mgr); + int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name); +const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id); + enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id); int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr); @@ -79,6 +82,14 @@ void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id, long long table_manager_runtime_rule_count(struct table_manager *tbl_mgr, int table_id); +long long table_manager_runtime_scan_count(struct table_manager *tbl_mgr, int table_id); + +long long table_manager_runtime_scan_cpu_time(struct table_manager *tbl_mgr, int table_id); + +long long table_manager_runtime_hit_count(struct table_manager *tbl_mgr, int table_id); + +long long table_manager_runtime_update_err_count(struct table_manager *tbl_mgr, int table_id); + #ifdef __cplusplus } #endif diff --git a/src/maat_api.c b/src/maat_api.c index 60eae45..71f8534 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -39,6 +39,7 @@ #include "maat_fqdn_plugin.h" #include "maat_bool_plugin.h" #include "maat_virtual.h" +#include "maat_stat.h" #define MODULE_MAAT_API module_name_str("maat.api") @@ -154,11 +155,21 @@ int maat_options_set_deferred_load_on(struct maat_options *opts) int maat_options_set_stat_on(struct maat_options *opts) { + if (NULL == opts) { + return -1; + } + + opts->stat_on = 1; return 0; } int maat_options_set_perf_on(struct maat_options *opts) { + if (NULL == opts) { + return -1; + } + + opts->perf_on = 1; return 0; } @@ -206,6 +217,16 @@ int maat_options_set_redis(struct maat_options *opts, const char *redis_ip, return 0; } +int maat_options_set_stat_file(struct maat_options *opts, const char *stat_filename) +{ + size_t str_len = MIN(sizeof(opts->stat_file), strlen(stat_filename)); + + memcpy(opts->stat_file, stat_filename, str_len); + opts->stat_on = 1; + + return 0; +} + int maat_options_set_logger(struct maat_options *opts, const char *log_path, enum log_level level) { if (NULL == opts || NULL == log_path || strlen(log_path) >= PATH_MAX) { @@ -290,6 +311,32 @@ void maat_read_full_config(struct maat *maat_instance) } } +void _maat_free(struct maat *maat_instance) +{ + if (NULL == maat_instance) { + return; + } + + if (maat_instance->logger != NULL) { + log_handle_destroy(maat_instance->logger); + maat_instance->logger = NULL; + } + + if (maat_instance->garbage_bin != NULL) { + maat_garbage_bin_free(maat_instance->garbage_bin); + maat_instance->garbage_bin = NULL; + } + + if (maat_instance->stat != NULL) { + maat_stat_free(maat_instance->stat); + maat_instance->stat = NULL; + } + + pthread_mutex_destroy(&(maat_instance->background_update_mutex)); + + FREE(maat_instance); +} + struct maat *maat_new(struct maat_options *opts, const char *table_info_path) { if (NULL == opts || NULL == table_info_path) { @@ -357,14 +404,14 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) maat_instance->rule_effect_interval_ms = opts->rule_effect_interval_ms; maat_instance->rule_update_checking_interval_ms = opts->rule_update_checking_interval_ms; maat_instance->gc_timeout_ms = opts->gc_timeout_ms; + maat_instance->stat_on = opts->stat_on; + maat_instance->perf_on = opts->perf_on; maat_instance->deferred_load = opts->deferred_load_on; memcpy(maat_instance->foreign_cont_dir, opts->foreign_cont_dir, strlen(opts->foreign_cont_dir)); garbage_gc_timeout_s = (maat_instance->rule_effect_interval_ms / 1000) + (maat_instance->gc_timeout_ms / 1000); maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s); - maat_instance->thread_call_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); - maat_instance->hit_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); - maat_instance->not_grp_hit_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); + maat_instance->stat = maat_stat_new(opts->stat_file, opts->nr_worker_threads, maat_instance->logger); pthread_mutex_init(&(maat_instance->background_update_mutex), NULL); @@ -381,18 +428,16 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) maat_read_full_config(maat_instance); } + if (1 == maat_instance->stat_on) { + maat_stat_init(maat_instance->stat, maat_instance->tbl_mgr, maat_instance->garbage_bin, + maat_instance->instance_name); + } + pthread_create(&(maat_instance->cfg_mon_thread), NULL, rule_monitor_loop, (void *)maat_instance); return maat_instance; failed: - log_handle_destroy(maat_instance->logger); - table_manager_destroy(maat_instance->tbl_mgr); - maat_garbage_bin_free(maat_instance->garbage_bin); - alignment_int64_array_free(maat_instance->thread_call_cnt); - alignment_int64_array_free(maat_instance->hit_cnt); - alignment_int64_array_free(maat_instance->not_grp_hit_cnt); - pthread_mutex_destroy(&(maat_instance->background_update_mutex)); - FREE(maat_instance); + _maat_free(maat_instance); return NULL; } @@ -881,7 +926,7 @@ int flag_scan(struct table_manager *tbl_mgr, int thread_id, long long flag, return group_hit_cnt; } - flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id); + flag_runtime_hit_inc((struct flag_runtime *)flag_rt, thread_id); return group_hit_cnt; } @@ -911,7 +956,7 @@ int interval_scan(struct table_manager *tbl_mgr, int thread_id, long long intege return group_hit_cnt; } - interval_runtime_scan_hit_inc((struct interval_runtime *)interval_rt, thread_id); + interval_runtime_hit_inc((struct interval_runtime *)interval_rt, thread_id); return group_hit_cnt; } @@ -938,7 +983,7 @@ int ipv4_scan(struct table_manager *tbl_mgr, int thread_id, uint32_t ip_addr, return group_hit_cnt; } - ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); + ip_runtime_hit_inc((struct ip_runtime *)ip_rt, thread_id); return group_hit_cnt; } @@ -964,7 +1009,7 @@ int ipv6_scan(struct table_manager *tbl_mgr, int thread_id, uint8_t *ip_addr, return group_hit_cnt; } - ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); + ip_runtime_hit_inc((struct ip_runtime *)ip_rt, thread_id); return group_hit_cnt; } @@ -994,7 +1039,7 @@ int string_scan(struct table_manager *tbl_mgr, int thread_id, const char *data, return group_hit_cnt; } - expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id); + expr_runtime_hit_inc((struct expr_runtime *)expr_rt, thread_id); return group_hit_cnt; } @@ -1029,7 +1074,7 @@ int expr_stream_scan(struct maat_stream *stream, const char *data, size_t data_l return group_hit_cnt; } - expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, stream->thread_id); + expr_runtime_hit_inc((struct expr_runtime *)expr_rt, stream->thread_id); return group_hit_cnt; } @@ -1070,6 +1115,11 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, return MAAT_SCAN_ERR; } + struct timespec start, end; + if (1 == maat_instance->perf_on) { + clock_gettime(CLOCK_MONOTONIC, &start); + } + state->scan_cnt++; if (NULL == maat_instance->maat_rt) { @@ -1096,7 +1146,7 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, } maat_runtime_ref_inc(maat_instance->maat_rt, state->thread_id); - alignment_int64_array_add(maat_instance->thread_call_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->thread_call_cnt, state->thread_id, 1); int hit_group_cnt = flag_scan(maat_instance->tbl_mgr, state->thread_id, flag, physical_table_id, vtable_id, state); @@ -1106,28 +1156,37 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, size_t sum_hit_compile_cnt = 0; if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { - // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT sum_hit_compile_cnt = group_to_compile(maat_instance, results, n_result, state); *n_hit_result = sum_hit_compile_cnt; } if (sum_hit_compile_cnt > 0) { - alignment_int64_array_add(maat_instance->hit_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->hit_cnt, state->thread_id, 1); if (0 == hit_group_cnt) { //hit NOT group - alignment_int64_array_add(maat_instance->not_grp_hit_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->not_grp_hit_cnt, state->thread_id, 1); } - return MAAT_SCAN_HIT; + } + + void *flag_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_id); + assert(flag_rt != NULL); + + if (1 == maat_instance->perf_on) { + clock_gettime(CLOCK_MONOTONIC, &end); + flag_runtime_perf_stat(flag_rt, &start, &end, state->thread_id); } else { - // sum_hit_compile_cnt == 0 - if (hit_group_cnt > 0) { - return MAAT_SCAN_HALF_HIT; - } + flag_runtime_perf_stat(flag_rt, NULL, NULL, state->thread_id); } maat_runtime_ref_dec(maat_instance->maat_rt, state->thread_id); - return MAAT_SCAN_OK; + if (sum_hit_compile_cnt > 0) { + return MAAT_SCAN_HIT; + } else if (hit_group_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } else { + return MAAT_SCAN_OK; + } } int maat_scan_integer(struct maat *maat_instance, int table_id, @@ -1140,6 +1199,11 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, return MAAT_SCAN_ERR; } + struct timespec start, end; + if (1 == maat_instance->perf_on) { + clock_gettime(CLOCK_MONOTONIC, &start); + } + state->scan_cnt++; if (NULL == maat_instance->maat_rt) { @@ -1166,7 +1230,7 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, } maat_runtime_ref_inc(maat_instance->maat_rt, state->thread_id); - alignment_int64_array_add(maat_instance->thread_call_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->thread_call_cnt, state->thread_id, 1); int hit_group_cnt = interval_scan(maat_instance->tbl_mgr, state->thread_id, integer, physical_table_id, vtable_id, state); @@ -1176,28 +1240,37 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, size_t sum_hit_compile_cnt = 0; if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { - // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT sum_hit_compile_cnt = group_to_compile(maat_instance, results, n_result, state); *n_hit_result = sum_hit_compile_cnt; } if (sum_hit_compile_cnt > 0) { - alignment_int64_array_add(maat_instance->hit_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->hit_cnt, state->thread_id, 1); if (0 == hit_group_cnt) { //hit NOT group - alignment_int64_array_add(maat_instance->not_grp_hit_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->not_grp_hit_cnt, state->thread_id, 1); } - return MAAT_SCAN_HIT; + } + + void *interval_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_id); + assert(interval_rt != NULL); + + if (1 == maat_instance->perf_on) { + clock_gettime(CLOCK_MONOTONIC, &end); + interval_runtime_perf_stat(interval_rt, &start, &end, state->thread_id); } else { - // sum_hit_compile_cnt == 0 - if (hit_group_cnt > 0) { - return MAAT_SCAN_HALF_HIT; - } + interval_runtime_perf_stat(interval_rt, NULL, NULL, state->thread_id); } maat_runtime_ref_dec(maat_instance->maat_rt, state->thread_id); - return MAAT_SCAN_OK; + if (sum_hit_compile_cnt > 0) { + return MAAT_SCAN_HIT; + } else if (hit_group_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } else { + return MAAT_SCAN_OK; + } } int maat_scan_ipv4(struct maat *maat_instance, int table_id, @@ -1211,6 +1284,11 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, return MAAT_SCAN_ERR; } + struct timespec start, end; + if (1 == maat_instance->perf_on) { + clock_gettime(CLOCK_MONOTONIC, &start); + } + state->scan_cnt++; if (NULL == maat_instance->maat_rt) { @@ -1237,7 +1315,7 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, } maat_runtime_ref_inc(maat_instance->maat_rt, state->thread_id); - alignment_int64_array_add(maat_instance->thread_call_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->thread_call_cnt, state->thread_id, 1); int hit_group_cnt = ipv4_scan(maat_instance->tbl_mgr, state->thread_id, ip_addr, port, protocol, physical_table_id, vtable_id, state); @@ -1247,36 +1325,37 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, size_t sum_hit_compile_cnt = 0; if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { - // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT sum_hit_compile_cnt = group_to_compile(maat_instance, results, n_result, state); *n_hit_result = sum_hit_compile_cnt; } if (sum_hit_compile_cnt > 0) { - alignment_int64_array_add(maat_instance->hit_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->hit_cnt, state->thread_id, 1); if (0 == hit_group_cnt) { //hit NOT group - alignment_int64_array_add(maat_instance->not_grp_hit_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->not_grp_hit_cnt, state->thread_id, 1); } - return MAAT_SCAN_HIT; + } + + void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_id); + assert(ip_rt != NULL); + + if (1 == maat_instance->perf_on) { + clock_gettime(CLOCK_MONOTONIC, &end); + ip_runtime_perf_stat(ip_rt, &start, &end, state->thread_id); } else { - // n_hit_compile == 0 - if (hit_group_cnt > 0) { - return MAAT_SCAN_HALF_HIT; - } + ip_runtime_perf_stat(ip_rt, NULL, NULL, state->thread_id); } maat_runtime_ref_dec(maat_instance->maat_rt, state->thread_id); - return MAAT_SCAN_OK; -} - -int maat_scan_ipv4_tuple4(struct maat *instance, int table_id, - const struct ipv4_tuple *tuple4, long long *results, - size_t n_result, size_t *n_hit_result, - struct maat_state *state) -{ - return MAAT_SCAN_OK; + if (sum_hit_compile_cnt > 0) { + return MAAT_SCAN_HIT; + } else if (hit_group_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } else { + return MAAT_SCAN_OK; + } } int maat_scan_ipv6(struct maat *maat_instance, int table_id, @@ -1290,6 +1369,11 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, return MAAT_SCAN_ERR; } + struct timespec start, end; + if (1 == maat_instance->perf_on) { + clock_gettime(CLOCK_MONOTONIC, &start); + } + state->scan_cnt++; if (NULL == maat_instance->maat_rt) { @@ -1316,7 +1400,7 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, } maat_runtime_ref_inc(maat_instance->maat_rt, state->thread_id); - alignment_int64_array_add(maat_instance->thread_call_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->thread_call_cnt, state->thread_id, 1); int hit_group_cnt = ipv6_scan(maat_instance->tbl_mgr, state->thread_id, ip_addr, port, protocol, physical_table_id, vtable_id, state); @@ -1326,36 +1410,37 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, size_t sum_hit_compile_cnt = 0; if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { - // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT sum_hit_compile_cnt = group_to_compile(maat_instance, results, n_result, state); *n_hit_result = sum_hit_compile_cnt; } if (sum_hit_compile_cnt > 0) { - alignment_int64_array_add(maat_instance->hit_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->hit_cnt, state->thread_id, 1); if (0 == hit_group_cnt) { //hit NOT group - alignment_int64_array_add(maat_instance->not_grp_hit_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->not_grp_hit_cnt, state->thread_id, 1); } - return MAAT_SCAN_HIT; + } + + void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_id); + assert(ip_rt != NULL); + + if (1 == maat_instance->perf_on) { + clock_gettime(CLOCK_MONOTONIC, &end); + ip_runtime_perf_stat(ip_rt, &start, &end, state->thread_id); } else { - // n_hit_compile == 0 - if (hit_group_cnt > 0) { - return MAAT_SCAN_HALF_HIT; - } + ip_runtime_perf_stat(ip_rt, NULL, NULL, state->thread_id); } maat_runtime_ref_dec(maat_instance->maat_rt, state->thread_id); - return MAAT_SCAN_OK; -} - -int maat_scan_ipv6_tuple4(struct maat *instance, int table_id, - const struct ipv6_tuple *tuple, long long *results, - size_t n_result, size_t *n_hit_result, - struct maat_state *state) -{ - return MAAT_SCAN_OK; + if (sum_hit_compile_cnt > 0) { + return MAAT_SCAN_HIT; + } else if (hit_group_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } else { + return MAAT_SCAN_OK; + } } int maat_scan_string(struct maat *maat_instance, int table_id, const char *data, @@ -1368,6 +1453,11 @@ int maat_scan_string(struct maat *maat_instance, int table_id, const char *data, return MAAT_SCAN_ERR; } + struct timespec start, end; + if (1 == maat_instance->perf_on) { + clock_gettime(CLOCK_MONOTONIC, &start); + } + state->scan_cnt++; if (NULL == maat_instance->maat_rt) { @@ -1394,7 +1484,7 @@ int maat_scan_string(struct maat *maat_instance, int table_id, const char *data, } maat_runtime_ref_inc(maat_instance->maat_rt, state->thread_id); - alignment_int64_array_add(maat_instance->thread_call_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->thread_call_cnt, state->thread_id, 1); int hit_group_cnt = string_scan(maat_instance->tbl_mgr, state->thread_id, data, data_len, physical_table_id, vtable_id, state); @@ -1404,28 +1494,37 @@ int maat_scan_string(struct maat *maat_instance, int table_id, const char *data, size_t sum_hit_compile_cnt = 0; if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { - // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT sum_hit_compile_cnt = group_to_compile(maat_instance, results, n_result, state); *n_hit_result = sum_hit_compile_cnt; } if (sum_hit_compile_cnt > 0) { - alignment_int64_array_add(maat_instance->hit_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->hit_cnt, state->thread_id, 1); if (0 == hit_group_cnt) { //hit NOT group - alignment_int64_array_add(maat_instance->not_grp_hit_cnt, state->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->not_grp_hit_cnt, state->thread_id, 1); } - return MAAT_SCAN_HIT; + } + + void *expr_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_id); + assert(expr_rt != NULL); + + if (1 == maat_instance->perf_on) { + clock_gettime(CLOCK_MONOTONIC, &end); + expr_runtime_perf_stat(expr_rt, data_len, &start, &end, state->thread_id); } else { - // n_hit_compile == 0 - if (hit_group_cnt > 0) { - return MAAT_SCAN_HALF_HIT; - } + expr_runtime_perf_stat(expr_rt, data_len, NULL, NULL, state->thread_id); } maat_runtime_ref_dec(maat_instance->maat_rt, state->thread_id); - return MAAT_SCAN_OK; + if (sum_hit_compile_cnt > 0) { + return MAAT_SCAN_HIT; + } else if (hit_group_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } else { + return MAAT_SCAN_OK; + } } struct maat_stream *maat_stream_new(struct maat *maat_instance, int table_id, @@ -1488,6 +1587,11 @@ int maat_stream_scan(struct maat_stream *maat_stream, const char *data, int data return MAAT_SCAN_ERR; } + struct timespec start, end; + if (1 == maat_stream->ref_maat_instance->perf_on) { + clock_gettime(CLOCK_MONOTONIC, &start); + } + state->scan_cnt++; struct maat *maat_instance = maat_stream->ref_maat_instance; @@ -1503,7 +1607,7 @@ int maat_stream_scan(struct maat_stream *maat_stream, const char *data, int data return MAAT_SCAN_OK; } - alignment_int64_array_add(maat_stream->ref_maat_instance->thread_call_cnt, maat_stream->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->thread_call_cnt, maat_stream->thread_id, 1); int hit_group_cnt = expr_stream_scan(maat_stream, data, data_len, state); if (hit_group_cnt < 0) { @@ -1512,26 +1616,32 @@ int maat_stream_scan(struct maat_stream *maat_stream, const char *data, int data size_t sum_hit_compile_cnt = 0; if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { - // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT - sum_hit_compile_cnt = group_to_compile(maat_stream->ref_maat_instance, results, n_result, state); + sum_hit_compile_cnt = group_to_compile(maat_instance, results, n_result, state); *n_hit_result = sum_hit_compile_cnt; } if (sum_hit_compile_cnt > 0) { - alignment_int64_array_add(maat_stream->ref_maat_instance->hit_cnt, maat_stream->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->hit_cnt, maat_stream->thread_id, 1); if (0 == hit_group_cnt) { //hit NOT group - alignment_int64_array_add(maat_stream->ref_maat_instance->not_grp_hit_cnt, maat_stream->thread_id, 1); + alignment_int64_array_add(maat_instance->stat->not_grp_hit_cnt, maat_stream->thread_id, 1); } - return MAAT_SCAN_HIT; - } else { - // n_hit_compile == 0 - if (hit_group_cnt > 0) { - return MAAT_SCAN_HALF_HIT; - } - } + } - return sum_hit_compile_cnt; + if (1 == maat_instance->perf_on) { + clock_gettime(CLOCK_MONOTONIC, &end); + expr_runtime_perf_stat(expr_rt, data_len, &start, &end, state->thread_id); + } else { + expr_runtime_perf_stat(expr_rt, data_len, NULL, NULL, state->thread_id); + } + + if (sum_hit_compile_cnt > 0) { + return MAAT_SCAN_HIT; + } else if (hit_group_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } else { + return MAAT_SCAN_OK; + } } void maat_stream_free(struct maat_stream *maat_stream) @@ -1540,7 +1650,11 @@ void maat_stream_free(struct maat_stream *maat_stream) return; } - expr_runtime_stream_close(maat_stream->s_handle); + void *expr_rt = table_manager_get_runtime(maat_stream->ref_maat_instance->tbl_mgr, + maat_stream->physical_table_id); + assert(expr_rt != NULL); + + expr_runtime_stream_close(expr_rt, maat_stream->thread_id, maat_stream->s_handle); FREE(maat_stream); } diff --git a/src/maat_bool_plugin.c b/src/maat_bool_plugin.c index 11ecc8d..7692a47 100644 --- a/src/maat_bool_plugin.c +++ b/src/maat_bool_plugin.c @@ -10,6 +10,7 @@ #include +#include "alignment.h" #include "maat_bool_plugin.h" #include "bool_matcher.h" #include "maat_utils.h" @@ -36,8 +37,13 @@ struct bool_plugin_runtime { struct ex_data_runtime *ex_data_rt; long long version; long long rule_num; + size_t n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; + + long long update_err_cnt; + long long *scan_cnt; + long long *scan_cpu_time; }; /* bool plugin schema API */ @@ -169,7 +175,7 @@ size_t ull_dedup(unsigned long long item_ids[], size_t n_item) return index + 1; } -void *bool_plugin_runtime_new(void *bool_plugin_schema, int max_thread_num, +void *bool_plugin_runtime_new(void *bool_plugin_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { @@ -185,8 +191,10 @@ void *bool_plugin_runtime_new(void *bool_plugin_schema, int max_thread_num, ex_data_runtime_set_ex_container_schema(bool_plugin_rt->ex_data_rt, &(schema->container_schema)); } + bool_plugin_rt->n_worker_thread = max_thread_num; bool_plugin_rt->ref_garbage_bin = garbage_bin; bool_plugin_rt->logger = logger; + bool_plugin_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); return bool_plugin_rt; } @@ -208,6 +216,16 @@ void bool_plugin_runtime_free(void *bool_plugin_runtime) bool_plugin_rt->ex_data_rt = NULL; } + if (bool_plugin_rt->scan_cnt != NULL) { + alignment_int64_array_free(bool_plugin_rt->scan_cnt); + bool_plugin_rt->scan_cnt = NULL; + } + + if (bool_plugin_rt->scan_cpu_time != NULL) { + alignment_int64_array_free(bool_plugin_rt->scan_cpu_time); + bool_plugin_rt->scan_cpu_time = NULL; + } + FREE(bool_plugin_rt); } @@ -364,11 +382,13 @@ int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_sche int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { + bool_plugin_rt->update_err_cnt++; return -1; } int ret = get_column_pos(line, schema->item_id_column, &item_id_offset, &item_id_len); if (ret < 0) { + bool_plugin_rt->update_err_cnt++; return -1; } @@ -377,6 +397,7 @@ int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_sche // add bool_expr = bool_plugin_expr_new(line, schema, table_name, bool_plugin_rt->logger); if (NULL == bool_expr) { + bool_plugin_rt->update_err_cnt++; return -1; } } @@ -389,6 +410,7 @@ int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_sche if (bool_expr != NULL) { bool_plugin_expr_free(bool_expr); } + bool_plugin_rt->update_err_cnt++; return -1; } } else { @@ -406,7 +428,8 @@ void garbage_bool_matcher_free(void *matcher, void *arg) bool_matcher_free(bm); } -int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name, long long maat_rt_version) +int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name, + long long maat_rt_version) { if (NULL == bool_plugin_runtime) { return -1; @@ -525,3 +548,61 @@ int bool_plugin_runtime_get_ex_data(void *bool_plugin_runtime, unsigned long lon return n_result; } + +void bool_plugin_runtime_perf_stat(void *bool_plugin_runtime, struct timespec *start, + struct timespec *end, int thread_id) +{ + if (NULL == bool_plugin_runtime || thread_id < 0) { + return; + } + + struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; + alignment_int64_array_add(bool_plugin_rt->scan_cnt, thread_id, 1); + + if (start != NULL && end != NULL) { + if (NULL == bool_plugin_rt->scan_cpu_time) { + bool_plugin_rt->scan_cpu_time = alignment_int64_array_alloc(bool_plugin_rt->n_worker_thread); + } + + long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 + (end->tv_nsec - start->tv_nsec); + alignment_int64_array_add(bool_plugin_rt->scan_cpu_time, thread_id, consume_time); + } +} + +long long bool_plugin_runtime_scan_count(void *bool_plugin_runtime) +{ + if (NULL == bool_plugin_runtime) { + return 0; + } + + struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; + long long sum = alignment_int64_array_sum(bool_plugin_rt->scan_cnt, + bool_plugin_rt->n_worker_thread); + alignment_int64_array_reset(bool_plugin_rt->scan_cnt, bool_plugin_rt->n_worker_thread); + + return sum; +} + +long long bool_plugin_runtime_scan_cpu_time(void *bool_plugin_runtime) +{ + if (NULL == bool_plugin_runtime) { + return 0; + } + + struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; + long long sum = alignment_int64_array_sum(bool_plugin_rt->scan_cpu_time, + bool_plugin_rt->n_worker_thread); + alignment_int64_array_reset(bool_plugin_rt->scan_cpu_time, bool_plugin_rt->n_worker_thread); + + return sum; +} + +long long bool_plugin_runtime_update_err_count(void *bool_plugin_runtime) +{ + if (NULL == bool_plugin_runtime) { + return 0; + } + + struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; + return bool_plugin_rt->update_err_cnt; +} \ No newline at end of file diff --git a/src/maat_command.c b/src/maat_command.c index 9673f97..dc632b5 100644 --- a/src/maat_command.c +++ b/src/maat_command.c @@ -377,7 +377,7 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li } ret = success_cnt; - maat_instance->line_cmd_acc_num += success_cnt; + maat_instance->stat->line_cmd_acc_num += success_cnt; error_out: maat_cmd_clear_rule_cache(s_rule); diff --git a/src/maat_compile.c b/src/maat_compile.c index 2e48fbd..29c0598 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -75,7 +75,7 @@ struct compile_runtime { time_t version; struct maat_clause *clause_by_literals_hash; long long rule_num; - + long long update_err_cnt; struct bool_expr_match *expr_match_buff; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; @@ -85,6 +85,7 @@ struct group2compile_runtime { long long not_flag_group; long long version; long long rule_num; + long long update_err_cnt; struct compile_runtime *ref_compile_rt; struct group2group_runtime *ref_g2g_rt; }; @@ -552,7 +553,7 @@ void compile_item_free(struct compile_item *compile_item) FREE(compile_item); } -void *compile_runtime_new(void *compile_schema, int max_thread_num, +void *compile_runtime_new(void *compile_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { @@ -624,7 +625,7 @@ void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt) compile_rt->ref_maat_rt = maat_rt; } -void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, +void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { @@ -1576,7 +1577,7 @@ void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt, return ex_data; } -void compile_runtime_add_compile(struct compile_runtime *compile_rt, struct compile_schema *schema, +int compile_runtime_add_compile(struct compile_runtime *compile_rt, struct compile_schema *schema, long long compile_id, const char *table_name, const char *line) { struct compile_item *compile_item = NULL; @@ -1584,7 +1585,7 @@ void compile_runtime_add_compile(struct compile_runtime *compile_rt, struct comp compile_item = compile_item_new(line, schema, table_name, compile_rt->logger); if (NULL == compile_item) { - return; + return -1; } struct compile_rule *compile_rule = compile_rule_new(compile_item, schema, table_name, line); @@ -1646,6 +1647,8 @@ void compile_runtime_add_compile(struct compile_runtime *compile_rt, struct comp rcu_hash_add(compile_rt->cfg_hash_tbl, (char *)&compile_id, sizeof(long long), compile); } } + + return 0; } void compile_runtime_del_compile(struct compile_runtime *compile_rt, long long compile_id) @@ -1717,11 +1720,13 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { + compile_rt->update_err_cnt++; return -1; } long long compile_id = get_column_value(line, schema->compile_id_column); if (compile_id < 0) { + compile_rt->update_err_cnt++; return -1; } @@ -1730,7 +1735,11 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, compile_runtime_del_compile(compile_rt, compile_id); } else { // add - compile_runtime_add_compile(compile_rt, schema, compile_id, table_name, line); + int ret = compile_runtime_add_compile(compile_rt, schema, compile_id, + table_name, line); + if (ret < 0) { + compile_rt->update_err_cnt++; + } } return 0; @@ -1750,6 +1759,7 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, struct group2group_runtime *g2g_rt = g2c_rt->ref_g2g_rt; int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { + g2c_rt->update_err_cnt++; return -1; } @@ -1757,6 +1767,7 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, struct group2compile_item *g2c_item = group2compile_item_new(line, schema, table_name, compile_rt->logger); if (NULL == g2c_item) { + g2c_rt->update_err_cnt++; return -1; } @@ -1769,6 +1780,7 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, "[%s:%d] Remove group %d from compile %d failed, group is not exisited.", __FUNCTION__, __LINE__, g2c_item->group_id, g2c_item->compile_id); group2compile_item_free(g2c_item); + g2c_rt->update_err_cnt++; return -1; } @@ -1779,6 +1791,8 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, } maat_group_ref_dec(g2g_rt, group); g2c_rt->rule_num--; + } else { + g2c_rt->update_err_cnt++; } } else { //add @@ -1794,14 +1808,25 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, } maat_group_ref_inc(g2g_rt, group); g2c_rt->rule_num++; + } else { + g2c_rt->update_err_cnt++; } } - group2compile_item_free(g2c_item); return ret; } +long long group2compile_runtime_not_group_count(void *g2c_runtime) +{ + if (NULL == g2c_runtime) { + return 0; + } + + struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime; + return g2c_rt->not_flag_group; +} + long long group2compile_runtime_rule_count(void *g2c_runtime) { if (NULL == g2c_runtime) { @@ -1812,6 +1837,16 @@ long long group2compile_runtime_rule_count(void *g2c_runtime) return g2c_rt->rule_num; } +long long group2compile_runtime_update_err_count(void *g2c_runtime) +{ + if (NULL == g2c_runtime) { + return 0; + } + + struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime; + return g2c_rt->update_err_cnt; +} + int compile_runtime_commit(void *compile_runtime, const char *table_name, long long maat_rt_version) { if (NULL == compile_runtime) { @@ -1864,6 +1899,16 @@ long long compile_runtime_rule_count(void *compile_runtime) return compile_rt->rule_num; } +long long compile_runtime_update_err_count(void *compile_runtime) +{ + if (NULL == compile_runtime) { + return 0; + } + + struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; + return compile_rt->update_err_cnt; +} + static int compile_sort_para_compare(const struct compile_sort_para *a, const struct compile_sort_para *b) { diff --git a/src/maat_expr.c b/src/maat_expr.c index 284956f..917248d 100644 --- a/src/maat_expr.c +++ b/src/maat_expr.c @@ -72,8 +72,10 @@ struct expr_runtime { struct rcu_hash_table *item_htable; // store this expr table's all maat_item which will be used in expr_runtime_scan long long version; //expr_rt version + long long regex_rule_num; long long rule_num; - int n_worker_thread; + long long update_err_cnt; + size_t n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; int district_num; @@ -81,8 +83,10 @@ struct expr_runtime { struct maat_kv_store *tmp_district_map; long long *scan_cnt; + long long *scan_cpu_time; long long *hit_cnt; - // long long *stream_num; + long long *stream_num; + long long *scan_bytes; }; enum expr_type int_to_expr_type(int expr_type) @@ -446,7 +450,7 @@ void expr_maat_item_free(void *user_ctx, void *data) maat_item_free(item); } -void *expr_runtime_new(void *expr_schema, int max_thread_num, +void *expr_runtime_new(void *expr_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { @@ -465,6 +469,8 @@ void *expr_runtime_new(void *expr_schema, int max_thread_num, expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); expr_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); + expr_rt->scan_bytes = alignment_int64_array_alloc(max_thread_num); + expr_rt->stream_num = alignment_int64_array_alloc(max_thread_num); return expr_rt; } @@ -498,14 +504,29 @@ void expr_runtime_free(void *expr_runtime) expr_rt->district_map = NULL; } + if (expr_rt->scan_cnt != NULL) { + alignment_int64_array_free(expr_rt->scan_cnt); + expr_rt->scan_cnt = NULL; + } + + if (expr_rt->scan_cpu_time != NULL) { + alignment_int64_array_free(expr_rt->scan_cpu_time); + expr_rt->scan_cpu_time = NULL; + } + if (expr_rt->hit_cnt != NULL) { alignment_int64_array_free(expr_rt->hit_cnt); expr_rt->hit_cnt = NULL; } - if (expr_rt->scan_cnt != NULL) { - alignment_int64_array_free(expr_rt->scan_cnt); - expr_rt->scan_cnt = NULL; + if (expr_rt->stream_num != NULL) { + alignment_int64_array_free(expr_rt->stream_num); + expr_rt->stream_num = NULL; + } + + if (expr_rt->scan_bytes != NULL) { + alignment_int64_array_free(expr_rt->scan_bytes); + expr_rt->scan_bytes = NULL; } FREE(expr_rt); @@ -746,12 +767,13 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, long long item_id = get_column_value(line, schema->item_id_column); if (item_id < 0) { + expr_rt->update_err_cnt++; return -1; } int is_valid = get_column_value(line, valid_column); - //printf(" item_id:%lld is_valid:%d\n", item_id, is_valid); if (is_valid < 0) { + expr_rt->update_err_cnt++; return -1; } else if (0 == is_valid) { //delete @@ -760,6 +782,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, //add struct expr_item *expr_item = expr_item_new(line, schema, expr_rt); if (NULL == expr_item) { + expr_rt->update_err_cnt++; return -1; } @@ -772,6 +795,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, __FUNCTION__, __LINE__, table_name, item_id); expr_item_free(expr_item); maat_item_free(item); + expr_rt->update_err_cnt++; return -1; } @@ -781,6 +805,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, log_error(expr_rt->logger, MODULE_EXPR, "[%s:%d] [table:%s] transform expr_item(item_id:%lld) to expr_rule failed", __FUNCTION__, __LINE__, table_name, item_id); + expr_rt->update_err_cnt++; return -1; } } @@ -792,6 +817,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, if (expr_rule != NULL) { expr_rule_free(expr_rule); } + expr_rt->update_err_cnt++; return -1; } @@ -887,6 +913,16 @@ long long expr_runtime_rule_count(void *expr_runtime) return expr_rt->rule_num; } +long long expr_runtime_regex_rule_count(void *expr_runtime) +{ + if (NULL == expr_runtime) { + return 0; + } + + struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; + return expr_rt->regex_rule_num; +} + long long expr_runtime_get_version(void *expr_runtime) { if (NULL == expr_runtime) { @@ -955,6 +991,8 @@ struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, return NULL; } + alignment_int64_array_add(expr_rt->stream_num, thread_id, 1); + return adapter_hs_stream_open(expr_rt->hs, thread_id); } @@ -1001,17 +1039,116 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_str return group_hit_cnt; } -void expr_runtime_stream_close(struct adapter_hs_stream *s_handle) +void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id, + struct adapter_hs_stream *s_handle) { + alignment_int64_array_add(expr_rt->stream_num, thread_id, -1); adapter_hs_stream_close(s_handle); } -void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id) +void expr_runtime_hit_inc(struct expr_runtime *expr_rt, int thread_id) { + if (NULL == expr_rt || thread_id < 0) { + return; + } + alignment_int64_array_add(expr_rt->hit_cnt, thread_id, 1); } -long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread) +void expr_runtime_perf_stat(struct expr_runtime *expr_rt, size_t scan_len, + struct timespec *start, struct timespec *end, + int thread_id) { - return alignment_int64_array_sum(expr_rt->hit_cnt, n_thread); + if (NULL == expr_rt || thread_id < 0) { + return; + } + + alignment_int64_array_add(expr_rt->scan_cnt, thread_id, 1); + alignment_int64_array_add(expr_rt->scan_bytes, thread_id, scan_len); + + if (start != NULL && end != NULL) { + if (NULL == expr_rt->scan_cpu_time) { + expr_rt->scan_cpu_time = alignment_int64_array_alloc(expr_rt->n_worker_thread); + } + + long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 + end->tv_nsec - start->tv_nsec; + alignment_int64_array_add(expr_rt->scan_cpu_time, thread_id, consume_time); + } } + +long long expr_runtime_scan_count(void *expr_runtime) +{ + if (NULL == expr_runtime) { + return 0; + } + + struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; + long long sum = alignment_int64_array_sum(expr_rt->scan_cnt, + expr_rt->n_worker_thread); + alignment_int64_array_reset(expr_rt->scan_cnt, expr_rt->n_worker_thread); + + return sum; +} + +long long expr_runtime_scan_cpu_time(void *expr_runtime) +{ + if (NULL == expr_runtime) { + return 0; + } + + struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; + long long sum = alignment_int64_array_sum(expr_rt->scan_cpu_time, + expr_rt->n_worker_thread); + alignment_int64_array_reset(expr_rt->scan_cpu_time, expr_rt->n_worker_thread); + + return sum; +} + +long long expr_runtime_hit_count(void *expr_runtime) +{ + if (NULL == expr_runtime) { + return 0; + } + + struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; + long long sum = alignment_int64_array_sum(expr_rt->hit_cnt, + expr_rt->n_worker_thread); + alignment_int64_array_reset(expr_rt->hit_cnt, expr_rt->n_worker_thread); + + return sum; +} + +long long expr_runtime_update_err_count(void *expr_runtime) +{ + if (NULL == expr_runtime) { + return 0; + } + + struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; + return expr_rt->update_err_cnt; +} + +long long expr_runtime_scan_bytes(struct expr_runtime *expr_rt) +{ + if (NULL == expr_rt) { + return 0; + } + + long long sum = alignment_int64_array_sum(expr_rt->scan_bytes, + expr_rt->n_worker_thread); + alignment_int64_array_reset(expr_rt->scan_bytes, expr_rt->n_worker_thread); + + return sum; +} + +long long expr_runtime_stream_num(struct expr_runtime *expr_rt) +{ + if (NULL == expr_rt) { + return 0; + } + + long long sum = alignment_int64_array_sum(expr_rt->stream_num, expr_rt->n_worker_thread); + alignment_int64_array_reset(expr_rt->stream_num, expr_rt->n_worker_thread); + + return sum; +} \ No newline at end of file diff --git a/src/maat_flag.c b/src/maat_flag.c index eb165c9..e70d01b 100644 --- a/src/maat_flag.c +++ b/src/maat_flag.c @@ -48,13 +48,16 @@ struct flag_runtime { struct rcu_hash_table *item_htable; //store this flag table's all maat_item which will be used in flag_runtime_scan long long rule_num; long long version; + size_t n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; int district_num; struct maat_kv_store *district_map; struct maat_kv_store *tmp_district_map; + long long update_err_cnt; long long *scan_cnt; + long long *scan_cpu_time; long long *hit_cnt; }; @@ -164,7 +167,7 @@ void flag_maat_item_free(void *user_ctx, void *data) maat_item_free(item); } -void *flag_runtime_new(void *flag_schema, int max_thread_num, +void *flag_runtime_new(void *flag_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { @@ -176,6 +179,7 @@ void *flag_runtime_new(void *flag_schema, int max_thread_num, flag_rt->htable = rcu_hash_new(flag_ex_data_free, NULL); flag_rt->item_htable = rcu_hash_new(flag_maat_item_free, NULL); + flag_rt->n_worker_thread = max_thread_num; flag_rt->ref_garbage_bin = garbage_bin; flag_rt->logger = logger; flag_rt->district_map = maat_kv_store_new(); @@ -225,6 +229,11 @@ void flag_runtime_free(void *flag_runtime) flag_rt->scan_cnt = NULL; } + if (flag_rt->scan_cpu_time != NULL) { + alignment_int64_array_free(flag_rt->scan_cpu_time); + flag_rt->scan_cpu_time = NULL; + } + FREE(flag_rt); } @@ -399,6 +408,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { + flag_rt->update_err_cnt++; return -1; } else if (0 == is_valid) { //delete @@ -407,6 +417,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table //add struct flag_item *flag_item = flag_item_new(line, schema, flag_rt); if (NULL == flag_item) { + flag_rt->update_err_cnt++; return -1; } @@ -419,6 +430,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table __FUNCTION__, __LINE__, table_name, item_id); flag_item_free(flag_item); maat_item_free(item); + flag_rt->update_err_cnt++; return -1; } @@ -428,6 +440,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table log_error(flag_rt->logger, MODULE_FLAG, "[%s:%d] [table:%s] transform flag_item(item_id:%lld) to flag_rule failed", __FUNCTION__, __LINE__, table_name, item_id); + flag_rt->update_err_cnt++; return -1; } } @@ -439,6 +452,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table flag_rule_free(flag_rule); flag_rule = NULL; } + flag_rt->update_err_cnt++; return -1; } @@ -579,12 +593,83 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, return group_hit_cnt; } -void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id) +void flag_runtime_hit_inc(struct flag_runtime *flag_rt, int thread_id) { + if (NULL == flag_rt || thread_id < 0) { + return; + } + alignment_int64_array_add(flag_rt->hit_cnt, thread_id, 1); } -long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread) +void flag_runtime_perf_stat(struct flag_runtime *flag_rt, struct timespec *start, + struct timespec *end, int thread_id) { - return alignment_int64_array_sum(flag_rt->hit_cnt, n_thread); + if (NULL == flag_rt || thread_id < 0) { + return; + } + + alignment_int64_array_add(flag_rt->scan_cnt, thread_id, 1); + + if (start != NULL && end != NULL) { + if (NULL == flag_rt->scan_cpu_time) { + flag_rt->scan_cpu_time = alignment_int64_array_alloc(flag_rt->n_worker_thread); + } + + long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 + + (end->tv_nsec - start->tv_nsec); + alignment_int64_array_add(flag_rt->scan_cpu_time, thread_id, consume_time); + } } + +long long flag_runtime_scan_count(void *flag_runtime) +{ + if (NULL == flag_runtime) { + return 0; + } + + struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime; + long long sum = alignment_int64_array_sum(flag_rt->scan_cnt, + flag_rt->n_worker_thread); + alignment_int64_array_reset(flag_rt->scan_cnt, flag_rt->n_worker_thread); + + return sum; +} + +long long flag_runtime_scan_cpu_time(void *flag_runtime) +{ + if (NULL == flag_runtime) { + return 0; + } + + struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime; + long long sum = alignment_int64_array_sum(flag_rt->scan_cpu_time, + flag_rt->n_worker_thread); + alignment_int64_array_reset(flag_rt->scan_cpu_time, flag_rt->n_worker_thread); + + return sum; +} + +long long flag_runtime_hit_count(void *flag_runtime) +{ + if (NULL == flag_runtime) { + return 0; + } + + struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime; + long long sum = alignment_int64_array_sum(flag_rt->hit_cnt, + flag_rt->n_worker_thread); + alignment_int64_array_reset(flag_rt->hit_cnt, flag_rt->n_worker_thread); + + return sum; +} + +long long flag_runtime_update_err_count(void *flag_runtime) +{ + if (NULL == flag_runtime) { + return 0; + } + + struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime; + return flag_rt->update_err_cnt; +} \ No newline at end of file diff --git a/src/maat_fqdn_plugin.c b/src/maat_fqdn_plugin.c index 12f4672..704b4ae 100644 --- a/src/maat_fqdn_plugin.c +++ b/src/maat_fqdn_plugin.c @@ -10,6 +10,7 @@ #include +#include "alignment.h" #include "maat_fqdn_plugin.h" #include "maat_utils.h" #include "maat_table.h" @@ -37,8 +38,13 @@ struct fqdn_plugin_runtime { struct ex_data_runtime *ex_data_rt; long long version; long long rule_num; + long long update_err_cnt; + size_t n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; + + long long *scan_cnt; + long long *scan_cpu_time; }; void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -162,7 +168,7 @@ void fqdn_rule_free(struct FQDN_rule *fqdn_rule) free(fqdn_rule); } -void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, int max_thread_num, +void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { @@ -178,8 +184,10 @@ void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, int max_thread_num, ex_data_runtime_set_ex_container_schema(fqdn_plugin_rt->ex_data_rt, &(schema->container_schema)); } + fqdn_plugin_rt->n_worker_thread = max_thread_num; fqdn_plugin_rt->ref_garbage_bin = garbage_bin; fqdn_plugin_rt->logger = logger; + fqdn_plugin_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); return fqdn_plugin_rt; } @@ -201,6 +209,16 @@ void fqdn_plugin_runtime_free(void *fqdn_plugin_runtime) fqdn_plugin_rt->ex_data_rt = NULL; } + if (fqdn_plugin_rt->scan_cnt != NULL) { + alignment_int64_array_free(fqdn_plugin_rt->scan_cnt); + fqdn_plugin_rt->scan_cnt = NULL; + } + + if (fqdn_plugin_rt->scan_cpu_time != NULL) { + alignment_int64_array_free(fqdn_plugin_rt->scan_cpu_time); + fqdn_plugin_rt->scan_cpu_time = NULL; + } + FREE(fqdn_plugin_rt); } @@ -360,11 +378,13 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_sche int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { + fqdn_plugin_rt->update_err_cnt++; return -1; } int ret = get_column_pos(line, schema->item_id_column, &item_id_offset, &item_id_len); if (ret < 0) { + fqdn_plugin_rt->update_err_cnt++; return -1; } @@ -373,6 +393,7 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_sche // add fqdn_plugin_rule = fqdn_plugin_rule_new(line, schema, table_name, fqdn_plugin_rt->logger); if (NULL == fqdn_plugin_rule) { + fqdn_plugin_rt->update_err_cnt++; return -1; } } @@ -385,6 +406,7 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_sche if (fqdn_plugin_rule != NULL) { fqdn_plugin_rule_free(fqdn_plugin_rule); } + fqdn_plugin_rt->update_err_cnt++; return -1; } } else { @@ -520,3 +542,61 @@ int fqdn_plugin_runtime_get_ex_data(void *fqdn_plugin_runtime, const char *query return n_result; } + +void fqdn_plugin_runtime_perf_stat(void *fqdn_plugin_runtime, struct timespec *start, + struct timespec *end, int thread_id) +{ + if (NULL == fqdn_plugin_runtime || thread_id < 0) { + return; + } + + struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; + alignment_int64_array_add(fqdn_plugin_rt->scan_cnt, thread_id, 1); + + if (start != NULL && end != NULL) { + if (NULL == fqdn_plugin_rt->scan_cpu_time) { + fqdn_plugin_rt->scan_cpu_time = alignment_int64_array_alloc(fqdn_plugin_rt->n_worker_thread); + } + + long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 + (end->tv_nsec - start->tv_nsec); + alignment_int64_array_add(fqdn_plugin_rt->scan_cpu_time, thread_id, consume_time); + } +} + +long long fqdn_plugin_runtime_scan_count(void *fqdn_plugin_runtime) +{ + if (NULL == fqdn_plugin_runtime) { + return 0; + } + + struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; + long long sum = alignment_int64_array_sum(fqdn_plugin_rt->scan_cnt, + fqdn_plugin_rt->n_worker_thread); + alignment_int64_array_reset(fqdn_plugin_rt->scan_cnt, fqdn_plugin_rt->n_worker_thread); + + return sum; +} + +long long fqdn_plugin_runtime_scan_cpu_time(void *fqdn_plugin_runtime) +{ + if (NULL == fqdn_plugin_runtime) { + return 0; + } + + struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; + long long sum = alignment_int64_array_sum(fqdn_plugin_rt->scan_cpu_time, + fqdn_plugin_rt->n_worker_thread); + alignment_int64_array_reset(fqdn_plugin_rt->scan_cpu_time, fqdn_plugin_rt->n_worker_thread); + + return sum; +} + +long long fqdn_plugin_runtime_update_err_count(void *fqdn_plugin_runtime) +{ + if (NULL == fqdn_plugin_runtime) { + return 0; + } + + struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; + return fqdn_plugin_rt->update_err_cnt; +} \ No newline at end of file diff --git a/src/maat_group.c b/src/maat_group.c index c4f6546..7cb641f 100644 --- a/src/maat_group.c +++ b/src/maat_group.c @@ -61,7 +61,7 @@ struct group2group_runtime { struct maat_group_topology *group_topo; long long version; long long rule_num; - + long long update_err_cnt; int updating_flag; pthread_rwlock_t rwlock; struct maat_garbage_bin *ref_garbage_bin; @@ -137,7 +137,7 @@ struct maat_group_topology *maat_group_topology_new(struct log_handle *logger) return group_topo; } -void *group2group_runtime_new(void *g2g_schema, int max_thread_num, +void *group2group_runtime_new(void *g2g_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { @@ -609,12 +609,14 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { + g2g_rt->update_err_cnt++; return -1; } struct group2group_item *g2g_item = group2group_item_new(line, schema, table_name, g2g_rt->logger); if (NULL == g2g_item) { + g2g_rt->update_err_cnt++; return -1; } @@ -624,6 +626,8 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, g2g_item->super_group_id); if (0 == ret) { g2g_rt->rule_num--; + } else { + g2g_rt->update_err_cnt++; } } else { //add @@ -631,6 +635,8 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, g2g_item->super_group_id); if (0 == ret) { g2g_rt->rule_num++; + } else { + g2g_rt->update_err_cnt++; } } group2group_item_free(g2g_item); @@ -674,6 +680,16 @@ long long group2group_runtime_rule_count(void *g2g_runtime) return g2g_rt->rule_num; } +long long group2group_runtime_update_err_count(void *g2g_runtime) +{ + if (NULL == g2g_runtime) { + return 0; + } + + struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + return g2g_rt->update_err_cnt; +} + int group2group_runtime_get_top_groups(void *g2g_runtime, long long *group_ids, size_t n_group_ids, long long *top_group_ids) { diff --git a/src/maat_interval.c b/src/maat_interval.c index 86d066c..3268787 100644 --- a/src/maat_interval.c +++ b/src/maat_interval.c @@ -45,6 +45,8 @@ struct interval_runtime { struct rcu_hash_table *item_htable; //store this interval table's all maat_item which will be used in interval_runtime_scan long long version; long long rule_num; + long long update_err_cnt; + size_t n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; int district_num; @@ -52,6 +54,7 @@ struct interval_runtime { struct maat_kv_store *tmp_district_map; long long *scan_cnt; + long long *scan_cpu_time; long long *hit_cnt; }; @@ -161,7 +164,7 @@ void interval_maat_item_free(void *user_ctx, void *data) maat_item_free(item); } -void *interval_runtime_new(void *interval_schema, int max_thread_num, +void *interval_runtime_new(void *interval_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { @@ -173,6 +176,7 @@ void *interval_runtime_new(void *interval_schema, int max_thread_num, interval_rt->htable = rcu_hash_new(interval_ex_data_free, NULL); interval_rt->item_htable = rcu_hash_new(interval_maat_item_free, NULL); + interval_rt->n_worker_thread = max_thread_num; interval_rt->ref_garbage_bin = garbage_bin; interval_rt->logger = logger; interval_rt->district_map = maat_kv_store_new(); @@ -222,6 +226,11 @@ void interval_runtime_free(void *interval_runtime) interval_rt->scan_cnt = NULL; } + if (interval_rt->scan_cpu_time != NULL) { + alignment_int64_array_free(interval_rt->scan_cpu_time); + interval_rt->scan_cpu_time = NULL; + } + FREE(interval_rt); } @@ -391,11 +400,13 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema, long long item_id = get_column_value(line, schema->item_id_column); if (item_id < 0) { + interval_rt->update_err_cnt++; return -1; } int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { + interval_rt->update_err_cnt++; return -1; } else if (0 == is_valid) { //delete @@ -404,6 +415,7 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema, //add struct interval_item *interval_item = interval_item_new(line, schema, interval_rt); if (NULL == interval_item) { + interval_rt->update_err_cnt++; return -1; } @@ -416,6 +428,7 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema, __FUNCTION__, __LINE__, table_name, item_id); interval_item_free(interval_item); maat_item_free(item); + interval_rt->update_err_cnt++; return -1; } @@ -425,6 +438,7 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema, log_error(interval_rt->logger, MODULE_INTERVAL, "[%s:%d] [table:%s] transform interval_item(item_id:%lld) to interval_rule failed", __FUNCTION__, __LINE__, table_name, item_id); + interval_rt->update_err_cnt++; return -1; } } @@ -436,6 +450,7 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema, interval_rule_free(interval_rule); interval_rule = NULL; } + interval_rt->update_err_cnt++; return -1; } @@ -576,12 +591,80 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, return group_hit_cnt; } -void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id) +void interval_runtime_hit_inc(struct interval_runtime *interval_rt, int thread_id) { + if (NULL == interval_rt || thread_id < 0) { + return; + } + alignment_int64_array_add(interval_rt->hit_cnt, thread_id, 1); } -long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread) +void interval_runtime_perf_stat(struct interval_runtime *interval_rt, struct timespec *start, + struct timespec *end, int thread_id) { - return alignment_int64_array_sum(interval_rt->hit_cnt, n_thread); + if (NULL == interval_rt || thread_id < 0) { + return; + } + + alignment_int64_array_add(interval_rt->scan_cnt, thread_id, 1); + + if (start != NULL && end != NULL) { + if (NULL == interval_rt->scan_cpu_time) { + interval_rt->scan_cpu_time = alignment_int64_array_alloc(interval_rt->n_worker_thread); + } + + long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 + end->tv_nsec - start->tv_nsec; + alignment_int64_array_add(interval_rt->scan_cpu_time, thread_id, consume_time); + } } + +long long interval_runtime_scan_count(void *interval_runtime) +{ + if (NULL == interval_runtime) { + return 0; + } + + struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime; + long long sum = alignment_int64_array_sum(interval_rt->scan_cnt, interval_rt->n_worker_thread); + alignment_int64_array_reset(interval_rt->scan_cnt, interval_rt->n_worker_thread); + + return sum; +} + +long long interval_runtime_scan_cpu_time(void *interval_runtime) +{ + if (NULL == interval_runtime) { + return 0; + } + + struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime; + long long sum = alignment_int64_array_sum(interval_rt->scan_cpu_time, + interval_rt->n_worker_thread); + alignment_int64_array_reset(interval_rt->scan_cpu_time, interval_rt->n_worker_thread); + + return sum; +} + +long long interval_runtime_hit_count(void *interval_runtime) +{ + if (NULL == interval_runtime) { + return 0; + } + + struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime; + long long sum = alignment_int64_array_sum(interval_rt->hit_cnt, interval_rt->n_worker_thread); + alignment_int64_array_reset(interval_rt->hit_cnt, interval_rt->n_worker_thread); + + return sum; +} + +long long interval_runtime_update_err_cnt(void *interval_runtime) +{ + if (NULL == interval_runtime) { + return 0; + } + + struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime; + return interval_rt->update_err_cnt; +} \ No newline at end of file diff --git a/src/maat_ip.c b/src/maat_ip.c index 0a27407..ed55630 100644 --- a/src/maat_ip.c +++ b/src/maat_ip.c @@ -71,10 +71,14 @@ struct ip_runtime { struct rcu_hash_table *item_htable; //store this ip table's all maat_item which will be used in ip_runtime_scan long long version; long long rule_num; + long long ipv6_rule_num; + long long update_err_cnt; + size_t n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; long long *scan_cnt; + long long *scan_cpu_time; long long *hit_cnt; }; @@ -375,7 +379,7 @@ void ip_maat_item_free(void *user_ctx, void *data) maat_item_free(item); } -void *ip_runtime_new(void *ip_schema, int max_thread_num, +void *ip_runtime_new(void *ip_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { @@ -387,6 +391,7 @@ void *ip_runtime_new(void *ip_schema, int max_thread_num, ip_rt->htable = rcu_hash_new(ip_ex_data_free, NULL); ip_rt->item_htable = rcu_hash_new(ip_maat_item_free, NULL); + ip_rt->n_worker_thread = max_thread_num; ip_rt->ref_garbage_bin = garbage_bin; ip_rt->logger = logger; @@ -433,6 +438,11 @@ void ip_runtime_free(void *ip_runtime) ip_rt->scan_cnt = NULL; } + if (ip_rt->scan_cpu_time != NULL) { + alignment_int64_array_free(ip_rt->scan_cpu_time); + ip_rt->scan_cpu_time = NULL; + } + FREE(ip_rt); } @@ -497,11 +507,13 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema, struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; long long item_id = get_column_value(line, schema->item_id_column); if (item_id < 0) { + ip_rt->update_err_cnt++; return -1; } int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { + ip_rt->update_err_cnt++; return -1; } else if (0 == is_valid) { //delete @@ -510,6 +522,7 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema, //add ip_item = ip_item_new(line, schema, ip_rt->logger); if (NULL == ip_item) { + ip_rt->update_err_cnt++; return -1; } @@ -522,6 +535,7 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema, __FUNCTION__, __LINE__, table_name, item_id); ip_item_free(ip_item); maat_item_free(item); + ip_rt->update_err_cnt++; return -1; } } @@ -533,6 +547,7 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema, ip_item_free(ip_item); ip_item = NULL; } + ip_rt->update_err_cnt++; return -1; } @@ -559,6 +574,7 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_r } rcu_hash_commit(ip_rt->htable); + ip_rt->ipv6_rule_num = 0; struct ip_rule *rules = NULL; struct interval_rule *intval_rules = NULL; @@ -569,6 +585,10 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_r intval_rules = ALLOC(struct interval_rule, rule_cnt); for (size_t i = 0; i < rule_cnt; i++) { struct ip_item *item = (struct ip_item *)ex_data_array[i]; + if (item->addr_type == IPv6) { + ip_rt->ipv6_rule_num++; + } + ip_item_to_ip_rule(item, &rules[i]); ip_item_to_port_rule(item, &intval_rules[i]); } @@ -647,6 +667,16 @@ long long ip_runtime_rule_count(void *ip_runtime) return ip_rt->rule_num; } +long long ip_runtime_ipv6_rule_count(void *ip_runtime) +{ + if (NULL == ip_runtime) { + return 0; + } + + struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; + return ip_rt->ipv6_rule_num; +} + int validate_port(struct rcu_hash_table *htable, const char *key, size_t key_len, uint16_t port, int proto) { @@ -777,12 +807,80 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, return group_hit_cnt; } -void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id) +void ip_runtime_hit_inc(struct ip_runtime *ip_rt, int thread_id) { + if (NULL == ip_rt || thread_id < 0) { + return; + } + alignment_int64_array_add(ip_rt->hit_cnt, thread_id, 1); } -long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread) +void ip_runtime_perf_stat(struct ip_runtime *ip_rt, struct timespec *start, + struct timespec *end, int thread_id) { - return alignment_int64_array_sum(ip_rt->hit_cnt, n_thread); + if (NULL == ip_rt || thread_id < 0) { + return; + } + + alignment_int64_array_add(ip_rt->scan_cnt, thread_id, 1); + + if (start != NULL && end != NULL) { + if (NULL == ip_rt->scan_cpu_time) { + ip_rt->scan_cpu_time = alignment_int64_array_alloc(ip_rt->n_worker_thread); + } + + long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 + end->tv_nsec - start->tv_nsec; + alignment_int64_array_add(ip_rt->scan_cpu_time, thread_id, consume_time); + } } + +long long ip_runtime_scan_count(void *ip_runtime) +{ + if (NULL == ip_runtime) { + return 0; + } + + struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; + long long sum = alignment_int64_array_sum(ip_rt->scan_cnt, ip_rt->n_worker_thread); + alignment_int64_array_reset(ip_rt->scan_cnt, ip_rt->n_worker_thread); + + return sum; +} + +long long ip_runtime_scan_cpu_time(void *ip_runtime) +{ + if (NULL == ip_runtime) { + return 0; + } + + struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; + long long sum = alignment_int64_array_sum(ip_rt->scan_cpu_time, + ip_rt->n_worker_thread); + alignment_int64_array_reset(ip_rt->scan_cpu_time, ip_rt->n_worker_thread); + + return sum; +} + +long long ip_runtime_hit_count(void *ip_runtime) +{ + if (NULL == ip_runtime) { + return 0; + } + + struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; + long long sum = alignment_int64_array_sum(ip_rt->hit_cnt, ip_rt->n_worker_thread); + alignment_int64_array_reset(ip_rt->hit_cnt, ip_rt->n_worker_thread); + + return sum; +} + +long long ip_runtime_update_err_count(void *ip_runtime) +{ + if (NULL == ip_runtime) { + return 0; + } + + struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; + return ip_rt->update_err_cnt; +} \ No newline at end of file diff --git a/src/maat_ip_plugin.c b/src/maat_ip_plugin.c index ebdf422..34e59be 100644 --- a/src/maat_ip_plugin.c +++ b/src/maat_ip_plugin.c @@ -10,6 +10,7 @@ #include +#include "alignment.h" #include "log/log.h" #include "maat_utils.h" #include "maat_ip_plugin.h" @@ -41,8 +42,13 @@ struct ip_plugin_runtime { struct ex_data_runtime *ex_data_rt; long long version; long long rule_num; + long long update_err_cnt; + size_t n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; + + long long *scan_cnt; + long long *scan_cpu_time; }; void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -346,7 +352,7 @@ int ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, return 0; } -void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, +void *ip_plugin_runtime_new(void *ip_plugin_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { @@ -362,9 +368,12 @@ void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, ex_data_runtime_set_ex_container_schema(ip_plugin_rt->ex_data_rt, &(schema->container_schema)); } + ip_plugin_rt->n_worker_thread = max_thread_num; ip_plugin_rt->ref_garbage_bin = garbage_bin; ip_plugin_rt->logger = logger; + ip_plugin_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); + return ip_plugin_rt; } @@ -385,6 +394,16 @@ void ip_plugin_runtime_free(void *ip_plugin_runtime) ip_plugin_rt->ex_data_rt = NULL; } + if (ip_plugin_rt->scan_cnt != NULL) { + alignment_int64_array_free(ip_plugin_rt->scan_cnt); + ip_plugin_rt->scan_cnt = NULL; + } + + if (ip_plugin_rt->scan_cpu_time != NULL) { + alignment_int64_array_free(ip_plugin_rt->scan_cpu_time); + ip_plugin_rt->scan_cpu_time = NULL; + } + FREE(ip_plugin_rt); } @@ -404,11 +423,13 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { + ip_plugin_rt->update_err_cnt++; return -1; } int ret = get_column_pos(line, schema->item_id_column, &item_id_offset, &item_id_len); if (ret < 0) { + ip_plugin_rt->update_err_cnt++; return -1; } @@ -417,6 +438,7 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, // add ip_plugin_rule = ip_plugin_rule_new(line, schema, table_name, ip_plugin_rt->logger); if (NULL == ip_plugin_rule) { + ip_plugin_rt->update_err_cnt++; return -1; } } @@ -429,6 +451,7 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, if (ip_plugin_rule != NULL) { ip_plugin_rule_free(ip_plugin_rule); } + ip_plugin_rt->update_err_cnt++; return -1; } } else { @@ -562,4 +585,62 @@ int ip_plugin_runtime_get_ex_data(void *ip_plugin_runtime, const struct ip_addr (struct ex_container *)results[i].tag); } return n_result; +} + +void ip_plugin_runtime_perf_stat(void *ip_plugin_runtime, struct timespec *start, + struct timespec *end, int thread_id) +{ + if (NULL == ip_plugin_runtime || thread_id < 0) { + return; + } + + struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; + alignment_int64_array_add(ip_plugin_rt->scan_cnt, thread_id, 1); + + if (start != NULL && end != NULL) { + if (NULL == ip_plugin_rt->scan_cpu_time) { + ip_plugin_rt->scan_cpu_time = alignment_int64_array_alloc(ip_plugin_rt->n_worker_thread); + } + + long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 + (end->tv_nsec - start->tv_nsec); + alignment_int64_array_add(ip_plugin_rt->scan_cpu_time, thread_id, consume_time); + } +} + +long long ip_plugin_runtime_scan_count(void *ip_plugin_runtime) +{ + if (NULL == ip_plugin_runtime) { + return 0; + } + + struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; + long long sum = alignment_int64_array_sum(ip_plugin_rt->scan_cnt, + ip_plugin_rt->n_worker_thread); + alignment_int64_array_reset(ip_plugin_rt->scan_cnt, ip_plugin_rt->n_worker_thread); + + return sum; +} + +long long ip_plugin_runtime_scan_cpu_time(void *ip_plugin_runtime) +{ + if (NULL == ip_plugin_runtime) { + return 0; + } + + struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; + long long sum = alignment_int64_array_sum(ip_plugin_rt->scan_cpu_time, + ip_plugin_rt->n_worker_thread); + alignment_int64_array_reset(ip_plugin_rt->scan_cpu_time, ip_plugin_rt->n_worker_thread); + + return sum; +} + +long long ip_plugin_runtime_update_err_count(void *ip_plugin_runtime) +{ + if (NULL == ip_plugin_runtime) { + return 0; + } + + struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; + return ip_plugin_rt->update_err_cnt; } \ No newline at end of file diff --git a/src/maat_plugin.c b/src/maat_plugin.c index e02d246..7e08a12 100644 --- a/src/maat_plugin.c +++ b/src/maat_plugin.c @@ -33,6 +33,7 @@ struct plugin_runtime { struct ex_data_runtime *ex_data_rt; long long version; long long rule_num; + long long update_err_cnt; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; }; @@ -56,9 +57,6 @@ struct plugin_schema { int table_id; //ugly struct table_manager *ref_tbl_mgr; struct log_handle *logger; - - unsigned long long update_err_cnt; - unsigned long long unmatch_tag_cnt; }; static int read_integer_array(char *string, int *array, int size) @@ -263,7 +261,7 @@ struct ex_container_schema *plugin_table_get_ex_container_schema(void *plugin_sc return &(schema->container_schema); } -void *plugin_runtime_new(void *plugin_schema, int max_thread_num, +void *plugin_runtime_new(void *plugin_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { @@ -357,7 +355,6 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *table_name log_error(logger, MODULE_PLUGIN, "[%s:%d] plugin table:%s has no rule_tag, line:%s", __FUNCTION__, __LINE__, table_name, line); - schema->update_err_cnt++; return TAG_MATCH_ERR; } @@ -370,12 +367,10 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *table_name log_error(logger, MODULE_PLUGIN, "[%s:%d] plugin table:%s has invalid tag format, line:%s", __FUNCTION__, __LINE__, table_name, line); - schema->update_err_cnt++; return TAG_MATCH_ERR; } if (TAG_MATCH_UNMATCHED == ret) { - schema->unmatch_tag_cnt++; return TAG_MATCH_UNMATCHED; } } @@ -397,17 +392,20 @@ int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { + plugin_rt->update_err_cnt++; return -1; } int ret = plugin_accept_tag_match(schema, table_name, line, plugin_rt->logger); if (ret == TAG_MATCH_UNMATCHED) { + plugin_rt->update_err_cnt++; return -1; } size_t key_offset = 0, key_len = 0; ret = get_column_pos(line, schema->key_column, &key_offset, &key_len); if (ret < 0) { + plugin_rt->update_err_cnt++; return -1; } @@ -422,7 +420,7 @@ int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, ret = plugin_runtime_update_row(plugin_rt, schema, table_name, line, key, key_len, is_valid); if (ret < 0) { - schema->update_err_cnt++; + plugin_rt->update_err_cnt++; return -1; } @@ -470,6 +468,16 @@ long long plugin_runtime_rule_count(void *plugin_runtime) return plugin_rt->rule_num; } +long long plugin_runtime_update_err_count(void *plugin_runtime) +{ + if (NULL == plugin_runtime) { + return 0; + } + + struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; + return plugin_rt->update_err_cnt; +} + struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime) { if (NULL == plugin_runtime) { diff --git a/src/maat_rule.c b/src/maat_rule.c index 4339f3d..e0250f1 100644 --- a/src/maat_rule.c +++ b/src/maat_rule.c @@ -25,6 +25,7 @@ #include "maat_table.h" #include "maat_compile.h" #include "maat_plugin.h" +#include "maat_stat.h" #include "ip_matcher.h" #include "alignment.h" #include "maat_garbage_collection.h" @@ -561,15 +562,19 @@ void *rule_monitor_loop(void *arg) pthread_mutex_unlock(&(maat_instance->background_update_mutex)); } maat_garbage_collect_routine(maat_instance->garbage_bin); + if ((1 == maat_instance->stat_on) && (time(NULL) % 2 == 0)) { + maat_stat_output(maat_instance->stat, maat_instance->maat_version, maat_instance->perf_on); + } } maat_runtime_destroy(maat_instance->maat_rt); maat_garbage_bin_free(maat_instance->garbage_bin); table_manager_destroy(maat_instance->tbl_mgr); //table manager MUST be freed at last. - alignment_int64_array_free(maat_instance->thread_call_cnt); - alignment_int64_array_free(maat_instance->hit_cnt); - alignment_int64_array_free(maat_instance->not_grp_hit_cnt); + if (maat_instance->stat != NULL) { + maat_stat_free(maat_instance->stat); + maat_instance->stat = NULL; + } if (maat_instance->input_mode == DATA_SOURCE_REDIS) { if (maat_instance->mr_ctx.read_ctx != NULL) { diff --git a/src/maat_stat.c b/src/maat_stat.c new file mode 100644 index 0000000..9e0c35b --- /dev/null +++ b/src/maat_stat.c @@ -0,0 +1,373 @@ +/* +********************************************************************************************** +* File: maat_stat.c +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include + +#include "maat_utils.h" +#include "maat_stat.h" +#include "fieldstat.h" +#include "alignment.h" +#include "maat_ip.h" +#include "maat_compile.h" +#include "maat_group.h" +#include "maat_plugin.h" +#include "maat_expr.h" + +#define MODULE_MAAT_STAT module_name_str("maat.stat") + +enum MAAT_FS_STATUS { + STATUS_VERSION = 0, + STATUS_THRED_NUM, + STATUS_TABLE_NUM, + STATUS_PLUGIN_CACHE_NUM, + STATUS_PLUGIN_ACC_NUM, + STATUS_GROUP_REF_NUM, + STATUS_GROUP_REF_NOT_NUM, + STATUS_COMPILE_RULE_NUM, + STATUS_OUTER_MID_NUM, + STATUS_INNER_MID_NUM, + STATUS_GARBAGE_QSIZE, + STATUS_TOTAL_SCAN_LEN, + STATUS_TOTAL_SCAN_CNT, + STATUS_UPDATE_ERR_CNT, + STATUS_ICONV_ERR_CNT, + STATUS_SCAN_ERR_CNT, + STATUS_ZOMBIE_RS_STREAM, + STATUS_NOT_GROUP_HIT, + STATUS_CMD_NUM, + STATUS_CMD_Q_SIZE, + STATUS_CMD_LINE_NUM +}; + +enum MAAT_FS_COLUMN { + COLUMN_TABLE_RULE_NUM = 0, + COLUMN_TABLE_REGEX_NUM, + COLUMN_TABLE_IPV6_NUM, + COLUMN_TABLE_STREAM_NUM, + COLUMN_TABLE_SCAN_CNT, + COLUMN_TABLE_SCAN_BYTES, + COLUMN_TABLE_CPU_TIME,//microseconds + COLUMN_TABLE_HIT_CNT, +}; + +const char *common_column_name[] = {"rule", "reg", "v6", "stream", "scan_cnt", "scan_bytes", "scan_cpu_time", "hit_cnt"}; +enum field_type common_column_type[] = { + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE, + FIELD_TYPE_GAUGE +}; + +#define MAX_CONJ_NAME_LEN 22 +void maat_fieldstat_register(struct maat_stat *stat) +{ + stat->fs_status_id[STATUS_VERSION] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "version", NULL, 0); + stat->fs_status_id[STATUS_THRED_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "threads", NULL, 0); + stat->fs_status_id[STATUS_TABLE_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "tables", NULL, 0); + stat->fs_status_id[STATUS_PLUGIN_CACHE_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "plug_cached", NULL, 0); + stat->fs_status_id[STATUS_PLUGIN_ACC_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "plug_acc", NULL, 0); + stat->fs_status_id[STATUS_GROUP_REF_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "group", NULL, 0); + stat->fs_status_id[STATUS_GROUP_REF_NOT_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "not_grp", NULL, 0); + stat->fs_status_id[STATUS_COMPILE_RULE_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "compile", NULL, 0); + stat->fs_status_id[STATUS_GARBAGE_QSIZE] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "garbage_num", NULL, 0); + stat->fs_status_id[STATUS_OUTER_MID_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "outer_mid", NULL, 0); + stat->fs_status_id[STATUS_ZOMBIE_RS_STREAM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "z_stream", NULL, 0); + stat->fs_status_id[STATUS_NOT_GROUP_HIT] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "nt_grp_hit", NULL, 0); + stat->fs_status_id[STATUS_TOTAL_SCAN_LEN] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "scan_bytes", NULL, 0); + stat->fs_status_id[STATUS_TOTAL_SCAN_CNT] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "scan_times", NULL, 0); + stat->fs_status_id[STATUS_UPDATE_ERR_CNT] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "update_err", NULL, 0); + stat->fs_status_id[STATUS_SCAN_ERR_CNT] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "scan_error", NULL, 0); + stat->fs_status_id[STATUS_CMD_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "cmd_commit", NULL, 0); + stat->fs_status_id[STATUS_CMD_Q_SIZE] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "cmd_in_q", NULL, 0); + stat->fs_status_id[STATUS_CMD_LINE_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE, + "line_cmd/s", NULL, 0); +} + +void maat_fieldstat_table_row_register(struct maat_stat *stat, struct table_manager *tbl_mgr, + int table_id) +{ + size_t max_table_cnt = table_manager_table_size(tbl_mgr); + int ret = fieldstat_register_table_row(stat->fs_handle, table_id, "Sum", NULL, 0, + stat->total_stat_id); + if (ret < 0) { + log_error(stat->logger, MODULE_MAAT_STAT, "fieldstat_register_table_row Sum failed."); + return; + } + + for (size_t i = 0; i < max_table_cnt; i++) { + void *schema = table_manager_get_schema(tbl_mgr, i); + if (NULL == schema) { + continue; + } + + const char *table_name = table_manager_get_table_name(tbl_mgr, i); + assert(table_name != NULL); + + ret = fieldstat_register_table_row(stat->fs_handle, table_id, table_name, NULL, 0, + stat->fs_column_id[i]); + if (ret < 0) { + log_error(stat->logger, MODULE_MAAT_STAT, + "fieldstat_register_table_row %s failed.", table_name); + return; + } + } +} + +struct maat_stat *maat_stat_new(const char *stat_file, size_t max_thread_num, struct log_handle *logger) +{ + struct maat_stat *stat = ALLOC(struct maat_stat, 1); + + size_t str_len = MIN(sizeof(stat->stat_file), strlen(stat_file)); + memcpy(stat->stat_file, stat_file, str_len); + + stat->nr_worker_thread = max_thread_num; + stat->logger = logger; + stat->thread_call_cnt = alignment_int64_array_alloc(max_thread_num); + stat->hit_cnt = alignment_int64_array_alloc(max_thread_num); + stat->not_grp_hit_cnt = alignment_int64_array_alloc(max_thread_num); + + return stat; +} + +void maat_stat_free(struct maat_stat *stat) +{ + if (NULL == stat) { + return; + } + + if (stat->thread_call_cnt != NULL) { + alignment_int64_array_free(stat->thread_call_cnt); + stat->thread_call_cnt = NULL; + } + + if (stat->hit_cnt != NULL) { + alignment_int64_array_free(stat->hit_cnt); + stat->hit_cnt = NULL; + } + + if (stat->not_grp_hit_cnt != NULL) { + alignment_int64_array_free(stat->not_grp_hit_cnt); + stat->not_grp_hit_cnt = NULL; + } + + if (stat->fs_handle != NULL) { + fieldstat_instance_free(stat->fs_handle); + stat->fs_handle = NULL; + } + + free(stat); +} + +void maat_stat_init(struct maat_stat *stat, struct table_manager *tbl_mgr, + struct maat_garbage_bin *garbage_bin, const char *stat_inst_name) +{ + stat->fs_handle = fieldstat_instance_new(stat_inst_name); + if (NULL == stat->fs_handle) { + log_error(stat->logger, MODULE_MAAT_STAT, + "fieldstat_instance_new failed."); + return; + } + + int ret = fieldstat_set_local_output(stat->fs_handle, stat->stat_file, "default"); + if (ret < 0) { + log_error(stat->logger, MODULE_MAAT_STAT, + "fieldstat_set_local_output failed."); + return; + } + + ret = fieldstat_disable_background_thread(stat->fs_handle); + if (ret < 0) { + log_error(stat->logger, MODULE_MAAT_STAT, + "fieldstat_disable_background_thread failed."); + return; + } + + maat_fieldstat_register(stat); + + stat->ref_tbl_mgr = tbl_mgr; + stat->ref_garbage_bin = garbage_bin; + size_t n_column = sizeof(common_column_name) / sizeof(common_column_name[0]); + int fs_table_id = fieldstat_register_table(stat->fs_handle, stat_inst_name, + common_column_name, common_column_type, + n_column); + + maat_fieldstat_table_row_register(stat, tbl_mgr, fs_table_id); + + fieldstat_instance_start(stat->fs_handle); +} + +void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on) +{ + long long plugin_cache_num = 0, plugin_rule_num = 0, not_group_num = 0; + long long compile_rule_num = 0, g2c_rule_num = 0, g2g_rule_num = 0; + long long total_rule_num = 0, total_stream_num = 0, total_input_bytes = 0; + long long total_scan_cnt = 0, total_scan_cpu_time = 0, total_regex_num = 0; + long long total_ipv6_num = 0, total_hit_cnt = 0, total_update_err = 0; + size_t max_table_count = table_manager_table_size(stat->ref_tbl_mgr); + + for (size_t i = 0; i < max_table_count; i++) { + long long regex_rule_num = 0; + long long ipv6_rule_num = 0; + void *schema = table_manager_get_schema(stat->ref_tbl_mgr, i); + if (NULL == schema) { + continue; + } + + enum table_type table_type = table_manager_get_table_type(stat->ref_tbl_mgr, i); + assert(table_type != TABLE_TYPE_INVALID); + + void *runtime = table_manager_get_runtime(stat->ref_tbl_mgr, i); + + switch (table_type) { + case TABLE_TYPE_PLUGIN: + plugin_cache_num += plugin_runtime_cached_row_count(runtime); + plugin_rule_num += plugin_runtime_rule_count(runtime); + break; + case TABLE_TYPE_COMPILE: + compile_rule_num += compile_runtime_rule_count(runtime); + break; + case TABLE_TYPE_GROUP2COMPILE: + g2c_rule_num += group2compile_runtime_rule_count(runtime); + not_group_num += group2compile_runtime_not_group_count(runtime); + break; + case TABLE_TYPE_GROUP2GROUP: + g2g_rule_num += group2group_runtime_rule_count(runtime); + break; + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + //regex rule cnt + regex_rule_num = expr_runtime_regex_rule_count(runtime); + break; + case TABLE_TYPE_IP_PLUS: + ipv6_rule_num = ip_runtime_ipv6_rule_count(runtime); + break; + default: + break; + } + + if (table_type == TABLE_TYPE_PLUGIN || table_type == TABLE_TYPE_COMPILE || + table_type == TABLE_TYPE_GROUP2COMPILE || table_type == TABLE_TYPE_GROUP2GROUP) { + continue; + } + + total_regex_num += regex_rule_num; + total_ipv6_num += ipv6_rule_num; + + long long rule_num = table_manager_runtime_rule_count(stat->ref_tbl_mgr, i); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_RULE_NUM], rule_num); + total_rule_num += rule_num; + + if (table_type == TABLE_TYPE_EXPR || table_type == TABLE_TYPE_EXPR_PLUS) { + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_REGEX_NUM], regex_rule_num); + long long stream_num = expr_runtime_stream_num(runtime); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_STREAM_NUM], stream_num); + total_stream_num += stream_num; + + long long input_bytes = expr_runtime_scan_bytes(runtime); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_SCAN_BYTES], input_bytes); + total_input_bytes += input_bytes; + } + + if (table_type == TABLE_TYPE_IP_PLUS) { + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_IPV6_NUM], ipv6_rule_num); + } + + if (1 == perf_on) { + long long scan_cpu_time = table_manager_runtime_scan_cpu_time(stat->ref_tbl_mgr, i); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_CPU_TIME], scan_cpu_time); + total_scan_cpu_time += scan_cpu_time; + } + + long long scan_cnt = table_manager_runtime_scan_count(stat->ref_tbl_mgr, i); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_SCAN_CNT], scan_cnt); + total_scan_cnt += scan_cnt; + + long long hit_cnt = table_manager_runtime_hit_count(stat->ref_tbl_mgr, i); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_TABLE_HIT_CNT], hit_cnt); + total_hit_cnt += hit_cnt; + + total_update_err += table_manager_runtime_update_err_count(stat->ref_tbl_mgr, i); + } + + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_RULE_NUM], total_rule_num); + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_REGEX_NUM], total_regex_num); + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_IPV6_NUM], total_ipv6_num); + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_STREAM_NUM], total_stream_num); + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_SCAN_CNT], total_scan_cnt); + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_SCAN_BYTES], total_input_bytes); + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_HIT_CNT], total_hit_cnt); + if (1 == perf_on) { + fieldstat_value_set(stat->fs_handle, stat->total_stat_id[COLUMN_TABLE_CPU_TIME], total_scan_cpu_time); + } + + stat->scan_cnt += total_scan_cnt; + stat->scan_bytes += total_input_bytes; + stat->update_err_cnt += total_update_err; + + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TOTAL_SCAN_LEN], stat->scan_bytes); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TOTAL_SCAN_CNT], stat->scan_cnt); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_UPDATE_ERR_CNT], stat->update_err_cnt); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_SCAN_ERR_CNT], stat->scan_err_cnt); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_ZOMBIE_RS_STREAM], stat->zombie_rs_stream); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_PLUGIN_CACHE_NUM], plugin_cache_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_PLUGIN_ACC_NUM], plugin_rule_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_NUM], g2c_rule_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_NOT_NUM], not_group_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_COMPILE_RULE_NUM], compile_rule_num); +} + +void maat_stat_output(struct maat_stat *stat, long long maat_version, int perf_on) +{ + if (NULL == stat) { + return; + } + + long long active_thread_num = alignment_int64_array_cnt(stat->thread_call_cnt, + stat->nr_worker_thread); + //TODO: maat_state_cnt + long long not_grp_hit_cnt = alignment_int64_array_sum(stat->not_grp_hit_cnt, + stat->nr_worker_thread); + size_t table_cnt = table_manager_table_count(stat->ref_tbl_mgr); + size_t garbage_q_len = maat_garbage_bin_get_size(stat->ref_garbage_bin); + + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_VERSION], maat_version); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_THRED_NUM], active_thread_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_TABLE_NUM], table_cnt); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_NOT_GROUP_HIT], not_grp_hit_cnt); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_CMD_Q_SIZE], stat->cmd_q_cnt); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_CMD_LINE_NUM], stat->line_cmd_acc_num); + fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GARBAGE_QSIZE], garbage_q_len); + + maat_fieldstat_table_row_output(stat, perf_on); + + fieldstat_passive_output(stat->fs_handle); +} \ No newline at end of file diff --git a/src/maat_table.c b/src/maat_table.c index b2c2f5d..aefb898 100644 --- a/src/maat_table.c +++ b/src/maat_table.c @@ -67,7 +67,8 @@ struct table_operations { void (*free_schema)(void *schema); - void *(*new_runtime)(void *schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, + void *(*new_runtime)(void *schema, size_t max_thread_num, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void (*free_runtime)(void *runtime); @@ -78,7 +79,15 @@ struct table_operations { int (*commit_runtime)(void *runtime, const char *table_name, long long maat_rt_version); - long long (*runtime_rule_count)(void *runtime); + long long (*rule_count)(void *runtime); + + long long (*scan_count)(void *runtime); + + long long (*scan_cpu_time)(void *runtime); + + long long (*hit_count)(void *runtime); + + long long (*update_err_count)(void *runtime); }; struct table_operations table_ops[TABLE_TYPE_MAX] = { @@ -90,7 +99,11 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = flag_runtime_free, .update_runtime = flag_runtime_update, .commit_runtime = flag_runtime_commit, - .runtime_rule_count = flag_runtime_rule_count + .rule_count = flag_runtime_rule_count, + .scan_count = flag_runtime_scan_count, + .scan_cpu_time = flag_runtime_scan_cpu_time, + .hit_count = flag_runtime_hit_count, + .update_err_count = flag_runtime_update_err_count }, { .type = TABLE_TYPE_FLAG_PLUS, @@ -100,7 +113,11 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = flag_runtime_free, .update_runtime = flag_runtime_update, .commit_runtime = flag_runtime_commit, - .runtime_rule_count = flag_runtime_rule_count + .rule_count = flag_runtime_rule_count, + .scan_count = flag_runtime_scan_count, + .scan_cpu_time = flag_runtime_scan_cpu_time, + .hit_count = flag_runtime_hit_count, + .update_err_count = flag_runtime_update_err_count }, { .type = TABLE_TYPE_EXPR, @@ -110,7 +127,11 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = expr_runtime_free, .update_runtime = expr_runtime_update, .commit_runtime = expr_runtime_commit, - .runtime_rule_count = expr_runtime_rule_count + .rule_count = expr_runtime_rule_count, + .scan_count = expr_runtime_scan_count, + .scan_cpu_time = expr_runtime_scan_cpu_time, + .hit_count = expr_runtime_hit_count, + .update_err_count = expr_runtime_update_err_count }, { .type = TABLE_TYPE_EXPR_PLUS, @@ -120,7 +141,11 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = expr_runtime_free, .update_runtime = expr_runtime_update, .commit_runtime = expr_runtime_commit, - .runtime_rule_count = expr_runtime_rule_count + .rule_count = expr_runtime_rule_count, + .scan_count = expr_runtime_scan_count, + .scan_cpu_time = expr_runtime_scan_cpu_time, + .hit_count = expr_runtime_hit_count, + .update_err_count = expr_runtime_update_err_count }, { .type = TABLE_TYPE_IP_PLUS, @@ -130,7 +155,11 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = ip_runtime_free, .update_runtime = ip_runtime_update, .commit_runtime = ip_runtime_commit, - .runtime_rule_count = ip_runtime_rule_count + .rule_count = ip_runtime_rule_count, + .scan_count = ip_runtime_scan_count, + .scan_cpu_time = ip_runtime_scan_cpu_time, + .hit_count = ip_runtime_hit_count, + .update_err_count = ip_runtime_update_err_count }, { .type = TABLE_TYPE_INTERVAL, @@ -140,7 +169,11 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = interval_runtime_free, .update_runtime = interval_runtime_update, .commit_runtime = interval_runtime_commit, - .runtime_rule_count = interval_runtime_rule_count + .rule_count = interval_runtime_rule_count, + .scan_count = interval_runtime_scan_count, + .scan_cpu_time = interval_runtime_scan_cpu_time, + .hit_count = interval_runtime_hit_count, + .update_err_count = interval_runtime_update_err_cnt }, { .type = TABLE_TYPE_INTERVAL_PLUS, @@ -150,7 +183,11 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = interval_runtime_free, .update_runtime = interval_runtime_update, .commit_runtime = interval_runtime_commit, - .runtime_rule_count = interval_runtime_rule_count + .rule_count = interval_runtime_rule_count, + .scan_count = interval_runtime_scan_count, + .scan_cpu_time = interval_runtime_scan_cpu_time, + .hit_count = interval_runtime_hit_count, + .update_err_count = interval_runtime_update_err_cnt }, { .type = TABLE_TYPE_PLUGIN, @@ -160,7 +197,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = plugin_runtime_free, .update_runtime = plugin_runtime_update, .commit_runtime = plugin_runtime_commit, - .runtime_rule_count = plugin_runtime_rule_count + .rule_count = plugin_runtime_rule_count, + .update_err_count = plugin_runtime_update_err_count }, { .type = TABLE_TYPE_IP_PLUGIN, @@ -170,7 +208,10 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = ip_plugin_runtime_free, .update_runtime = ip_plugin_runtime_update, .commit_runtime = ip_plugin_runtime_commit, - .runtime_rule_count = ip_plugin_runtime_rule_count + .rule_count = ip_plugin_runtime_rule_count, + .scan_count = ip_plugin_runtime_scan_count, + .scan_cpu_time = ip_plugin_runtime_scan_cpu_time, + .update_err_count = ip_plugin_runtime_update_err_count }, { .type = TABLE_TYPE_FQDN_PLUGIN, @@ -180,7 +221,10 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = fqdn_plugin_runtime_free, .update_runtime = fqdn_plugin_runtime_update, .commit_runtime = fqdn_plugin_runtime_commit, - .runtime_rule_count = fqdn_plugin_runtime_rule_count + .rule_count = fqdn_plugin_runtime_rule_count, + .scan_count = fqdn_plugin_runtime_scan_count, + .scan_cpu_time = fqdn_plugin_runtime_scan_cpu_time, + .update_err_count = fqdn_plugin_runtime_update_err_count }, { .type = TABLE_TYPE_BOOL_PLUGIN, @@ -190,7 +234,10 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = bool_plugin_runtime_free, .update_runtime = bool_plugin_runtime_update, .commit_runtime = bool_plugin_runtime_commit, - .runtime_rule_count = bool_plugin_runtime_rule_count + .rule_count = bool_plugin_runtime_rule_count, + .scan_count = bool_plugin_runtime_scan_count, + .scan_cpu_time = bool_plugin_runtime_scan_cpu_time, + .update_err_count = bool_plugin_runtime_update_err_count }, { .type = TABLE_TYPE_VIRTUAL, @@ -201,15 +248,6 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .update_runtime = NULL, .commit_runtime = NULL }, - { - .type = TABLE_TYPE_COMPOSITION, - .new_schema = NULL, - .free_schema = NULL, - .new_runtime = NULL, - .free_runtime = NULL, - .update_runtime = NULL, - .commit_runtime = NULL - }, { .type = TABLE_TYPE_COMPILE, .new_schema = compile_schema_new, @@ -218,7 +256,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = compile_runtime_free, .update_runtime = compile_runtime_update, .commit_runtime = compile_runtime_commit, - .runtime_rule_count = compile_runtime_rule_count + .rule_count = compile_runtime_rule_count, + .update_err_count = compile_runtime_update_err_count }, { .type = TABLE_TYPE_GROUP2GROUP, @@ -228,7 +267,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = group2group_runtime_free, .update_runtime = group2group_runtime_update, .commit_runtime = group2group_runtime_commit, - .runtime_rule_count = group2group_runtime_rule_count + .rule_count = group2group_runtime_rule_count, + .update_err_count = group2group_runtime_update_err_count }, { .type = TABLE_TYPE_GROUP2COMPILE, @@ -238,7 +278,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = group2compile_runtime_free, .update_runtime = group2compile_runtime_update, .commit_runtime = NULL, - .runtime_rule_count = group2compile_runtime_rule_count + .rule_count = group2compile_runtime_rule_count, + .update_err_count = group2compile_runtime_update_err_count } }; @@ -284,7 +325,6 @@ static void register_reserved_word(struct maat_kv_store *reserved_word_map) maat_kv_register(reserved_word_map, "bool_plugin", TABLE_TYPE_BOOL_PLUGIN); maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN); maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); - maat_kv_register(reserved_word_map, "composition", TABLE_TYPE_COMPOSITION); } static int register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id_map, @@ -572,7 +612,7 @@ next: return tbl_mgr; } -void *maat_table_runtime_new(void *schema, enum table_type table_type, int max_thread_num, +void *maat_table_runtime_new(void *schema, enum table_type table_type, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { void *runtime = NULL; @@ -591,7 +631,7 @@ void garbage_maat_table_runtime_free(void *runtime, void *arg) maat_table_runtime_free(runtime, type); } -int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_num, +int table_manager_runtime_create(struct table_manager *tbl_mgr, size_t max_thread_num, struct maat_garbage_bin *garbage_bin) { if (NULL == tbl_mgr) { @@ -699,11 +739,20 @@ void table_manager_destroy(struct table_manager *tbl_mgr) FREE(tbl_mgr); } -size_t table_manager_table_count(struct table_manager *tbl_mgr) +size_t table_manager_table_size(struct table_manager *tbl_mgr) { return MAX_TABLE_NUM; } +size_t table_manager_table_count(struct table_manager *tbl_mgr) +{ + if (NULL == tbl_mgr) { + return 0; + } + + return tbl_mgr->n_table; +} + int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name) { if (NULL == tbl_mgr || NULL == name) { @@ -722,6 +771,19 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name) return (int)table_id; } +const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id) +{ + if (NULL == tbl_mgr || table_id < 0) { + return NULL; + } + + if (NULL == tbl_mgr->tbl[table_id]) { + return NULL; + } + + return tbl_mgr->tbl[table_id]->table_name; +} + enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id) { if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) { @@ -944,5 +1006,97 @@ long long table_manager_runtime_rule_count(struct table_manager *tbl_mgr, int ta return 0; } - return table_ops[table_type].runtime_rule_count(runtime); + if (NULL == table_ops[table_type].rule_count) { + return 0; + } + + return table_ops[table_type].rule_count(runtime); +} + +long long table_manager_runtime_scan_count(struct table_manager *tbl_mgr, int table_id) +{ + void *runtime = table_manager_get_runtime(tbl_mgr, table_id); + if (NULL == runtime) { + return 0; + } + + enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); + if (table_type == TABLE_TYPE_INVALID) { + log_error(tbl_mgr->logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) table_type is invalid, can't get scan count", + __FUNCTION__, __LINE__, table_id); + return 0; + } + + if (NULL == table_ops[table_type].scan_count) { + return 0; + } + + return table_ops[table_type].scan_count(runtime); +} + +long long table_manager_runtime_scan_cpu_time(struct table_manager *tbl_mgr, int table_id) +{ + void *runtime = table_manager_get_runtime(tbl_mgr, table_id); + if (NULL == runtime) { + return 0; + } + + enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); + if (table_type == TABLE_TYPE_INVALID) { + log_error(tbl_mgr->logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) table_type is invalid, can't get scan cpu time", + __FUNCTION__, __LINE__, table_id); + return 0; + } + + if (NULL == table_ops[table_type].scan_cpu_time) { + return 0; + } + + return table_ops[table_type].scan_cpu_time(runtime); +} + +long long table_manager_runtime_hit_count(struct table_manager *tbl_mgr, int table_id) +{ + void *runtime = table_manager_get_runtime(tbl_mgr, table_id); + if (NULL == runtime) { + return 0; + } + + enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); + if (table_type == TABLE_TYPE_INVALID) { + log_error(tbl_mgr->logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) table_type is invalid, can't get hit count", + __FUNCTION__, __LINE__, table_id); + return 0; + } + + if (NULL == table_ops[table_type].hit_count) { + return 0; + } + + return table_ops[table_type].hit_count(runtime); +} + +long long table_manager_runtime_update_err_count(struct table_manager *tbl_mgr, int table_id) +{ + void *runtime = table_manager_get_runtime(tbl_mgr, table_id); + if (NULL == runtime) { + return 0; + } + + enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); + if (table_type == TABLE_TYPE_INVALID) { + log_error(tbl_mgr->logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) table_type is invalid, can't get hit count", + __FUNCTION__, __LINE__, table_id); + return 0; + } + + if (NULL == table_ops[table_type].update_err_count) { + return 0; + } + + return table_ops[table_type].update_err_count(runtime); } \ No newline at end of file diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 4dbffa8..b72d369 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -417,7 +417,6 @@ TEST_F(MaatIris, basic) { } #endif -#if 1 class MaatFlagScan : public testing::Test { protected: @@ -436,6 +435,7 @@ protected: struct maat_options *opts = maat_options_new(); maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); @@ -678,6 +678,7 @@ protected: struct maat_options *opts = maat_options_new(); maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); @@ -685,7 +686,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in MaatStringScan failed.", __FUNCTION__, __LINE__); } } @@ -1382,6 +1383,7 @@ protected: maat_options_set_foreign_cont_dir(opts, "./foreign_files/"); maat_options_set_rule_effect_interval_ms(opts, 0); maat_options_set_gc_timeout_ms(opts, 0); // start GC immediately + maat_options_set_stat_file(opts, "./stat.log"); _shared_maat_instance = maat_new(opts, table_info_path); maat_options_free(opts); } @@ -1473,6 +1475,7 @@ protected: struct maat_options *opts = maat_options_new(); maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); @@ -1480,7 +1483,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in MaatIPScan failed.", __FUNCTION__, __LINE__); } } @@ -1710,7 +1713,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in MaatIntervalScan failed.", __FUNCTION__, __LINE__); } } @@ -1801,7 +1804,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in NOTLogic failed.", __FUNCTION__, __LINE__); } } @@ -2093,7 +2096,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in PluginTable failed.", __FUNCTION__, __LINE__); } } @@ -2249,7 +2252,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in IPPluginTable failed.", __FUNCTION__, __LINE__); } } @@ -2390,7 +2393,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in FQDNPluginTable failed.", __FUNCTION__, __LINE__); } } @@ -2558,7 +2561,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in BoolPluginTable failed.", __FUNCTION__, __LINE__); } } @@ -2648,7 +2651,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in VirtualTable failed.", __FUNCTION__, __LINE__); } } @@ -2709,7 +2712,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in CompileTable failed.", __FUNCTION__, __LINE__); } } @@ -2883,7 +2886,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in Policy failed.", __FUNCTION__, __LINE__); } } @@ -3132,7 +3135,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in TableInfo failed.", __FUNCTION__, __LINE__); } } @@ -3296,7 +3299,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in HierarchyTest failed.", __FUNCTION__, __LINE__); } } @@ -4477,7 +4480,7 @@ TEST_F(MaatCmdTest, PluginEXData) { ret = maat_cmd_set_line(maat_instance, &line_rule); EXPECT_GT(ret, 0); - sleep(WAIT_FOR_EFFECTIVE_S); + sleep(WAIT_FOR_EFFECTIVE_S * 2); const char *key2 = "192.168.0.2"; uinfo = (struct user_info *)maat_plugin_table_get_ex_data(maat_instance, table_id, key2); @@ -5484,7 +5487,7 @@ protected: maat_options_free(opts); if (NULL == _shared_maat_instance) { log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", + "[%s:%d] create maat instance in MaatRollbackTest failed.", __FUNCTION__, __LINE__); } } @@ -5635,7 +5638,7 @@ TEST_F(MaatRollbackTest, FullConfigRollback) { maat_state_free(state); state = NULL; } -#endif + int main(int argc, char ** argv) { int ret=0;