gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support maat stat

Browse Source
This commit is contained in:
liuwentan
2023-04-20 15:34:56 +08:00
parent ff4666ca9d
commit af6df5951a
35 changed files with 1758 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/inc_internal/maat_bool_plugin.h
View File

@@ -37,7 +37,7 @@ int bool_plugin_table_set_ex_container_schema(void *bool_plugin_schema, int tabl
struct ex_container_schema *bool_plugin_table_get_ex_container_schema(void *bool_plugin_schema);
/* ip plugin runtime API */
void *bool_plugin_runtime_new(void *bool_plugin_schema, int max_thread_num,
void *bool_plugin_runtime_new(void *bool_plugin_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void bool_plugin_runtime_free(void *bool_plugin_runtime);
@@ -45,7 +45,8 @@ void bool_plugin_runtime_free(void *bool_plugin_runtime);
int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_schema,
const char *table_name, const char *line, int valid_column);
int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name, long long maat_rt_version);
int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name,
long long maat_rt_version);
long long bool_plugin_runtime_rule_count(void *bool_plugin_runtime);
@@ -54,6 +55,15 @@ struct ex_data_runtime *bool_plugin_runtime_get_ex_data_rt(void *bool_plugin_run
int bool_plugin_runtime_get_ex_data(void *bool_plugin_runtime, unsigned long long *item_ids,
size_t n_item, void **ex_data_array, size_t n_ex_data);
void bool_plugin_runtime_perf_stat(void *bool_plugin_runtime, struct timespec *start,
struct timespec *end, int thread_id);
long long bool_plugin_runtime_scan_count(void *bool_plugin_runtime);
long long bool_plugin_runtime_scan_cpu_time(void *bool_plugin_runtime);
long long bool_plugin_runtime_update_err_count(void *bool_plugin_runtime);
#ifdef __cplusplus
}
#endif

3
src/inc_internal/maat_common.h
View File

@@ -26,11 +26,14 @@ struct maat_options {
char instance_name[NAME_MAX];
char foreign_cont_dir[NAME_MAX];
char log_path[PATH_MAX];
char stat_file[PATH_MAX];
size_t nr_worker_threads;
char *accept_tags;
int rule_effect_interval_ms;
int rule_update_checking_interval_ms;
int gc_timeout_ms;
int stat_on;
int perf_on;
int deferred_load_on;
int log_level;
enum data_source input_mode;

10
src/inc_internal/maat_compile.h
View File

@@ -44,7 +44,7 @@ int compile_table_set_ex_data_schema(struct compile_schema *compile_schema, int
long argl, void *argp);
/* compile runtime API */
void *compile_runtime_new(void *compile_schema, int max_thread_num,
void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void compile_runtime_free(void *compile_runtime);
@@ -58,6 +58,8 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name, long l
long long compile_runtime_rule_count(void *compile_runtime);
long long compile_runtime_update_err_count(void *compile_runtime);
int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile_ids,
size_t compile_ids_size, struct maat_state *state);
@@ -73,7 +75,7 @@ void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt,
struct compile_schema *compile_schema);
/* group2compile runtime API */
void *group2compile_runtime_new(void *g2c_schema, int max_thread_num,
void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, void *g2g_runtime);
@@ -82,8 +84,12 @@ void group2compile_runtime_free(void *g2c_runtime);
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *table_name,
const char *line, int valid_column);
long long group2compile_runtime_not_group_count(void *g2c_runtime);
long long group2compile_runtime_rule_count(void *g2c_runtime);
long long group2compile_runtime_update_err_count(void *g2c_runtime);
/* maat compile state API */
struct maat_compile_state;
struct maat_compile_state *maat_compile_state_new(int thread_id);

27
src/inc_internal/maat_expr.h
View File

@@ -27,7 +27,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void expr_schema_free(void *expr_schema);
/* expr runtime API */
void *expr_runtime_new(void *expr_schema, int max_thread_num,
void *expr_runtime_new(void *expr_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void expr_runtime_free(void *expr_runtime);
@@ -39,6 +39,8 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, long long ma
long long expr_runtime_rule_count(void *expr_runtime);
long long expr_runtime_regex_rule_count(void *expr_runtime);
long long expr_runtime_get_version(void *expr_runtime);
/* expr runtime scan API */
@@ -55,13 +57,30 @@ struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt,
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_stream *s_handle,
const char *data, size_t data_len,
int vtable_id, struct maat_state *state);
void expr_runtime_stream_close(struct adapter_hs_stream *s_handle);
void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id,
struct adapter_hs_stream *s_handle);
int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *district,
size_t district_len, long long *district_id);
void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id);
long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread);
void expr_runtime_hit_inc(struct expr_runtime *expr_rt, int thread_id);
void expr_runtime_perf_stat(struct expr_runtime *flag_rt, size_t scan_len,
struct timespec *start, struct timespec *end,
int thread_id);
long long expr_runtime_scan_count(void *expr_runtime);
long long expr_runtime_scan_cpu_time(void *expr_runtime);
long long expr_runtime_hit_count(void *expr_runtime);
long long expr_runtime_update_err_count(void *expr_runtime);
long long expr_runtime_scan_bytes(struct expr_runtime *expr_rt);
long long expr_runtime_stream_num(struct expr_runtime *expr_rt);
#ifdef __cplusplus
}

16
src/inc_internal/maat_flag.h
View File

@@ -29,7 +29,7 @@ void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void flag_schema_free(void *flag_schema);
/* flag runtime API */
void *flag_runtime_new(void *flag_schema, int max_thread_num,
void *flag_runtime_new(void *flag_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void flag_runtime_free(void *flag_runtime);
@@ -53,8 +53,18 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, long long fla
int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *district,
size_t district_len, long long *district_id);
void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id);
long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread);
void flag_runtime_hit_inc(struct flag_runtime *flag_rt, int thread_id);
void flag_runtime_perf_stat(struct flag_runtime *flag_rt, struct timespec *start,
struct timespec *end, int thread_id);
long long flag_runtime_scan_count(void *flag_runtime);
long long flag_runtime_scan_cpu_time(void *flag_runtime);
long long flag_runtime_hit_count(void *flag_runtime);
long long flag_runtime_update_err_count(void *flag_runtime);
#ifdef __cplusplus
}

14
src/inc_internal/maat_fqdn_plugin.h
View File

@@ -39,7 +39,7 @@ int fqdn_plugin_table_set_ex_container_schema(void *fqdn_plugin_schema, int tabl
struct ex_container_schema *fqdn_plugin_table_get_ex_container_schema(void *fqdn_plugin_schema);
/* fqdn plugin runtime API */
void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, int max_thread_num,
void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void fqdn_plugin_runtime_free(void *fqdn_plugin_runtime);
@@ -47,7 +47,8 @@ void fqdn_plugin_runtime_free(void *fqdn_plugin_runtime);
int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_schema,
const char *table_name, const char *line, int valid_column);
int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name, long long maat_rt_version);
int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name,
long long maat_rt_version);
long long fqdn_plugin_runtime_rule_count(void *fqdn_plugin_runtime);
@@ -58,6 +59,15 @@ int fqdn_plugin_runtime_get_ex_data(void *fqdn_plugin_runtime, const char *fqdn,
void fqdn_rule_free(struct FQDN_rule *fqdn_rule);
void fqdn_plugin_runtime_perf_stat(void *fqdn_plugin_runtime, struct timespec *start,
struct timespec *end, int thread_id);
long long fqdn_plugin_runtime_scan_count(void *fqdn_plugin_runtime);
long long fqdn_plugin_runtime_scan_cpu_time(void *fqdn_plugin_runtime);
long long fqdn_plugin_runtime_update_err_count(void *fqdn_plugin_runtime);
#ifdef __cplusplus
}
#endif

4
src/inc_internal/maat_group.h
View File

@@ -31,7 +31,7 @@ void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void group2group_schema_free(void *g2g_schema);
/* group2group runtime API */
void *group2group_runtime_new(void *g2g_schema, int max_thread_num,
void *group2group_runtime_new(void *g2g_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void group2group_runtime_free(void *g2g_runtime);
@@ -53,6 +53,8 @@ int group2group_runtime_commit(void *g2g_runtime, const char *table_name, long l
long long group2group_runtime_rule_count(void *g2g_runtime);
long long group2group_runtime_update_err_count(void *g2g_runtime);
#ifdef __cplusplus
}
#endif

16
src/inc_internal/maat_interval.h
View File

@@ -28,7 +28,7 @@ void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void interval_schema_free(void *interval_schema);
/* interval runtime API */
void *interval_runtime_new(void *interval_schema, int max_thread_num,
void *interval_runtime_new(void *interval_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void interval_runtime_free(void *interval_runtime);
@@ -52,8 +52,18 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
int interval_runtime_set_scan_district(struct interval_runtime *interval_rt, const char *district,
size_t district_len, long long *district_id);
void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id);
long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread);
void interval_runtime_hit_inc(struct interval_runtime *interval_rt, int thread_id);
void interval_runtime_perf_stat(struct interval_runtime *interval_rt, struct timespec *start,
struct timespec *end, int thread_id);
long long interval_runtime_scan_count(void *interval_runtime);
long long interval_runtime_scan_cpu_time(void *interval_runtime);
long long interval_runtime_hit_count(void *interval_runtime);
long long interval_runtime_update_err_cnt(void *interval_runtime);
#ifdef __cplusplus
}

18
src/inc_internal/maat_ip.h
View File

@@ -26,7 +26,7 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void ip_schema_free(void *ip_schema);
/* ip runtime API */
void *ip_runtime_new(void *ip_schema, int max_thread_num,
void *ip_runtime_new(void *ip_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void ip_runtime_free(void *ip_runtime);
@@ -38,13 +38,25 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_r
long long ip_runtime_rule_count(void *ip_runtime);
long long ip_runtime_ipv6_rule_count(void *ip_runtime);
/* ip runtime scan API */
int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
uint8_t *ip_addr, uint16_t port, int proto, int vtable_id,
struct maat_state *state);
void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id);
long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread);
void ip_runtime_hit_inc(struct ip_runtime *ip_rt, int thread_id);
void ip_runtime_perf_stat(struct ip_runtime *ip_rt, struct timespec *start,
struct timespec *end, int thread_id);
long long ip_runtime_scan_count(void *ip_runtime);
long long ip_runtime_scan_cpu_time(void *ip_runtime);
long long ip_runtime_hit_count(void *ip_runtime);
long long ip_runtime_update_err_count(void *ip_runtime);
#ifdef __cplusplus
}

11
src/inc_internal/maat_ip_plugin.h
View File

@@ -38,7 +38,7 @@ int ip_plugin_table_set_ex_container_schema(void *ip_plugin_schema, int table_id
struct ex_container_schema *ip_plugin_table_get_ex_container_schema(void *ip_plugin_schema);
/* ip plugin runtime API */
void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num,
void *ip_plugin_runtime_new(void *ip_plugin_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void ip_plugin_runtime_free(void *ip_plugin_runtime);
@@ -55,6 +55,15 @@ struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime
int ip_plugin_runtime_get_ex_data(void *ip_plugin_runtime, const struct ip_addr *ip_addr,
void **ex_data_array, size_t n_ex_data_array);
void ip_plugin_runtime_perf_stat(void *ip_plugin_runtime, struct timespec *start,
struct timespec *end, int thread_id);
long long ip_plugin_runtime_scan_count(void *ip_plugin_runtime);
long long ip_plugin_runtime_scan_cpu_time(void *ip_plugin_runtime);
long long ip_plugin_runtime_update_err_count(void *ip_plugin_runtime);
#ifdef __cplusplus
}
#endif

4
src/inc_internal/maat_limits.h
View File

@@ -17,7 +17,9 @@ extern "C"
#endif
#define MAX_KEYWORDS_STR 1024
#define MAX_MAAT_STAT_NUM 64
#ifdef __cplusplus
}
#endif

4
src/inc_internal/maat_plugin.h
View File

@@ -50,7 +50,7 @@ int plugin_table_set_ex_container_schema(void *plugin_schema, int table_id,
struct ex_container_schema *plugin_table_get_ex_container_schema(void *plugin_schema);
/* plugin runtime API */
void *plugin_runtime_new(void *plugin_schema, int max_thread_num,
void *plugin_runtime_new(void *plugin_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void plugin_runtime_free(void *plugin_runtime);
@@ -62,6 +62,8 @@ int plugin_runtime_commit(void *plugin_runtime, const char *table_name, long lon
long long plugin_runtime_rule_count(void *plugin_runtime);
long long plugin_runtime_update_err_count(void *plugin_runtime);
struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime);
size_t plugin_runtime_cached_row_count(void *plugin_runtime);

42
src/inc_internal/maat_rule.h
View File

@@ -27,12 +27,15 @@ extern "C"
#include "hiredis/hiredis.h"
#include "uthash/uthash.h"
#include "log/log.h"
#include "fieldstat.h"
#include "maat_command.h"
//#include "ip_matcher.h"
#include "maat_limits.h"
#include "maat.h"
#include "maat_kv.h"
#include "maat_table.h"
#include "maat_virtual.h"
#include "maat_stat.h"
#define MAX_TABLE_NUM 1024
#define MAX_COMPILE_TABLE_NUM 16
@@ -168,6 +171,31 @@ struct rule_tag {
char *tag_val;
};
struct maat_stat {
char stat_file[PATH_MAX];
struct fieldstat_instance *fs_handle;
int total_stat_id[MAX_MAAT_STAT_NUM];
int fs_status_id[MAX_MAAT_STAT_NUM];
int fs_column_id[MAX_TABLE_NUM][MAX_MAAT_STAT_NUM];
struct log_handle *logger;
struct table_manager *ref_tbl_mgr;
struct maat_garbage_bin *ref_garbage_bin;
size_t nr_worker_thread;
int cmd_q_cnt;
long long *thread_call_cnt;
long long *hit_cnt;
long long *not_grp_hit_cnt;
long long scan_bytes;
long long scan_cnt;
long long update_err_cnt;
long long scan_err_cnt;
long long zombie_rs_stream;
long long line_cmd_acc_num;
};
struct maat {
char instance_name[NAME_MAX];
@@ -188,7 +216,7 @@ struct maat {
int is_running;
pthread_mutex_t background_update_mutex;
int nr_worker_thread;
size_t nr_worker_thread;
long long maat_version;
long long last_full_version;
@@ -200,6 +228,8 @@ struct maat {
int cumulative_update_off; //Default: cumulative update on
int stat_on;
int perf_on;
struct maat_garbage_bin *garbage_bin;
int default_compile_table_id;
@@ -216,13 +246,7 @@ struct maat {
long long new_version;
/* statistics */
long long line_cmd_acc_num;
long long *thread_call_cnt;
long long *hit_cnt;
long long *not_grp_hit_cnt;
long long scan_err_cnt;
struct maat_stat *stat;
};
enum district_flag {

35
src/inc_internal/maat_stat.h Normal file
View File

@@ -0,0 +1,35 @@
/*
**********************************************************************************************
* File: maat_stat.h
* Description: maat statistics
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_STAT_H_
#define _MAAT_STAT_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include "maat_rule.h"
struct maat_stat *maat_stat_new(const char *stat_file, size_t max_thread_num,
struct log_handle *logger);
void maat_stat_free(struct maat_stat *stat);
void maat_stat_init(struct maat_stat *stat, struct table_manager *tbl_mgr,
struct maat_garbage_bin *garbage_bin, const char *stat_inst_name);
void maat_stat_output(struct maat_stat *stat, long long maat_version, int perf_on);
#ifdef __cplusplus
}
#endif
#endif

15
src/inc_internal/maat_table.h
View File

@@ -37,7 +37,6 @@ enum table_type {
TABLE_TYPE_BOOL_PLUGIN,
//above are physical table
TABLE_TYPE_VIRTUAL,
TABLE_TYPE_COMPOSITION,
TABLE_TYPE_COMPILE,
TABLE_TYPE_GROUP2GROUP,
TABLE_TYPE_GROUP2COMPILE,
@@ -49,14 +48,18 @@ struct table_manager;
struct table_manager *
table_manager_create(const char *table_info_path, const char *accept_tags,
struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_num,
int table_manager_runtime_create(struct table_manager *tbl_mgr, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin);
void table_manager_runtime_destroy(struct table_manager *tbl_mgr);
void table_manager_destroy(struct table_manager *tbl_mgr);
size_t table_manager_table_size(struct table_manager *tbl_mgr);
size_t table_manager_table_count(struct table_manager *tbl_mgr);
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name);
const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id);
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id);
int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr);
@@ -79,6 +82,14 @@ void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id,
long long table_manager_runtime_rule_count(struct table_manager *tbl_mgr, int table_id);
long long table_manager_runtime_scan_count(struct table_manager *tbl_mgr, int table_id);
long long table_manager_runtime_scan_cpu_time(struct table_manager *tbl_mgr, int table_id);
long long table_manager_runtime_hit_count(struct table_manager *tbl_mgr, int table_id);
long long table_manager_runtime_update_err_count(struct table_manager *tbl_mgr, int table_id);
#ifdef __cplusplus
}
#endif