将scanner中的运行态数据拆分到Maat_table_runtime.cpp中。
This commit is contained in:
zhengchao
@@ -1,4 +1,3 @@
|
||||
#define MAX_TABLE_NUM 256
|
||||
#define MAX_TABLE_LINE_SIZE (1024*16)
|
||||
#define MAX_EXPR_KEYLEN 1024
|
||||
#define MAX_DISTRICT_LEN 64
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#include "Maat_command.h"
|
||||
#include "Maat_limits.h"
|
||||
#include "Maat_table.h"
|
||||
#include "Maat_table_runtime.h"
|
||||
|
||||
#include <MESA/MESA_htable.h>
|
||||
#include <MESA/MESA_list_queue.h>
|
||||
@@ -218,54 +219,7 @@ struct _stream_para_t
|
||||
pthread_mutex_t fuzzy_mutex;
|
||||
unsigned char query_point[8];
|
||||
};
|
||||
struct similar_runtime
|
||||
{
|
||||
GIE_handle_t* gie_handle;
|
||||
MESA_lqueue_head update_q;
|
||||
};
|
||||
|
||||
struct plugin_runtime
|
||||
{
|
||||
dynamic_array_t *cache_lines;
|
||||
long long cache_line_num;
|
||||
long long acc_line_num;
|
||||
long long cache_size;
|
||||
MESA_htable_handle key2ex_hash;
|
||||
};
|
||||
struct expr_runtime
|
||||
{
|
||||
long long expr_rule_cnt; //expr_type=0,1,3
|
||||
long long regex_rule_cnt; //expr_type=2
|
||||
};
|
||||
struct ip_runtime
|
||||
{
|
||||
long long ipv4_rule_cnt;
|
||||
long long ipv6_rule_cnt;
|
||||
|
||||
};
|
||||
struct group_runtime
|
||||
{
|
||||
long long not_flag_group;
|
||||
};
|
||||
struct Maat_table_runtime
|
||||
{
|
||||
enum MAAT_TABLE_TYPE table_type;
|
||||
long origin_rule_num;
|
||||
union
|
||||
{
|
||||
struct similar_runtime similar; //for digest and similarity
|
||||
struct plugin_runtime plugin;
|
||||
struct expr_runtime expr;
|
||||
struct ip_runtime ip;
|
||||
struct group_runtime group;
|
||||
void * other;
|
||||
};
|
||||
mcore_long_t scan_cnt;
|
||||
mcore_long_t scan_cpu_time; //nano
|
||||
mcore_long_t input_bytes;
|
||||
mcore_long_t stream_num;
|
||||
mcore_long_t hit_cnt;
|
||||
};
|
||||
struct rule_tag
|
||||
{
|
||||
char* tag_name;
|
||||
@@ -280,9 +234,10 @@ struct Maat_scanner
|
||||
size_t gie_update_q_size;
|
||||
size_t to_update_group_cnt;
|
||||
size_t to_update_compile_cnt;
|
||||
|
||||
struct Maat_table_runtime* table_rt[MAX_TABLE_NUM];
|
||||
|
||||
struct Maat_table_runtime_manager* table_rt_mgr;
|
||||
size_t max_table_num;
|
||||
|
||||
MESA_htable_handle region_hash; //key: region_id, value: struct region_group_relation*
|
||||
MESA_htable_handle exprid_hash; //key: expr_id, value: int array_idx of Maat_group_inner->regions;
|
||||
MESA_htable_handle group_hash; //key: group_id, value: struct Maat_group_inner*
|
||||
@@ -344,7 +299,8 @@ struct _Maat_feather_t
|
||||
struct Maat_scanner *scanner;
|
||||
struct Maat_scanner *update_tmp_scanner;
|
||||
MESA_lqueue_head garbage_q;
|
||||
int table_cnt;
|
||||
struct Maat_table_manager* table_mgr;
|
||||
|
||||
int DEFERRED_LOAD_ON;
|
||||
int REDIS_MODE_ON;
|
||||
enum data_source input_mode;
|
||||
@@ -360,8 +316,7 @@ struct _Maat_feather_t
|
||||
int cumulative_update_off;
|
||||
int stat_on;
|
||||
int perf_on;
|
||||
struct Maat_table_desc *p_table_info[MAX_TABLE_NUM];
|
||||
MESA_htable_handle map_tablename2id;
|
||||
|
||||
void* logger;
|
||||
long long maat_version;
|
||||
long long last_full_version;
|
||||
@@ -394,8 +349,6 @@ struct _Maat_feather_t
|
||||
int foreign_cont_linger;
|
||||
//internal states
|
||||
long long new_version;
|
||||
int active_plugin_table_num;
|
||||
int is_last_plugin_table_updating;
|
||||
|
||||
//for scanner independent stat>>>>
|
||||
int backgroud_update_enabled;
|
||||
|
||||
@@ -20,6 +20,7 @@ enum USER_REGION_ENCODE
|
||||
|
||||
enum MAAT_TABLE_TYPE
|
||||
{
|
||||
TABLE_TYPE_INVALID=-1,
|
||||
TABLE_TYPE_EXPR=0,
|
||||
TABLE_TYPE_IP,
|
||||
TABLE_TYPE_IP_PLUS,
|
||||
@@ -50,6 +51,21 @@ struct compile_table_desc
|
||||
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
|
||||
};
|
||||
|
||||
struct expr_table_desc
|
||||
{
|
||||
enum MAAT_CHARSET src_charset;
|
||||
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
|
||||
int src_charset_in_dst;
|
||||
int do_charset_merge;
|
||||
int cross_cache_size;
|
||||
int quick_expr_switch;//obsolete since 20190401
|
||||
long long iconv_err_cnt;
|
||||
};
|
||||
struct virtual_table_desc
|
||||
{
|
||||
int real_table_id;
|
||||
char real_table_name[MAX_TABLE_NAME_LEN];
|
||||
};
|
||||
struct plugin_table_callback_desc
|
||||
{
|
||||
Maat_start_callback_t *start;
|
||||
@@ -79,26 +95,7 @@ struct plugin_table_desc
|
||||
struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE];
|
||||
struct plugin_table_ex_data_desc ex_desc;
|
||||
};
|
||||
struct expr_table_desc
|
||||
{
|
||||
enum MAAT_CHARSET src_charset;
|
||||
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
|
||||
int src_charset_in_dst;
|
||||
int do_charset_merge;
|
||||
int cross_cache_size;
|
||||
int quick_expr_switch;//obsolete since 20190401
|
||||
long long iconv_err_cnt;
|
||||
};
|
||||
struct ip_table_desc
|
||||
{
|
||||
int ipv4_rule_cnt;
|
||||
int ipv6_rule_cnt;
|
||||
};
|
||||
struct virtual_table_desc
|
||||
{
|
||||
int real_table_id;
|
||||
char real_table_name[MAX_TABLE_NAME_LEN];
|
||||
};
|
||||
|
||||
struct Maat_table_desc
|
||||
{
|
||||
int table_id;
|
||||
@@ -110,17 +107,52 @@ struct Maat_table_desc
|
||||
{
|
||||
struct compile_table_desc compile;
|
||||
struct expr_table_desc expr;
|
||||
struct ip_table_desc ip;
|
||||
struct plugin_table_desc plugin;
|
||||
struct virtual_table_desc virtual_table;;
|
||||
void* others;//group, interval and digest don't have sperate description info.
|
||||
void* others;//group, ip, interval and digest don't have sperate description info.
|
||||
};
|
||||
//for stat>>>>>>>>
|
||||
unsigned long long udpate_err_cnt;
|
||||
unsigned long long unmatch_tag_cnt;
|
||||
int stat_line_id;
|
||||
};
|
||||
int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_table, const char* table_info_path, void* logger);
|
||||
MESA_htable_handle Maat_table_build(struct Maat_table_desc** p_table_info, size_t n_table, char* compile_tn, size_t n_ctn, char* group_tn, size_t n_gtn, void* logger);
|
||||
void Maat_table_clear(struct Maat_table_desc** p_table_info, size_t n_table);
|
||||
struct Maat_table_manager;
|
||||
struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger);
|
||||
void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr);
|
||||
size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr);
|
||||
size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr);
|
||||
|
||||
struct Maat_table_desc * Maat_table_get_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id);
|
||||
struct Maat_table_desc * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id);
|
||||
|
||||
int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
|
||||
int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
|
||||
int table_id,
|
||||
Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
|
||||
Maat_update_callback_t *update,//table line ,u_para
|
||||
Maat_finish_callback_t *finish,//u_para
|
||||
void* u_para);
|
||||
int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
|
||||
int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
|
||||
const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id);
|
||||
enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id);
|
||||
|
||||
int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name,
|
||||
Maat_rule_EX_new_func_t *new_func,
|
||||
Maat_rule_EX_free_func_t* free_func,
|
||||
Maat_rule_EX_dup_func_t* dup_func,
|
||||
long argl, void *argp);
|
||||
struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx);
|
||||
int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id,
|
||||
Maat_plugin_EX_new_func_t* new_func,
|
||||
Maat_plugin_EX_free_func_t* free_func,
|
||||
Maat_plugin_EX_dup_func_t* dup_func,
|
||||
Maat_plugin_EX_key2index_func_t* key2index_func,
|
||||
long argl, void *argp);
|
||||
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type);
|
||||
void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr);
|
||||
|
||||
int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr);
|
||||
struct Maat_table_desc* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
|
||||
void Maat_table_set_updating_name(struct Maat_table_desc* p_table, const char* table_name);
|
||||
|
||||
|
||||
69
src/inc_internal/Maat_table_runtime.h
Normal file
69
src/inc_internal/Maat_table_runtime.h
Normal file
@@ -0,0 +1,69 @@
|
||||
#include "Maat_table.h"
|
||||
#include "gram_index_engine.h"
|
||||
#include "alignment_int64.h"
|
||||
#include "dynamic_array.h"
|
||||
#include <MESA/MESA_htable.h>
|
||||
#include <MESA/MESA_list_queue.h>
|
||||
|
||||
struct similar_runtime
|
||||
{
|
||||
GIE_handle_t* gie_handle;
|
||||
MESA_lqueue_head update_q;
|
||||
};
|
||||
|
||||
struct plugin_runtime
|
||||
{
|
||||
dynamic_array_t *cache_lines;
|
||||
long long cache_line_num;
|
||||
long long acc_line_num;
|
||||
long long cache_size;
|
||||
MESA_htable_handle key2ex_hash;
|
||||
};
|
||||
struct expr_runtime
|
||||
{
|
||||
long long expr_rule_cnt; //expr_type=0,1,3
|
||||
long long regex_rule_cnt; //expr_type=2
|
||||
};
|
||||
struct ip_runtime
|
||||
{
|
||||
long long ipv4_rule_cnt;
|
||||
long long ipv6_rule_cnt;
|
||||
|
||||
};
|
||||
struct group_runtime
|
||||
{
|
||||
long long not_flag_group;
|
||||
};
|
||||
struct Maat_table_runtime
|
||||
{
|
||||
enum MAAT_TABLE_TYPE table_type;
|
||||
long origin_rule_num;
|
||||
union
|
||||
{
|
||||
struct similar_runtime similar; //for digest and similarity
|
||||
struct plugin_runtime plugin;
|
||||
struct expr_runtime expr;
|
||||
struct ip_runtime ip;
|
||||
struct group_runtime group;
|
||||
void * other;
|
||||
};
|
||||
mcore_long_t scan_cnt;
|
||||
mcore_long_t scan_cpu_time; //nano
|
||||
mcore_long_t input_bytes;
|
||||
mcore_long_t stream_num;
|
||||
mcore_long_t hit_cnt;
|
||||
};
|
||||
struct Maat_table_runtime_manager;
|
||||
struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num);
|
||||
void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr);
|
||||
struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id);
|
||||
long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt);
|
||||
const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line);
|
||||
|
||||
int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, void* logger);
|
||||
MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, const char* key);
|
||||
void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag);
|
||||
void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id);
|
||||
int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt);
|
||||
|
||||
|
||||
@@ -72,5 +72,6 @@ char* md5_file(const char* filename, char* md5string);
|
||||
int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len);
|
||||
const char** charset_get_all_name(void);
|
||||
const char* charset_get_name(enum MAAT_CHARSET charset);
|
||||
int lqueue_destroy_cb(void *data, long data_len, void *arg);
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user