重构Maat table相关代码。

This commit is contained in:
zhengchao
2019-07-25 14:49:11 +06:00
parent c189b90e6d
commit 2909cb1997
10 changed files with 562 additions and 439 deletions

View File

@@ -0,0 +1,12 @@
#define MAX_TABLE_NUM 256
#define MAX_TABLE_LINE_SIZE (1024*16)
#define MAX_EXPR_KEYLEN 1024
#define MAX_DISTRICT_LEN 64
#define MAX_SCANNER_HIT_NUM 64
#define MAX_GROUP_CACHE 128
#define MAX_FAILED_NUM 128
#define MAX_MAAT_STAT_NUM 64

View File

@@ -1,9 +1,9 @@
#ifndef H_MAAT_RULE_INTERNAL_H_INCLUDE
#define H_MAAT_RULE_INTERNAL_H_INCLUDE
#pragma once
#include "Maat_rule.h"
#include "Maat_command.h"
#include "Maat_table_description.h"
#include "Maat_limits.h"
#include "Maat_table.h"
#include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h>
@@ -27,17 +27,7 @@ extern const char *maat_module;
#define mr_group_id_var "SEQUENCE_GROUP"
#define MAX_TABLE_NUM 256
#define MAX_TABLE_LINE_SIZE (1024*16)
#define MAX_EXPR_KEYLEN 1024
#define MAX_DISTRICT_LEN 64
#define MAX_SCANNER_HIT_NUM 64
#define MAX_GROUP_CACHE 128
#define MAX_FAILED_NUM 128
#define MAX_MAAT_STAT_NUM 64
typedef void* rule_scanner_t;
@@ -210,11 +200,12 @@ struct iconv_handle_t
struct _stream_para_t
{
struct _Maat_feather_t* feather;
const struct Maat_table_desc* p_real_table;
int virtual_table_id;
int version;
int thread_num;
int max_cross_size;
int caching_size;
unsigned short table_id;
char do_merge;
char do_expr:4;
char do_regex:4;
@@ -355,7 +346,6 @@ struct _Maat_feather_t
MESA_lqueue_head garbage_q;
int table_cnt;
int DEFERRED_LOAD_ON;
int GROUP_MODE_ON;
int REDIS_MODE_ON;
enum data_source input_mode;
union
@@ -471,7 +461,6 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(struct Maat_compile_group_relation* compile_rule, struct bool_expr* a_set, unsigned char *has_not);
int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
void maat_start_cb(long long new_version,int update_type,void*u_para);
int maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para);
@@ -517,6 +506,3 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
_Maat_feather_t* feather);
#endif

View File

@@ -1,5 +1,8 @@
#pragma once
#include <MESA/MESA_htable.h>
#include "Maat_rule.h"
#include "Maat_limits.h"
#define MAX_COMPILE_EX_DATA_NUM 2
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_PLUGIN_PER_TABLE 32
@@ -7,6 +10,7 @@
#define MAX_CONJUNCTION_TABLE_NUM 8
#define MAX_TABLE_NAME_LEN 256
enum USER_REGION_ENCODE
{
USER_REGION_ENCODE_NONE=0,
@@ -93,6 +97,7 @@ struct ip_table_desc
struct virtual_table_desc
{
int real_table_id;
char real_table_name[MAX_TABLE_NAME_LEN];
};
struct Maat_table_desc
{
@@ -115,4 +120,7 @@ struct Maat_table_desc
unsigned long long unmatch_tag_cnt;
int stat_line_id;
};
int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_table, const char* table_info_path, void* logger);
MESA_htable_handle Maat_table_build(struct Maat_table_desc** p_table_info, size_t n_table, char* compile_tn, size_t n_ctn, char* group_tn, size_t n_gtn, void* logger);
void Maat_table_clear(struct Maat_table_desc** p_table_info, size_t n_table);

View File

@@ -1,4 +1,5 @@
#pragma once
#include "Maat_rule.h"
#include <arpa/inet.h>
#include <unistd.h>
#include <sys/syscall.h>
@@ -69,5 +70,7 @@ int system_cmd_mv(const char* src_file,const char*dst_file);
int system_cmd_cp(const char* src_file,const char*dst_file);
char* md5_file(const char* filename, char* md5string);
int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len);
const char** charset_get_all_name(void);
const char* charset_get_name(enum MAAT_CHARSET charset);