gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor table schema and runtime to support boolean expression plugin.

Browse Source
This commit is contained in:
zhengchao
2022-08-23 15:04:55 +08:00
parent 47a71011db
commit ddd36c2969
16 changed files with 646 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/inc_internal/Maat_rule_internal.h
View File

@@ -3,7 +3,7 @@
#include "Maat_rule.h"
#include "Maat_command.h"
#include "Maat_limits.h"
#include "Maat_table.h"
#include "Maat_table_schema.h"
#include "Maat_table_runtime.h"
#include <MESA/MESA_htable.h>

27
src/inc_internal/Maat_table_runtime.h
View File

@@ -1,8 +1,9 @@
#include "Maat_table.h"
#include "Maat_table_schema.h"
#include "Maat_ex_data.h"
#include "IPMatcher.h"
#include "gram_index_engine.h"
#include "bool_matcher.h"
#include "FQDN_engine.h"
#include "alignment_int64.h"
#include <MESA/MESA_list_queue.h>
@@ -18,6 +19,12 @@ struct fqdn_plugin_runtime
struct EX_data_rt* ex_data_rt;
int changed_flag;
};
struct bool_plugin_runtime
{
struct bool_matcher *matcher;
struct EX_data_rt* ex_data_rt;
int changed_flag;
};
struct plugin_runtime
{
struct EX_data_rt* ex_data_rt;
@@ -54,6 +61,7 @@ struct Maat_table_runtime
{
struct similar_runtime similar; //for digest and similarity
struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin
struct bool_plugin_runtime bool_plugin;
struct plugin_runtime plugin;
struct ip_plugin_runtime ip_plugin;
struct expr_runtime expr;
@@ -80,24 +88,25 @@ enum MAAT_TABLE_TYPE Maat_table_runtime_get_type(struct Maat_table_runtime* tabl
size_t Maat_table_runtime_plugin_cached_row_count(struct Maat_table_runtime* table_rt);
const char* Maat_table_runtime_plugin_get_cached_row(struct Maat_table_runtime* table_rt, size_t Nth_row);
int Maat_table_runtime_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, void* logger);
MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, const char* key);
int Maat_table_runtime_commit_EX_data_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);
void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag);
void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id);
int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
void Maat_table_runtime_plugin_update_commit(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_plugin_commit_update(struct Maat_table_runtime* table_rt);
MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, const char* key);
void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);
int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size);
int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_ip_plugin_commit_update(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
int Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);
int Maat_table_runtime_fqdn_plugin_commit_update(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size);
void Maat_table_runtime_bool_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
int Maat_table_runtime_bool_plugin_commit_update(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_bool_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, unsigned long long item_id[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size);

65
src/inc_internal/Maat_table.h → src/inc_internal/Maat_table_schema.h
View File

@@ -24,6 +24,7 @@ enum MAAT_SCAN_TYPE
SCAN_TYPE_PLUGIN,
SCAN_TYPE_IP_PLUGIN,
SCAN_TYPE_FQDN_PLUGIN,
SCAN_TYPE_BOOL_PLUGIN,
SCAN_TYPE_IP,
SCAN_TYPE_INTERVAL,
SCAN_TYPE_STRING,
@@ -60,7 +61,8 @@ enum MAAT_TABLE_TYPE
TABLE_TYPE_COMPILE,
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN,
TABLE_TYPE_FQDN_PLUGIN
TABLE_TYPE_FQDN_PLUGIN,
TABLE_TYPE_BOOL_PLUGIN
};
struct compile_ex_data_idx
@@ -113,6 +115,7 @@ struct EX_data_schema
Maat_plugin_EX_key2index_func_t* key2index_func;
long argl;
void *argp;
int set_flag;
};
struct plugin_table_schema
{
@@ -122,7 +125,6 @@ struct plugin_table_schema
int n_foreign;
int foreign_columns[MAX_FOREIGN_CLMN_NUM];
int cb_plug_cnt;
int have_exdata;
struct plugin_table_callback_schema cb_plug[MAX_PLUGIN_PER_TABLE];
struct EX_data_schema ex_schema;
};
@@ -148,6 +150,16 @@ struct fqdn_plugin_table_schema
int have_exdata;
struct EX_data_schema ex_schema;
};
struct bool_plugin_table_schema
{
int row_id_column;
int bool_expr_column;
int rule_tag_column;
int valid_flag_column;
int have_exdata;
struct EX_data_schema ex_schema;
};
struct Maat_table_schema
{
int table_id;
@@ -162,6 +174,7 @@ struct Maat_table_schema
struct plugin_table_schema plugin;
struct ip_plugin_table_schema ip_plugin;
struct fqdn_plugin_table_schema fqdn_plugin;
struct bool_plugin_table_schema bool_plugin;
struct virtual_table_schema virtual_table;
struct composition_table_schema composition;
void* others;//group, ip, interval and digest don't have any special schema.
@@ -177,59 +190,47 @@ void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr);
size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr);
size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr);
struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id);
struct Maat_table_schema * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id);
struct Maat_table_schema * Maat_table_manager_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id);
struct Maat_table_schema * Maat_table_manager_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id);
enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type);
int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
int Maat_table_manager_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
int Maat_table_manager_add_callback_func(struct Maat_table_manager* table_mgr,
int table_id,
Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
Maat_update_callback_t *update,//table line ,u_para
Maat_finish_callback_t *finish,//u_para
void* u_para);
int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_manager_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_manager_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
int Maat_table_manager_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id);
enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id);
const char* Maat_table_manager_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id);
enum MAAT_TABLE_TYPE Maat_table_manager_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id);
int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name,
int Maat_table_manager_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name,
Maat_rule_EX_new_func_t *new_func,
Maat_rule_EX_free_func_t* free_func,
Maat_rule_EX_dup_func_t* dup_func,
long argl, void *argp);
struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx);
int Maat_table_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger);
int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger);
int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
struct compile_ex_data_idx* Maat_table_manager_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx);
int Maat_table_schema_set_EX_data_schema(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger);
struct EX_data_schema *Maat_table_schema_get_EX_data_schema(struct Maat_table_schema *table_schema);
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type);
void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr);
int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr);
struct Maat_table_schema* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char* table_name);
struct Maat_table_schema* Maat_table_manager_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
void Maat_table_schema_set_updating_name(struct Maat_table_schema* p_table, const char* table_name);
int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type);
int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table);
int Maat_table_manager_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type);
int Maat_table_schema_get_valid_flag_column(struct Maat_table_schema* p_table);