gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

unfinished work

Browse Source
This commit is contained in:
liuwentan
2023-01-30 21:59:35 +08:00
parent 3d4b833e48
commit 25f944a1d1
49 changed files with 6537 additions and 6149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/inc_internal/maat_common.h
View File

@@ -23,8 +23,10 @@ extern "C"
struct maat_options {
char instance_name[NAME_MAX];
char compile_tablename[NAME_MAX];
size_t nr_worker_threads;
const char *accept_tags;
int rule_effect_interval_ms;
int rule_update_checking_interval_ms;
int gc_timeout_ms;

90
src/inc_internal/maat_compile.h Normal file
View File

@@ -0,0 +1,90 @@
/*
**********************************************************************************************
* File: maat_compile.h
* Description:
* Authors: Zheng Chao <zhengchao@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_COMPILE_H_
#define _MAAT_COMPILE_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "log/log.h"
#include "cJSON/cJSON.h"
#include "maat.h"
#include "maat_kv.h"
#include "maat_rule.h"
struct compile_ex_data_schema {
maat_rule_ex_new_func_t *new_func;
maat_rule_ex_free_func_t *free_func;
maat_rule_ex_dup_func_t *dup_func;
long argl;
void *argp;
int idx;
int table_id;
};
/* compile schema API */
void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void compile_schema_free(void *compile_schema);
void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void group2compile_schema_free(void *g2c_schema);
int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, int table_id,
maat_rule_ex_new_func_t *new_func,
maat_rule_ex_free_func_t *free_func,
maat_rule_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger);
struct compile_ex_data_schema *
compile_table_get_rule_ex_data_schema(struct compile_schema *compile_schema, size_t idx);
size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema);
/* compile runtime API */
void *compile_runtime_new(void *compile_schema, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void compile_runtime_free(void *compile_runtime);
int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line,
int valid_column);
int compile_runtime_commit(void *compile_runtime);
int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids,
int *compile_ids, size_t compile_ids_size, struct maat_state *state);
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct maat_group_topology *group_topo,
struct maat_compile_state *compile_state,
struct maat_hit_path *hit_paths, size_t hit_path_siz);
/* group2compile runtime API */
void *group2compile_runtime_new(void *g2c_schema, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void group2compile_runtime_free(void *g2c_runtime);
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line,
int valid_column);
/* maat compile state API */
struct maat_compile_state;
struct maat_compile_state *maat_compile_state_new(int thread_id);
void maat_compile_state_free(struct maat_compile_state *compile_state);
void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, int item_id, int group_id, int virtual_table_id,
int Nth_scan, int Nth_item_result);
void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, struct maat_compile **compile_hash,
int group_id, int virtual_table_id);
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state);
#ifdef __cpluscplus
}
#endif
#endif

31
src/inc_internal/maat_ex_data.h
View File

@@ -16,6 +16,7 @@ extern "C"
{
#endif
#include "maat/maat.h"
#include "rcu_hash.h"
struct ex_data_container {
@@ -23,6 +24,12 @@ struct ex_data_container {
void *custom_data;
};
struct ex_container_ctx {
int table_id;
void (*custom_data_free)(void *custom_data);
struct ex_data_schema *ex_schema;
};
struct ex_data_runtime;
/* ex_data_runtime API */
@@ -42,6 +49,12 @@ size_t ex_data_runtime_cached_row_count(struct ex_data_runtime *ex_data_rt);
void ex_data_runtime_clear_row_cache(struct ex_data_runtime *ex_data_rt);
/* set schema API */
struct ex_data_schema *ex_data_schema_new(maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp);
void ex_data_schema_free(struct ex_data_schema *ex_schema);
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema);
/* set user_ctx API */
@@ -49,16 +62,24 @@ void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, st
struct ex_container_ctx *ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt);
struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data);
void ex_data_container_free(void *ctx, void *data);
/* ex_data_runtime ex data API */
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len);
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt,
const char *row, const char *key, size_t key_len);
void ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct ex_data_container *ex_container);
int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len,
struct ex_data_container *ex_container);
void ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct log_handle *logger);
int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len);
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, struct ex_data_container ***ex_container);
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_data_container ***ex_container);
void *ex_data_runtime_dup_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);
void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);

54
src/inc_internal/maat_expr.h Normal file
View File

@@ -0,0 +1,54 @@
/*
**********************************************************************************************
* File: maat_expr.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_EXPR_H_
#define _MAAT_EXPR_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "log/log.h"
#include "cJSON/cJSON.h"
struct expr_runtime;
void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void expr_schema_free(void *expr_schema);
/* expr runtime API */
void *expr_runtime_new(void *expr_schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
void expr_runtime_free(void *expr_runtime);
int expr_runtime_updating_flag(struct expr_runtime *expr_rt);
int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column);
int expr_runtime_commit(void *expr_runtime);
/* expr runtime scan API */
/**
* @brief scan string to get hit group_ids
*
* @retval the num of hit group_id
*/
int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len, int group_ids[],
size_t max_hit_num, int virtual_table_id,
struct maat_state *state);
void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len, int results[], size_t *n_result);
void expr_runtime_stream_close(struct expr_runtime *expr_rt);
#ifdef __cpluscplus
}
#endif
#endif

34
src/inc_internal/maat_group.h
View File

@@ -16,15 +16,10 @@ extern "C"
{
#endif
struct maat_item {
int item_id;
int group_id;
struct maat_group *ref_parent_group;
UT_hash_handle hh;
void *user_data;
};
#include "cJSON/cJSON.h"
#include "uthash/uthash.h"
#include "igraph/igraph.h"
#include "maat_kv.h"
struct maat_group {
igraph_integer_t vertex_id;
@@ -32,16 +27,16 @@ struct maat_group {
int ref_by_compile_cnt;
int ref_by_superior_group_cnt;
int ref_by_subordinate_group_cnt;
int ref_by_item_cnt;
size_t top_group_cnt;
int *top_group_ids;
UT_hash_handle hh_group_id;
UT_hash_handle hh_vertex_id;
};
/* maat group topology API */
struct maat_group_topology;
struct maat_group_topology *maat_group_topology_new(struct log_handle *logger);
void maat_group_topology_free(struct maat_group_topology *group_topo);
@@ -59,6 +54,23 @@ int maat_group_topology_remove_group_from_group(struct maat_group_topology *grou
/* build top groups */
int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo);
/* group2group schema API */
void *group2group_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void group2group_schema_free(void *g2g_schema);
/* group2group runtime API */
struct group2group_runtime *
group2group_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void group2group_runtime_free(void *g2g_runtime);
int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, int valid_column);
int group2group_runtime_commit(void *g2g_runtime);
int group2group_runtime_get_top_groups(struct group2group_runtime *g2g_rt, int *group_ids, size_t n_group_ids,
int *top_group_ids);
#ifdef __cpluscplus
}
#endif

67
src/inc_internal/maat_hierarchy.h
View File

@@ -1,67 +0,0 @@
/*
**********************************************************************************************
* File: maat_hierarchy.h
* Description:
* Authors: Zheng Chao <zhengchao@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_HIERARCHY_H_
#define _MAAT_HIERARCHY_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "maat_garbage_collection.h"
/* maat hierarchy API */
struct maat_hierarchy;
struct maat_hierarchy *maat_hierarchy_new(int thread_num, struct maat_garbage_bin *bin, struct log_handle *logger);
void maat_hierarchy_free(struct maat_hierarchy *hier);
int maat_hierarchy_rebuild(struct maat_hierarchy *hier);
size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
struct maat_hit_path *hit_paths, size_t n_path);
/* maat hierarchy compile mid API */
struct maat_hierarchy_compile_mid;
struct maat_hierarchy_compile_mid *maat_hierarchy_compile_mid_new(struct maat_hierarchy *hier, int thread_id);
void maat_hierarchy_compile_mid_free(struct maat_hierarchy_compile_mid *mid);
void maat_hierarchy_compile_mid_update(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
int item_id, int virtual_table_id, int Nth_scan, int Nth_item_result);
int maat_hierarchy_compile_mid_has_NOT_clause(struct maat_hierarchy_compile_mid *mid);
/* maat hierarchy compile API */
int maat_hierarchy_compile_add(struct maat_hierarchy *hier, int compile_id, int declared_clause_num, void* user_data);
int maat_hierarchy_compile_remove(struct maat_hierarchy *hier, int compile_id);
void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
void *maat_hierarchy_compile_dettach_user_data(struct maat_hierarchy *hier, int compile_id);
void *maat_hierarchy_compile_read_user_data(struct maat_hierarchy *hier, int compile_id);
void maat_hierarchy_compile_user_data_iterate(struct maat_hierarchy *hier, void (*callback)(void *user_data, void *param), void *param);
/* maat hierarchy group2compile API */
int maat_hierarchy_add_group_to_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag,
int clause_index, int compile_id);
int maat_hierarchy_remove_group_from_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag,
int clause_index, int compile_id);
/* maat hierarchy item API */
int maat_hierarchy_item_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
int is_last_compile, void **user_data_array, size_t ud_array_sz);
void *maat_hierarchy_item_dettach_user_data(struct maat_hierarchy *hier, int item_id);
void maat_hierarchy_set_item_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
/* maat hierarchy item2group API */
int maat_hierarchy_add_item_to_group(struct maat_hierarchy *hier, int group_id, int item_id, void* user_data);
int maat_hierarchy_remove_item_from_group(struct maat_hierarchy *hier, int group_id, int item_id);
#ifdef __cpluscplus
}
#endif
#endif

42
src/inc_internal/maat_ip.h Normal file
View File

@@ -0,0 +1,42 @@
/*
**********************************************************************************************
* File: maat_ip.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_IP_H_
#define _MAAT_IP_H_
#ifdef __cpluscplus
extern "C"
{
#endif
struct ip_plus_runtime;
void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void ip_plus_schema_free(void *ip_plus_schema);
/* ip plus runtime API */
void *ip_plus_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
void ip_plus_runtime_free(void *ip_plus_runtime);
int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line);
int ip_plus_runtime_commit(void *ip_plus_runtime);
struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt);
/* ip runtime scan API */
int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr *data,
int *group_id_array, size_t n_group_id_array, int virtual_table_id,
struct maat_state *state);
#ifdef __cpluscplus
}
#endif
#endif

49
src/inc_internal/maat_ip_plugin.h Normal file
View File

@@ -0,0 +1,49 @@
/*
**********************************************************************************************
* File: maat_ip_plugin.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_IP_PLUGIN_H_
#define _MAAT_IP_PLUGIN_H_
#ifdef __cpluscplus
extern "C"
{
#endif
struct ip_plugin_runtime;
/* ip plugin schema API */
void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void ip_plugin_schema_free(void *ip_plugin_schema);
/* ip plugin table ex data API */
struct ex_data_schema *ip_plugin_table_get_ex_data_schema(void *ip_plugin_schema);
int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp);
/* ip plugin runtime API */
void *ip_plugin_runtime_new(void *ip_plugin_schema, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void ip_plugin_runtime_free(void *ip_plugin_runtime);
int ip_plugin_runtime_updating_flag(struct ip_plugin_runtime *ip_plugin_rt);
int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line);
int ip_plugin_runtime_commit(void *ip_plugin_runtime);
struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime);
#ifdef __cpluscplus
}
#endif
#endif

12
src/inc_internal/maat_kv.h
View File

@@ -18,13 +18,17 @@ extern "C"
struct maat_kv_store;
struct maat_kv_store* maat_kv_store_new(void);
struct maat_kv_store *maat_kv_store_new(void);
void maat_kv_store_free(struct maat_kv_store* store);
void maat_kv_store_free(struct maat_kv_store *store);
int maat_kv_register(struct maat_kv_store* store, const char* key, int value);
int maat_kv_register(struct maat_kv_store *store, const char *key, int value);
int maat_kv_read(struct maat_kv_store* store, const char* key, int* value);
int maat_kv_read(struct maat_kv_store *store, const char *key, int *value);
int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, int *value);
struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *store);
#ifdef __cpluscplus
}

25
src/inc_internal/maat_limits.h Normal file
View File

@@ -0,0 +1,25 @@
/*
**********************************************************************************************
* File: maat_limits.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_LIMITS_H_
#define _MAAT_LIMITS_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#define MAX_KEYWORDS_STR 1024
#ifdef __cpluscplus
}
#endif
#endif

60
src/inc_internal/maat_plugin.h Normal file
View File

@@ -0,0 +1,60 @@
/*
**********************************************************************************************
* File: maat_plugin.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_PLUGIN_H_
#define _MAAT_PLUGIN_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#define MAX_FOREIGN_CLMN_NUM 8
/* plugin schema API */
void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void plugin_schema_free(void *plugin_schema);
struct plugin_schema;
/* plugin table callback API */
int plugin_table_add_callback(void *plugin_schema, int table_id,
maat_start_callback_t *start,
maat_update_callback_t *update,
maat_finish_callback_t *finish,
void *u_para, struct log_handle *logger);
void plugin_table_all_callback_start(struct plugin_schema *plugin_schema, int update_type);
void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema);
int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns);
/* plugin table ex data API */
int plugin_table_set_ex_data_schema(void *custom_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp);
struct ex_data_schema *plugin_table_get_ex_data_schema(void *custom_schema);
/* plugin runtime API */
struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime);
int plugin_runtime_updating_flag(struct plugin_runtime *plugin_rt);
int plugin_runtime_update(struct plugin_runtime *plugin_rt, struct plugin_item *plugin_item,
struct plugin_schema *plugin_schema, const char *row,
struct log_handle *logger);
int plugin_runtime_commit(void *plugin_runtime);
#ifdef __cpluscplus
}
#endif
#endif

62
src/inc_internal/maat_rule.h
View File

@@ -27,9 +27,12 @@ extern "C"
#include "log/log.h"
#include "hiredis/hiredis.h"
#include "uthash/uthash.h"
#include "maat_table_schema.h"
#include "maat_command.h"
#include "IPMatcher.h"
#include "maat_kv.h"
#include "maat_table.h"
#define MAX_TABLE_NUM 256
struct maat_rule_head {
int config_id;
@@ -64,8 +67,15 @@ struct maat_item_inner {
int expr_id_ub; //up boundary
};
struct maat_item {
int item_id;
int group_id;
UT_hash_handle hh;
void *user_data;
};
#define COMPILE_RULE_MAGIC 0x1a2b3c4d
struct maat_compile_rule {
struct compile_rule {
long long magic_num;
int compile_id;
struct maat_rule_head head;// fix len of Maat_rule_t
@@ -73,7 +83,7 @@ struct maat_compile_rule {
int is_valid;
int declared_clause_num;
double evaluation_order;
struct table_schema *ref_table;
struct compile_schema *ref_table;
void **ex_data;
pthread_rwlock_t rwlock;
};
@@ -83,8 +93,9 @@ struct group2compile_rule {
int compile_id;
int is_valid;
int not_flag;
int virtual_table_id;
int vt_id; //virtual_table_id
int clause_index;
int associated_compile_table_id;
};
struct group2group_rule {
@@ -100,14 +111,19 @@ struct maat_runtime {
time_t last_update_time;
long long *ref_cnt;
struct table_runtime_manager *table_rt_mgr;
struct table_manager *ref_tbl_mgr; //share with maat instance
size_t max_table_num;
size_t max_thread_num;
uint32_t rule_num;
struct maat_garbage_bin *ref_garbage_bin;
struct scan_result *item_result_buff;
struct maat_kv_store *district_map;
struct maat_kv_store *tmp_district_map;
unsigned int district_num;
struct log_handle *logger;
};
@@ -179,7 +195,7 @@ struct maat {
struct maat_runtime *maat_rt;
struct maat_runtime *creating_maat_rt;
struct table_schema_manager *table_schema_mgr;
struct table_manager *tbl_mgr;
enum data_source input_mode;
union {
@@ -187,9 +203,6 @@ struct maat {
struct source_json_ctx json_ctx;
struct source_redis_ctx mr_ctx;
};
struct rule_tag *accept_tags;
int n_accept_tag;
struct log_handle *logger;
int deferred_load;
@@ -210,10 +223,8 @@ struct maat {
struct maat_garbage_bin *garbage_bin;
char compile_tablename[NAME_MAX];
//char group_tn[NAME_MAX];
//char group2compile_tn[NAME_MAX];
//char group2group_tn[NAME_MAX];
int default_compile_table_id;
int g2g_table_id; //group2group table id
char decrypt_key[NAME_MAX];
char decrypt_algo[NAME_MAX];
@@ -232,6 +243,29 @@ struct maat {
long long scan_err_cnt;
};
struct maat_state {
struct maat *maat_instance;
int16_t thread_id;
int compile_table_id; //caller can select compile table to scan
unsigned char is_set_district;
unsigned char is_last_scan;
int district_id; //-1: Any District; -2: Unkonwn District;
int scan_cnt;
struct maat_compile_state *compile_mid;
};
int parse_accept_tag(const char *value, struct rule_tag **result, void *logger);
int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, int n_tag);
struct maat_item *maat_item_new(int item_id, int group_id, void *user_data);
void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *));
struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int district_id);
void maat_item_inner_free(struct maat_item_inner *item);
void maat_start_cb(long long new_version, int update_type, void *u_para);
int maat_update_cb(const char *table_name, const char *line, void *u_para);

68
src/inc_internal/maat_table.h Normal file
View File

@@ -0,0 +1,68 @@
/*
**********************************************************************************************
* File: maat_table.h
* Description: maat table schema and runtime
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_TABLE_H_
#define _MAAT_TABLE_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include <stddef.h>
#include <cJSON/cJSON.h>
enum table_type {
TABLE_TYPE_INVALID = -1,
TABLE_TYPE_EXPR = 0,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_IP_PLUS,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_INTERVAL_PLUS,
TABLE_TYPE_DIGEST,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_CONJUNCTION,
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN,
TABLE_TYPE_FQDN_PLUGIN,
TABLE_TYPE_BOOL_PLUGIN,
//above are physical table
TABLE_TYPE_VIRTUAL,
TABLE_TYPE_COMPILE,
TABLE_TYPE_GROUP2GROUP,
TABLE_TYPE_GROUP2COMPILE,
TABLE_TYPE_MAX
};
struct table_manager;
struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags,
struct log_handle *logger);
int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *garbage_bin);
int table_manager_deinit(struct table_manager *tbl_mgr);
void table_manager_destroy(struct table_manager *tbl_mgr);
size_t table_manager_table_count(struct table_manager *tbl_mgr);
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name);
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id);
int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id);
int table_manager_accept_tags_match(const char *tags);
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id);
void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id);
void table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line);
void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id);
#ifdef __cpluscplus
}
#endif
#endif

86
src/inc_internal/maat_table_runtime.h
View File

@@ -1,86 +0,0 @@
/*
**********************************************************************************************
* File: maat_table_runtime.h
* Description: maat table runtime entry
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_TABLE_RUNTIME_H_
#define _MAAT_TABLE_RUNTIME_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "maat/maat.h"
#include "maat_table_schema.h"
#include "maat_garbage_collection.h"
struct ip_addr {
enum ip_type ip_type;
union {
uint32_t ipv4;
uint32_t ipv6[4];
};
};
struct table_item;
struct table_runtime;
struct table_runtime_manager;
/* table runtime manager API */
struct table_runtime_manager *
table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num,
struct maat_garbage_bin *bin, struct log_handle *logger);
void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr);
/* table runtime API */
struct table_runtime *table_runtime_get(struct table_runtime_manager *table_rt_mgr, int table_id);
size_t table_runtime_rule_count(struct table_runtime *table_rt);
enum table_type table_runtime_get_type(struct table_runtime* table_rt);
void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema,
const char *line, struct table_item *table_item, struct log_handle *logger);
/**
* @brief if table_runtime is updating
*
* @retval 1(yes) 0(no)
*/
int table_runtime_updating_flag(struct table_runtime *table_rt);
void table_runtime_commit(struct table_runtime *table_rt, long long version, size_t nr_worker_thread, struct log_handle *logger);
/* table runtime scan API */
int table_runtime_scan_string(struct table_runtime *table_rt, int thread_id, const char *data, size_t data_len,
int results[], size_t *n_result);
void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id);
int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, int results[], size_t *n_result);
void table_runtime_stream_close(struct table_runtime *table_rt);
int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct ip_addr *data, struct scan_result *results, size_t *n_result, size_t n_result_array);
/* table runtime cached row API */
size_t table_runtime_cached_row_count(struct table_runtime *table_rt);
const char* table_runtime_get_cached_row(struct table_runtime *table_rt, size_t row_seq);
/* table runtime ex data API */
struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt);
void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema,
int nr_worker_thread, long long version, struct log_handle *logger);
#ifdef __cpluscplus
}
#endif
#endif

310
src/inc_internal/maat_table_schema.h
View File

@@ -1,310 +0,0 @@
/*
**********************************************************************************************
* File: maat_table_schema.h
* Description: maat table schema entry
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_TABLE_SCHEMA_H_
#define _MAAT_TABLE_SCHEMA_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include <stddef.h>
#include <limits.h>
#include "maat/maat.h"
#include "adapter_hs.h"
#define MAX_TABLE_NUM 256
#define MAX_DISTRICT_STR 128
#define MAX_IP_STR 128
#define MAX_KEYWORDS_STR 1024
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_TABLE_LINE_SIZE (1024 * 16)
#define MAX_COMPILE_EX_DATA_NUM 2
enum component_table_type {
COMPONENT_TABLE_TYPE_NONE = -1,
COMPONENT_TABLE_TYPE_SIP = 0,
COMPONENT_TABLE_TYPE_DIP,
COMPONENT_TABLE_TYPE_SESSION,
COMPONENT_TABLE_TYPE_MAX
};
enum table_type {
TABLE_TYPE_INVALID = -1,
TABLE_TYPE_EXPR = 0,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_IP_PLUS,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_INTERVAL_PLUS,
TABLE_TYPE_DIGEST,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN,
TABLE_TYPE_FQDN_PLUGIN,
TABLE_TYPE_BOOL_PLUGIN,
//above are physical table
TABLE_TYPE_VIRTUAL,
TABLE_TYPE_COMPOSITION,
TABLE_TYPE_COMPILE,
TABLE_TYPE_GROUP,
TABLE_TYPE_GROUP2GROUP,
TABLE_TYPE_GROUP2COMPILE
};
enum expr_type {
EXPR_TYPE_STRING = 0,
EXPR_TYPE_AND,
EXPR_TYPE_REGEX,
EXPR_TYPE_MAX
};
enum scan_type {
SCAN_TYPE_INVALID = -1,
SCAN_TYPE_NONE = 0,
SCAN_TYPE_PLUGIN,
SCAN_TYPE_IP_PLUGIN,
SCAN_TYPE_FQDN_PLUGIN,
SCAN_TYPE_BOOL_PLUGIN,
SCAN_TYPE_IP,
SCAN_TYPE_INTERVAL,
SCAN_TYPE_STRING,
SCAN_TYPE_MAX
};
enum match_method {
MATCH_METHOD_SUB = 0,
MATCH_METHOD_RIGHT,
MATCH_METHOD_LEFT,
MATCH_METHOD_COMPLETE,
MATCH_METHOD_MAX
};
struct compile_item {
int compile_id;
int service_id;
int action;
int do_blacklist;
int do_log;
char user_region[MAX_TABLE_LINE_SIZE];
int is_valid;
int clause_num;
int evaluation_order;
};
struct group2compile_item {
int group_id;
int compile_id;
int is_valid;
int not_flag;
int virtual_table_id;
int clause_index;
int associated_compile_table_id;
};
struct group2group_item {
int group_id;
int superior_group_id;
int is_valid;
};
struct expr_item {
int item_id;
int group_id;
char district[MAX_DISTRICT_STR];
char keywords[MAX_KEYWORDS_STR];
enum expr_type expr_type;
enum match_method match_method;
int is_hexbin;
int is_case_sensitive;
int is_valid;
};
struct ipv4_item_rule {
uint32_t min_sip; /* 源地址下界；0表示忽略本字段 */
uint32_t max_sip; /* 源地址上界；0表示固定IP=min_saddr */
uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */
uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */
uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP；0表示忽略本字段 */
uint16_t direction; /* 方向，0表示双向，1表示单向 */
};
struct ipv6_item_rule {
uint32_t min_sip[4]; /* 源地址下界；全0表示忽略本字段 */
uint32_t max_sip[4]; /* 源地址上界；全0表示固定IP=min_saddr */
uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */
uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */
uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP，无限制默认为0 */
uint16_t direction; /* 方向，0表示双向，1表示单向 */
};
struct ip_plus_item {
int item_id;
int group_id;
int addr_type;
union {
struct ipv4_item_rule ipv4;
struct ipv6_item_rule ipv6;
};
int is_valid;
};
struct plugin_item {
char key[MAX_KEYWORDS_STR];
size_t key_len;
int is_valid;
};
struct ip_plugin_item {
int item_id;
int ip_type;
char start_ip[MAX_IP_STR];
char end_ip[MAX_IP_STR];
int is_valid;
int rule_tag;
};
struct table_item {
enum table_type table_type;
union {
struct compile_item compile_item;
struct group2compile_item g2c_item;
struct group2group_item g2g_item;
struct expr_item expr_item;
struct ip_plus_item ip_plus_item;
struct plugin_item plugin_item;
struct ip_plugin_item ip_plugin_item;
};
};
struct plugin_table_callback_schema
{
maat_start_callback_t *start;
maat_update_callback_t *update;
maat_finish_callback_t *finish;
void *u_para;
};
struct ex_data_schema
{
maat_plugin_ex_new_func_t *new_func;
maat_plugin_ex_free_func_t *free_func;
maat_plugin_ex_dup_func_t *dup_func;
//Maat_plugin_EX_key2index_func_t* key2index_func;
long argl;
void *argp;
int set_flag;
};
struct compile_ex_data_schema {
maat_rule_ex_new_func_t *new_func;
maat_rule_ex_free_func_t *free_func;
maat_rule_ex_dup_func_t *dup_func;
long argl;
void *argp;
int idx;
int table_id;
};
struct table_schema;
struct table_schema_manager;
/* table schema manager API */
struct table_schema_manager *table_schema_manager_create(const char *table_info_path, struct log_handle *logger);
void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr);
int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_mgr, const char *table_name);
/**
* @brief get composition table's child table(specified by type) id
*/
int table_schema_manager_get_child_table_id(struct table_schema_manager *table_schema_mgr, int parent_table_id,
enum component_table_type type);
enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int table_id);
size_t table_schema_manager_get_size(struct table_schema_manager* table_schema_mgr);
void table_schema_manager_all_plugin_cb_start(struct table_schema_manager* table_schema_mgr, int update_type);
void table_schema_manager_all_plugin_cb_finish(struct table_schema_manager* table_schema_mgr);
/* table schema generic API */
struct table_schema *table_schema_get(struct table_schema_manager *table_schema_mgr, int table_id);
struct table_schema *table_schema_get_by_scan_type(struct table_schema_manager *table_schema_mgr,
int table_id, enum scan_type type, int *virtual_table_id);
enum table_type table_schema_get_table_type(struct table_schema *table_schema);
int table_schema_get_table_id(struct table_schema *table_schema);
/* get group2compile table's associated compile table id */
int table_schema_get_associated_table_id(struct table_schema *table_schema);
enum scan_type table_schema_get_scan_type(struct table_schema *table_schema);
struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema,
struct rule_tag *accept_tags, int n_accept_tag, struct log_handle *logger);
void table_item_free(struct table_item *table_item);
int table_schema_get_valid_flag_column(struct table_schema *table_schema);
void table_schema_set_updating_name(struct table_schema *table_schema, const char *table_name);
const char *table_schema_get_updating_name(struct table_schema *table_schema);
/* expr table schema API */
enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);
/* table schema ex API */
int table_schema_set_ex_data_schema(struct table_schema *table_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger);
struct ex_data_schema *table_schema_get_ex_data_schema(struct table_schema *table_schema);
/* table schema compile rule ex API */
int table_schema_set_compile_rule_ex_data_schema(struct table_schema *table_schema,
maat_rule_ex_new_func_t *new_func,
maat_rule_ex_free_func_t *free_func,
maat_rule_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger);
struct compile_ex_data_schema *table_schema_get_compile_rule_ex_data_schema(struct table_schema *table_schema, size_t idx);
size_t table_schema_compile_rule_ex_data_schema_count(struct table_schema *table_schema);
/**
* @brief if plugin table schema's ex data schema set
*
* @retval 1(already Set) 0(Not set yet)
*/
int table_schema_ex_data_schema_flag(struct table_schema *table_schema);
int table_schema_add_callback(struct table_schema_manager *table_schema_mgr, int table_id,
maat_start_callback_t *start,
maat_update_callback_t *update,
maat_finish_callback_t *finish,
void *u_para, struct log_handle *logger);
/**
* @brief the number of callback function stored in plugin table schema
*/
size_t table_schema_callback_count(struct table_schema *table_schema);
void table_schema_all_cb_update(struct table_schema *table_schema, const char *row);
int table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns);
#ifdef __cpluscplus
}
#endif
#endif

7
src/inc_internal/maat_utils.h
View File

@@ -31,7 +31,9 @@ extern "C"
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#define MAX_SCANNER_HIT_NUM 4096
#define MAX_SCANNER_HIT_COMPILE_NUM 4096
#define MAX_SCANNER_HIT_GROUP_NUM 4096
#define MAX_SCANNER_HIT_ITEM_NUM 4096
enum maat_ip_format {
IP_FORMAT_RANGE,
@@ -59,6 +61,9 @@ char *maat_strdup(const char *s);
int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len);
/* the column value must be integer */
int get_column_value(const char *line, int column_seq);
int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *out_sz);
char *strtok_r_esc(char *s, const char delim, char **save_ptr);

38
src/inc_internal/maat_virtual.h Normal file
View File

@@ -0,0 +1,38 @@
/*
**********************************************************************************************
* File: maat_virtual.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_VIRTUAL_H_
#define _MAAT_VIRTUAL_H_
#ifdef __cpluscplus
extern "C"
{
#endif
enum scan_type {
SCAN_TYPE_INVALID = -1,
SCAN_TYPE_NONE = 0,
SCAN_TYPE_PLUGIN,
SCAN_TYPE_IP_PLUGIN,
SCAN_TYPE_FQDN_PLUGIN,
SCAN_TYPE_BOOL_PLUGIN,
SCAN_TYPE_IP,
SCAN_TYPE_INTERVAL,
SCAN_TYPE_STRING,
SCAN_TYPE_MAX
};
void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
#ifdef __cpluscplus
}
#endif
#endif

1
src/inc_internal/rcu_hash.h
View File

@@ -51,6 +51,7 @@ void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len
void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_len);
size_t rcu_hash_count(struct rcu_hash_table *htable);
size_t rcu_hash_updating_count(struct rcu_hash_table *htable);
/**
* @brief make add/del effective