gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hierarchy refactor unfinished

Browse Source
This commit is contained in:
liuwentan
2023-01-06 18:54:59 +08:00
parent 9778267b48
commit 3d4b833e48
18 changed files with 2314 additions and 848 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/inc_internal/maat_common.h
View File

@@ -23,6 +23,7 @@ extern "C"
struct maat_options {
char instance_name[NAME_MAX];
char compile_tablename[NAME_MAX];
size_t nr_worker_threads;
int rule_effect_interval_ms;
int rule_update_checking_interval_ms;

66
src/inc_internal/maat_group.h Normal file
View File

@@ -0,0 +1,66 @@
/*
**********************************************************************************************
* File: maat_group.h
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_GROUP_H_
#define _MAAT_GROUP_H_
#ifdef __cpluscplus
extern "C"
{
#endif
struct maat_item {
int item_id;
int group_id;
struct maat_group *ref_parent_group;
UT_hash_handle hh;
void *user_data;
};
struct maat_group {
igraph_integer_t vertex_id;
int group_id;
int ref_by_compile_cnt;
int ref_by_superior_group_cnt;
int ref_by_subordinate_group_cnt;
int ref_by_item_cnt;
size_t top_group_cnt;
int *top_group_ids;
UT_hash_handle hh_group_id;
UT_hash_handle hh_vertex_id;
};
struct maat_group_topology;
struct maat_group_topology *maat_group_topology_new(struct log_handle *logger);
void maat_group_topology_free(struct maat_group_topology *group_topo);
struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id);
void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group);
/**
* @retval if not found, return NULL
*/
struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id);
int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id);
int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id);
/* build top groups */
int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo);
#ifdef __cpluscplus
}
#endif
#endif

31
src/inc_internal/maat_hierarchy.h
View File

@@ -23,46 +23,43 @@ struct maat_hierarchy;
struct maat_hierarchy *maat_hierarchy_new(int thread_num, struct maat_garbage_bin *bin, struct log_handle *logger);
void maat_hierarchy_free(struct maat_hierarchy *hier);
int maat_hierarchy_rebuild(struct maat_hierarchy *hier);
size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
struct maat_hit_path_t *hit_paths, size_t n_path);
void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
void maat_hierarchy_set_region_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
struct maat_hit_path *hit_paths, size_t n_path);
/* maat hierarchy compile mid API */
struct maat_hierarchy_compile_mid;
struct maat_hierarchy_compile_mid *maat_hierarchy_compile_mid_new(struct maat_hierarchy *hier, int thread_id);
void maat_hierarchy_compile_mid_free(struct maat_hierarchy_compile_mid *mid);
void maat_hierarchy_compile_mid_update(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result);
int item_id, int virtual_table_id, int Nth_scan, int Nth_item_result);
int maat_hierarchy_compile_mid_has_NOT_clause(struct maat_hierarchy_compile_mid *mid);
/* maat hierarchy compile API */
int maat_hierarchy_compile_add(struct maat_hierarchy *hier, int compile_id, int declared_clause_num, void* user_data);
int maat_hierarchy_compile_remove(struct maat_hierarchy *hier, int compile_id);
void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
void *maat_hierarchy_compile_dettach_user_data(struct maat_hierarchy *hier, int compile_id);
void *maat_hierarchy_compile_read_user_data(struct maat_hierarchy *hier, int compile_id);
void maat_hierarchy_compile_user_data_iterate(struct maat_hierarchy *hier, void (*callback)(void *user_data, void *param), void *param);
int maat_hierarchy_region_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
int is_last_compile, void **user_data_array, size_t ud_array_sz);
void *maat_hierarchy_region_dettach_user_data(struct maat_hierarchy *hier, int region_id);
/* maat hierarchy region2group API */
int maat_hierarchy_add_region_to_group(struct maat_hierarchy *hier, int group_id, int region_id, int table_id, void* user_data);
int maat_hierarchy_remove_region_from_group(struct maat_hierarchy *hier, int group_id, int region_id);
/* maat hierarchy group2group API */
int maat_hierarchy_add_group_to_group(struct maat_hierarchy *hier, int group_id, int superior_group_id);
int maat_hierarchy_remove_group_from_group(struct maat_hierarchy *hier, int group_id, int superior_group_id);
/* maat hierarchy group2compile API */
int maat_hierarchy_add_group_to_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag,
int clause_index, int compile_id);
int maat_hierarchy_remove_group_from_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag,
int clause_index, int compile_id);
/* maat hierarchy item API */
int maat_hierarchy_item_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
int is_last_compile, void **user_data_array, size_t ud_array_sz);
void *maat_hierarchy_item_dettach_user_data(struct maat_hierarchy *hier, int item_id);
void maat_hierarchy_set_item_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
/* maat hierarchy item2group API */
int maat_hierarchy_add_item_to_group(struct maat_hierarchy *hier, int group_id, int item_id, void* user_data);
int maat_hierarchy_remove_item_from_group(struct maat_hierarchy *hier, int group_id, int item_id);
#ifdef __cpluscplus
}
#endif

52
src/inc_internal/maat_rule.h
View File

@@ -53,34 +53,46 @@ struct maat_rule {
char service_defined[MAX_SERVICE_DEFINE_LEN];
};
#define REGION_RULE_MAGIC 0x4d3c2b1a
struct maat_region_inner
{
#define ITEM_RULE_MAGIC 0x4d3c2b1a
struct maat_item_inner {
long long magic_num;
int region_id;
int item_id;
int group_id;
int district_id;
int table_id;
int expr_id_cnt;
int expr_id_lb; //low boundary
int expr_id_ub; //up boundary
};
#define COMPILE_RULE_MAGIC 0x1a2b3c4d
struct maat_compile_rule
{
struct maat_compile_rule {
long long magic_num;
int compile_id;
struct maat_rule_head head;// fix len of Maat_rule_t
char *service_defined;
int is_valid;
int declared_clause_num;
double evaluation_order;
struct table_schema *ref_table;
void *ex_data;
int compile_id;
void **ex_data;
pthread_rwlock_t rwlock;
};
struct group2compile_rule {
int group_id;
int compile_id;
int is_valid;
int not_flag;
int virtual_table_id;
int clause_index;
};
struct group2group_rule {
int group_id;
int superior_group_id;
int is_valid;
};
struct maat_runtime {
/* maat_runtime can be created and destroy dynamic, so need version info */
long long version;
@@ -94,10 +106,8 @@ struct maat_runtime {
size_t max_thread_num;
uint32_t rule_num;
struct maat_hierarchy *hier;
struct maat_garbage_bin *ref_garbage_bin;
struct scan_result *region_result_buff;
struct scan_result *item_result_buff;
struct log_handle *logger;
};
@@ -158,6 +168,11 @@ struct expected_reply {
redisReply possible_replies[POSSIBLE_REDIS_REPLY_SIZE];
};
struct rule_tag {
char *tag_name;
char *tag_val;
};
struct maat {
char instance_name[NAME_MAX];
@@ -173,6 +188,9 @@ struct maat {
struct source_redis_ctx mr_ctx;
};
struct rule_tag *accept_tags;
int n_accept_tag;
struct log_handle *logger;
int deferred_load;
@@ -192,10 +210,10 @@ struct maat {
struct maat_garbage_bin *garbage_bin;
char compile_tn[NAME_MAX];
char group_tn[NAME_MAX];
char group2compile_tn[NAME_MAX];
char group2group_tn[NAME_MAX];
char compile_tablename[NAME_MAX];
//char group_tn[NAME_MAX];
//char group2compile_tn[NAME_MAX];
//char group2group_tn[NAME_MAX];
char decrypt_key[NAME_MAX];
char decrypt_algo[NAME_MAX];
@@ -263,6 +281,8 @@ void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule);
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id,
const char *table_name, const char *line, long long timeout);
void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head, const char *srv_def, int srv_def_len);
#ifdef __cpluscplus
}
#endif

11
src/inc_internal/maat_table_runtime.h
View File

@@ -20,9 +20,8 @@ extern "C"
#include "maat_table_schema.h"
#include "maat_garbage_collection.h"
struct table_rt_2tuple {
struct ip_addr {
enum ip_type ip_type;
uint16_t port;
union {
uint32_t ipv4;
uint32_t ipv6[4];
@@ -36,7 +35,7 @@ struct table_runtime_manager;
/* table runtime manager API */
struct table_runtime_manager *
table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num,
struct maat_garbage_bin *bin);
struct maat_garbage_bin *bin, struct log_handle *logger);
void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr);
@@ -57,7 +56,7 @@ void table_runtime_update(struct table_runtime *table_rt, struct table_schema *t
*/
int table_runtime_updating_flag(struct table_runtime *table_rt);
void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread, struct log_handle *logger);
void table_runtime_commit(struct table_runtime *table_rt, long long version, size_t nr_worker_thread, struct log_handle *logger);
/* table runtime scan API */
int table_runtime_scan_string(struct table_runtime *table_rt, int thread_id, const char *data, size_t data_len,
@@ -67,7 +66,7 @@ void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id);
int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, int results[], size_t *n_result);
void table_runtime_stream_close(struct table_runtime *table_rt);
int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data, struct scan_result *results, size_t *n_result, size_t n_result_array);
int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct ip_addr *data, struct scan_result *results, size_t *n_result, size_t n_result_array);
/* table runtime cached row API */
size_t table_runtime_cached_row_count(struct table_runtime *table_rt);
@@ -78,7 +77,7 @@ const char* table_runtime_get_cached_row(struct table_runtime *table_rt, size_t
struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt);
void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema,
int nr_worker_num, struct log_handle *logger);
int nr_worker_thread, long long version, struct log_handle *logger);
#ifdef __cpluscplus
}

51
src/inc_internal/maat_table_schema.h
View File

@@ -17,15 +17,18 @@ extern "C"
#endif
#include <stddef.h>
#include <limits.h>
#include "maat/maat.h"
#include "adapter_hs.h"
#define MAX_TABLE_NUM 256
#define MAX_DISTRICT_STR 128
#define MAX_IP_STR 128
#define MAX_KEYWORDS_STR 1024
#define MAX_TABLE_NUM 256
#define MAX_DISTRICT_STR 128
#define MAX_IP_STR 128
#define MAX_KEYWORDS_STR 1024
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_TABLE_LINE_SIZE (1024 * 16)
#define MAX_COMPILE_EX_DATA_NUM 2
enum component_table_type {
COMPONENT_TABLE_TYPE_NONE = -1,
@@ -85,6 +88,34 @@ enum match_method {
MATCH_METHOD_MAX
};
struct compile_item {
int compile_id;
int service_id;
int action;
int do_blacklist;
int do_log;
char user_region[MAX_TABLE_LINE_SIZE];
int is_valid;
int clause_num;
int evaluation_order;
};
struct group2compile_item {
int group_id;
int compile_id;
int is_valid;
int not_flag;
int virtual_table_id;
int clause_index;
int associated_compile_table_id;
};
struct group2group_item {
int group_id;
int superior_group_id;
int is_valid;
};
struct expr_item {
int item_id;
int group_id;
@@ -139,13 +170,14 @@ struct ip_plugin_item {
char end_ip[MAX_IP_STR];
int is_valid;
int rule_tag;
int have_exdata;
void *ex_data;
};
struct table_item {
enum table_type table_type;
union {
struct compile_item compile_item;
struct group2compile_item g2c_item;
struct group2group_item g2g_item;
struct expr_item expr_item;
struct ip_plus_item ip_plus_item;
struct plugin_item plugin_item;
@@ -214,14 +246,19 @@ enum table_type table_schema_get_table_type(struct table_schema *table_schema);
int table_schema_get_table_id(struct table_schema *table_schema);
/* get group2compile table's associated compile table id */
int table_schema_get_associated_table_id(struct table_schema *table_schema);
enum scan_type table_schema_get_scan_type(struct table_schema *table_schema);
struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema,
struct log_handle *logger);
struct rule_tag *accept_tags, int n_accept_tag, struct log_handle *logger);
void table_item_free(struct table_item *table_item);
int table_schema_get_valid_flag_column(struct table_schema *table_schema);
void table_schema_set_updating_name(struct table_schema *table_schema, const char *table_name);
const char *table_schema_get_updating_name(struct table_schema *table_schema);
/* expr table schema API */
enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);