gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hierarchy refactor unfinished

Browse Source
This commit is contained in:
liuwentan
2023-01-06 18:54:59 +08:00
parent 9778267b48
commit 3d4b833e48
18 changed files with 2314 additions and 848 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
include/maat/maat.h
View File

@@ -32,34 +32,45 @@ enum ip_type {
IP_TYPE_V6
};
struct maat_hit_path {
int Nth_scan;
int item_id;
int sub_group_id;
int top_group_id;
int virtual_table_id; // 0 is not a virtual table.
int compile_id;
};
struct maat_matched {
int virtual_table_id;
int group_id;
};
enum maat_scan_opt
{
MAAT_SET_SCAN_DISTRICT = 1, //VALUE is a const char*, SIZE= strlen(string). DEFAULT: no default.
MAAT_SET_SCAN_LAST_REGION, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination.
MAAT_GET_SCAN_HIT_PATH //VALUE is struct Maat_hit_path_t*, an array of struct Maat_hit_path_t, SIZE= sizeof(struct Maat_hit_path_t)*N,
MAAT_GET_SCAN_HIT_PATH, //VALUE is struct maat_hit_path*, an array of struct maat_hit_path, SIZE= sizeof(struct maat_hit_path)*N,
//Maat_get_scan_status returns actual got number.
MAAT_GET_SCAN_MATCHED //VALUE is struct maat_matched*, an array of struct maat_matched, SIZE= sizeof(struct maat_matched)*N,
};
/* network order */
struct ipv4_4tuple {
struct ipv4_2tuple {
uint32_t sip;
uint32_t dip;
uint16_t sport;
uint16_t dport;
};
struct ipv6_4tuple {
struct ipv6_2tuple {
uint8_t sip[16];
uint8_t dip[16];
uint16_t sport;
uint16_t dport;
};
struct addr_4tuple {
struct addr_2tuple {
enum ip_type type;
union {
struct ipv4_4tuple ipv4;
struct ipv6_4tuple ipv6;
struct ipv4_2tuple ipv4;
struct ipv6_2tuple ipv6;
};
};
@@ -83,18 +94,15 @@ typedef void maat_rule_ex_dup_func_t(int idx, void *to, void *from, long argl, v
/* maat_instance options API */
struct maat_options;
struct maat_options* maat_options_new(void);
int maat_options_set_worker_thread_number(struct maat_options *opts, size_t nr_worker_threads);
int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread);
int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, size_t name_len);
int maat_options_set_deferred_load_on(struct maat_options *opts);
int maat_options_set_iris_full_index_dir(struct maat_options *opts, const char *full_idx_dir);
int maat_options_set_iris_inc_index_dir(struct maat_options *opts, const char *inc_idx_dir);
int maat_options_set_iris(struct maat_options *opts, const char *full_directory, const char *increment_directory);
int maat_options_set_json_file(struct maat_options *opts, const char *json_filename);
int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip);
int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port);
int maat_options_set_redis_db(struct maat_options *opts, int db_index);
int maat_options_set_redis(struct maat_options *opts, const char *redis_ip, uint16_t redis_port, int redis_db);
int maat_options_set_logger(struct maat_options *opts, void *logger);
/* maat_instance API */
@@ -115,8 +123,9 @@ int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp);
void *maat_plugin_table_dup_ex_data(struct maat *instance, int table_id,
/* returned data is duplicated by dup_func of maat_plugin_table_ex_schema_register,
caller is responsible to free the data. */
void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id,
const char *key, size_t key_len);
/* maat scan API */
struct maat_state;
@@ -125,7 +134,7 @@ int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
struct maat_state **state);
int maat_scan_ip(struct maat *instance, int table_id, int thread_id,
struct addr_4tuple *addr, int results[], size_t *n_result,
struct addr_2tuple *addr, int results[], size_t *n_result,
struct maat_state **state);
int maat_scan_string(struct maat *instance, int table_id, int thread_id,
@@ -140,12 +149,16 @@ int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* dat
void maat_scan_stream_close(struct maat_stream **stream);
/* maat state API */
int maat_state_set(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, const void *value, int size);
//return >=0 if success, return -1 when failed;
int maat_state_get(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, void *value, int size);
void maat_state_reset(struct maat_state **state);
void maat_state_free(struct maat_state **state);
/* return matched compile_id */
int maat_matched_compile_id(struct maat *instance, struct maat_matched *matched);
#ifdef __cpluscplus
}

16
src/CMakeLists.txt
View File

@@ -29,15 +29,15 @@ set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static pthread crypto z)
# Shared Library Output
#add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES})
#set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX)
#set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe)
#set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1)
#set_target_properties(maat_frame_shared PROPERTIES VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION})
#set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION})
add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES})
set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX)
set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe)
set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1)
set_target_properties(maat_frame_shared PROPERTIES VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION})
set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION})
#target_link_libraries(maat_frame_shared hyperscan_static)
target_link_libraries(maat_frame_shared hyperscan_static)
# install
set(CMAKE_INSTALL_PREFIX /opt/MESA/)
install(FILES ${PROJECT_SOURCE_DIR}/inc/Maat_rule.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER)
#install(FILES ${PROJECT_SOURCE_DIR}/include/maat DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER)

1
src/inc_internal/maat_common.h
View File

@@ -23,6 +23,7 @@ extern "C"
struct maat_options {
char instance_name[NAME_MAX];
char compile_tablename[NAME_MAX];
size_t nr_worker_threads;
int rule_effect_interval_ms;
int rule_update_checking_interval_ms;

66
src/inc_internal/maat_group.h Normal file
View File

@@ -0,0 +1,66 @@
/*
**********************************************************************************************
* File: maat_group.h
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_GROUP_H_
#define _MAAT_GROUP_H_
#ifdef __cpluscplus
extern "C"
{
#endif
struct maat_item {
int item_id;
int group_id;
struct maat_group *ref_parent_group;
UT_hash_handle hh;
void *user_data;
};
struct maat_group {
igraph_integer_t vertex_id;
int group_id;
int ref_by_compile_cnt;
int ref_by_superior_group_cnt;
int ref_by_subordinate_group_cnt;
int ref_by_item_cnt;
size_t top_group_cnt;
int *top_group_ids;
UT_hash_handle hh_group_id;
UT_hash_handle hh_vertex_id;
};
struct maat_group_topology;
struct maat_group_topology *maat_group_topology_new(struct log_handle *logger);
void maat_group_topology_free(struct maat_group_topology *group_topo);
struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id);
void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group);
/**
* @retval if not found, return NULL
*/
struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id);
int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id);
int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id);
/* build top groups */
int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo);
#ifdef __cpluscplus
}
#endif
#endif

31
src/inc_internal/maat_hierarchy.h
View File

@@ -23,46 +23,43 @@ struct maat_hierarchy;
struct maat_hierarchy *maat_hierarchy_new(int thread_num, struct maat_garbage_bin *bin, struct log_handle *logger);
void maat_hierarchy_free(struct maat_hierarchy *hier);
int maat_hierarchy_rebuild(struct maat_hierarchy *hier);
size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
struct maat_hit_path_t *hit_paths, size_t n_path);
void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
void maat_hierarchy_set_region_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
struct maat_hit_path *hit_paths, size_t n_path);
/* maat hierarchy compile mid API */
struct maat_hierarchy_compile_mid;
struct maat_hierarchy_compile_mid *maat_hierarchy_compile_mid_new(struct maat_hierarchy *hier, int thread_id);
void maat_hierarchy_compile_mid_free(struct maat_hierarchy_compile_mid *mid);
void maat_hierarchy_compile_mid_update(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result);
int item_id, int virtual_table_id, int Nth_scan, int Nth_item_result);
int maat_hierarchy_compile_mid_has_NOT_clause(struct maat_hierarchy_compile_mid *mid);
/* maat hierarchy compile API */
int maat_hierarchy_compile_add(struct maat_hierarchy *hier, int compile_id, int declared_clause_num, void* user_data);
int maat_hierarchy_compile_remove(struct maat_hierarchy *hier, int compile_id);
void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
void *maat_hierarchy_compile_dettach_user_data(struct maat_hierarchy *hier, int compile_id);
void *maat_hierarchy_compile_read_user_data(struct maat_hierarchy *hier, int compile_id);
void maat_hierarchy_compile_user_data_iterate(struct maat_hierarchy *hier, void (*callback)(void *user_data, void *param), void *param);
int maat_hierarchy_region_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
int is_last_compile, void **user_data_array, size_t ud_array_sz);
void *maat_hierarchy_region_dettach_user_data(struct maat_hierarchy *hier, int region_id);
/* maat hierarchy region2group API */
int maat_hierarchy_add_region_to_group(struct maat_hierarchy *hier, int group_id, int region_id, int table_id, void* user_data);
int maat_hierarchy_remove_region_from_group(struct maat_hierarchy *hier, int group_id, int region_id);
/* maat hierarchy group2group API */
int maat_hierarchy_add_group_to_group(struct maat_hierarchy *hier, int group_id, int superior_group_id);
int maat_hierarchy_remove_group_from_group(struct maat_hierarchy *hier, int group_id, int superior_group_id);
/* maat hierarchy group2compile API */
int maat_hierarchy_add_group_to_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag,
int clause_index, int compile_id);
int maat_hierarchy_remove_group_from_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag,
int clause_index, int compile_id);
/* maat hierarchy item API */
int maat_hierarchy_item_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
int is_last_compile, void **user_data_array, size_t ud_array_sz);
void *maat_hierarchy_item_dettach_user_data(struct maat_hierarchy *hier, int item_id);
void maat_hierarchy_set_item_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
/* maat hierarchy item2group API */
int maat_hierarchy_add_item_to_group(struct maat_hierarchy *hier, int group_id, int item_id, void* user_data);
int maat_hierarchy_remove_item_from_group(struct maat_hierarchy *hier, int group_id, int item_id);
#ifdef __cpluscplus
}
#endif

52
src/inc_internal/maat_rule.h
View File

@@ -53,34 +53,46 @@ struct maat_rule {
char service_defined[MAX_SERVICE_DEFINE_LEN];
};
#define REGION_RULE_MAGIC 0x4d3c2b1a
struct maat_region_inner
{
#define ITEM_RULE_MAGIC 0x4d3c2b1a
struct maat_item_inner {
long long magic_num;
int region_id;
int item_id;
int group_id;
int district_id;
int table_id;
int expr_id_cnt;
int expr_id_lb; //low boundary
int expr_id_ub; //up boundary
};
#define COMPILE_RULE_MAGIC 0x1a2b3c4d
struct maat_compile_rule
{
struct maat_compile_rule {
long long magic_num;
int compile_id;
struct maat_rule_head head;// fix len of Maat_rule_t
char *service_defined;
int is_valid;
int declared_clause_num;
double evaluation_order;
struct table_schema *ref_table;
void *ex_data;
int compile_id;
void **ex_data;
pthread_rwlock_t rwlock;
};
struct group2compile_rule {
int group_id;
int compile_id;
int is_valid;
int not_flag;
int virtual_table_id;
int clause_index;
};
struct group2group_rule {
int group_id;
int superior_group_id;
int is_valid;
};
struct maat_runtime {
/* maat_runtime can be created and destroy dynamic, so need version info */
long long version;
@@ -94,10 +106,8 @@ struct maat_runtime {
size_t max_thread_num;
uint32_t rule_num;
struct maat_hierarchy *hier;
struct maat_garbage_bin *ref_garbage_bin;
struct scan_result *region_result_buff;
struct scan_result *item_result_buff;
struct log_handle *logger;
};
@@ -158,6 +168,11 @@ struct expected_reply {
redisReply possible_replies[POSSIBLE_REDIS_REPLY_SIZE];
};
struct rule_tag {
char *tag_name;
char *tag_val;
};
struct maat {
char instance_name[NAME_MAX];
@@ -173,6 +188,9 @@ struct maat {
struct source_redis_ctx mr_ctx;
};
struct rule_tag *accept_tags;
int n_accept_tag;
struct log_handle *logger;
int deferred_load;
@@ -192,10 +210,10 @@ struct maat {
struct maat_garbage_bin *garbage_bin;
char compile_tn[NAME_MAX];
char group_tn[NAME_MAX];
char group2compile_tn[NAME_MAX];
char group2group_tn[NAME_MAX];
char compile_tablename[NAME_MAX];
//char group_tn[NAME_MAX];
//char group2compile_tn[NAME_MAX];
//char group2group_tn[NAME_MAX];
char decrypt_key[NAME_MAX];
char decrypt_algo[NAME_MAX];
@@ -263,6 +281,8 @@ void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule);
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id,
const char *table_name, const char *line, long long timeout);
void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head, const char *srv_def, int srv_def_len);
#ifdef __cpluscplus
}
#endif

11
src/inc_internal/maat_table_runtime.h
View File

@@ -20,9 +20,8 @@ extern "C"
#include "maat_table_schema.h"
#include "maat_garbage_collection.h"
struct table_rt_2tuple {
struct ip_addr {
enum ip_type ip_type;
uint16_t port;
union {
uint32_t ipv4;
uint32_t ipv6[4];
@@ -36,7 +35,7 @@ struct table_runtime_manager;
/* table runtime manager API */
struct table_runtime_manager *
table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num,
struct maat_garbage_bin *bin);
struct maat_garbage_bin *bin, struct log_handle *logger);
void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr);
@@ -57,7 +56,7 @@ void table_runtime_update(struct table_runtime *table_rt, struct table_schema *t
*/
int table_runtime_updating_flag(struct table_runtime *table_rt);
void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread, struct log_handle *logger);
void table_runtime_commit(struct table_runtime *table_rt, long long version, size_t nr_worker_thread, struct log_handle *logger);
/* table runtime scan API */
int table_runtime_scan_string(struct table_runtime *table_rt, int thread_id, const char *data, size_t data_len,
@@ -67,7 +66,7 @@ void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id);
int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, int results[], size_t *n_result);
void table_runtime_stream_close(struct table_runtime *table_rt);
int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data, struct scan_result *results, size_t *n_result, size_t n_result_array);
int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct ip_addr *data, struct scan_result *results, size_t *n_result, size_t n_result_array);
/* table runtime cached row API */
size_t table_runtime_cached_row_count(struct table_runtime *table_rt);
@@ -78,7 +77,7 @@ const char* table_runtime_get_cached_row(struct table_runtime *table_rt, size_t
struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt);
void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema,
int nr_worker_num, struct log_handle *logger);
int nr_worker_thread, long long version, struct log_handle *logger);
#ifdef __cpluscplus
}

43
src/inc_internal/maat_table_schema.h
View File

@@ -17,6 +17,7 @@ extern "C"
#endif
#include <stddef.h>
#include <limits.h>
#include "maat/maat.h"
#include "adapter_hs.h"
@@ -26,6 +27,8 @@ extern "C"
#define MAX_IP_STR 128
#define MAX_KEYWORDS_STR 1024
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_TABLE_LINE_SIZE (1024 * 16)
#define MAX_COMPILE_EX_DATA_NUM 2
enum component_table_type {
COMPONENT_TABLE_TYPE_NONE = -1,
@@ -85,6 +88,34 @@ enum match_method {
MATCH_METHOD_MAX
};
struct compile_item {
int compile_id;
int service_id;
int action;
int do_blacklist;
int do_log;
char user_region[MAX_TABLE_LINE_SIZE];
int is_valid;
int clause_num;
int evaluation_order;
};
struct group2compile_item {
int group_id;
int compile_id;
int is_valid;
int not_flag;
int virtual_table_id;
int clause_index;
int associated_compile_table_id;
};
struct group2group_item {
int group_id;
int superior_group_id;
int is_valid;
};
struct expr_item {
int item_id;
int group_id;
@@ -139,13 +170,14 @@ struct ip_plugin_item {
char end_ip[MAX_IP_STR];
int is_valid;
int rule_tag;
int have_exdata;
void *ex_data;
};
struct table_item {
enum table_type table_type;
union {
struct compile_item compile_item;
struct group2compile_item g2c_item;
struct group2group_item g2g_item;
struct expr_item expr_item;
struct ip_plus_item ip_plus_item;
struct plugin_item plugin_item;
@@ -214,14 +246,19 @@ enum table_type table_schema_get_table_type(struct table_schema *table_schema);
int table_schema_get_table_id(struct table_schema *table_schema);
/* get group2compile table's associated compile table id */
int table_schema_get_associated_table_id(struct table_schema *table_schema);
enum scan_type table_schema_get_scan_type(struct table_schema *table_schema);
struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema,
struct log_handle *logger);
struct rule_tag *accept_tags, int n_accept_tag, struct log_handle *logger);
void table_item_free(struct table_item *table_item);
int table_schema_get_valid_flag_column(struct table_schema *table_schema);
void table_schema_set_updating_name(struct table_schema *table_schema, const char *table_name);
const char *table_schema_get_updating_name(struct table_schema *table_schema);
/* expr table schema API */
enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);

326
src/maat_api.cpp
View File

@@ -33,8 +33,7 @@
#define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2
struct maat_state
{
struct maat_state {
struct maat *maat_instance;
int16_t thread_id;
unsigned char is_set_district;
@@ -44,6 +43,26 @@ struct maat_state
struct maat_hierarchy_compile_mid *compile_mid;
};
struct scan_item_hit_wrapper {
int Nth_scan;
struct maat_item_inner* wrapped_items[MAX_SCANNER_HIT_NUM];
size_t n_wrapped_item;
int *virtual_table_ids;
int virtual_table_id;
int is_last_item;
};
inline int scan_state_should_compile_NOT(struct maat_state *mid)
{
if (mid && mid->is_last_scan==1 && mid->compile_mid &&
maat_hierarchy_compile_mid_has_NOT_clause(mid->compile_mid)) {
return 1;
} else {
return 0;
}
}
struct maat_options* maat_options_new(void)
{
struct maat_options *options = ALLOC(struct maat_options, 1);
@@ -59,9 +78,9 @@ struct maat_options* maat_options_new(void)
return options;
}
int maat_options_set_worker_thread_number(struct maat_options *opts, size_t n_worker_threads)
int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread)
{
opts->nr_worker_threads = n_worker_threads;
opts->nr_worker_threads = n_thread;
return 0;
}
@@ -101,25 +120,14 @@ int maat_options_set_deferred_load_on(struct maat_options *opts)
return 0;
}
int maat_options_set_iris_full_index_dir(struct maat_options *opts, const char *full_idx_dir)
int maat_options_set_iris(struct maat_options *opts, const char *full_directory, const char *increment_directory)
{
if (strlen(full_idx_dir) >= NAME_MAX) {
if (strlen(full_directory) >= NAME_MAX || strlen(increment_directory) >= NAME_MAX) {
return -1;
}
memcpy(opts->iris_ctx.full_idx_dir, full_idx_dir, strlen(full_idx_dir));
opts->input_mode = DATA_SOURCE_IRIS_FILE;
return 0;
}
int maat_options_set_iris_inc_index_dir(struct maat_options *opts, const char *inc_idx_dir)
{
if (strlen(inc_idx_dir) >= NAME_MAX) {
return -1;
}
memcpy(opts->iris_ctx.inc_idx_dir, inc_idx_dir, strlen(inc_idx_dir));
memcpy(opts->iris_ctx.full_idx_dir, full_directory, strlen(full_directory));
memcpy(opts->iris_ctx.inc_idx_dir, increment_directory, strlen(increment_directory));
opts->input_mode = DATA_SOURCE_IRIS_FILE;
return 0;
@@ -133,24 +141,12 @@ int maat_options_set_json_file(struct maat_options *opts, const char *json_filen
return 0;
}
int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip)
int maat_options_set_redis(struct maat_options *opts, const char *redis_ip, uint16_t redis_port, int redis_db)
{
memcpy(opts->redis_ctx.redis_ip, redis_ip, strlen(redis_ip));
opts->input_mode = DATA_SOURCE_REDIS;
return 0;
}
int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port)
{
opts->redis_ctx.redis_port = redis_port;
return 0;
}
int maat_options_set_redis_db(struct maat_options *opts, int db_index)
{
opts->redis_ctx.redis_db = db_index;
opts->redis_ctx.redis_db = redis_db;
opts->input_mode = DATA_SOURCE_REDIS;
return 0;
}
@@ -254,6 +250,9 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
goto failed;
}
if (opts->compile_tablename != NULL) {
memcpy(maat_instance->compile_tablename, opts->compile_tablename, strlen(opts->compile_tablename));
}
maat_instance->input_mode = opts->input_mode;
switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS:
@@ -334,16 +333,6 @@ inline void maat_runtime_ref_dec(struct maat_runtime *maat_rt, int thread_id)
alignment_int64_array_add(maat_rt->ref_cnt, thread_id, -1);
}
inline int scan_state_should_compile_NOT(struct maat_state *mid)
{
if (mid && (1 == mid->is_last_scan) && mid->compile_mid &&
maat_hierarchy_compile_mid_has_NOT_clause(mid->compile_mid)) {
return 1;
} else {
return 0;
}
}
int maat_table_callback_register(struct maat *maat_instance, int table_id,
maat_start_callback_t *start,
maat_update_callback_t *update,
@@ -410,15 +399,14 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i
if (maat_instance->maat_rt != NULL) {
table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id);
table_runtime_commit_ex_data_schema(table_rt, table_schema, maat_instance->nr_worker_thread,
maat_instance->logger);
maat_instance->maat_rt->version, maat_instance->logger);
}
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return 0;
}
void *maat_plugin_table_dup_ex_data(struct maat *maat_instance, int table_id,
const char *key, size_t key_len)
void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, const char *key, size_t key_len)
{
struct maat_runtime *maat_rt = maat_instance->maat_rt;
if (NULL == maat_rt) {
@@ -460,7 +448,7 @@ struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id)
return outer_state;
}
struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instance, int thread_id, int is_hit_region)
struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instance, int thread_id, int is_hit_item)
{
struct maat_state *mid = *state;
@@ -478,7 +466,7 @@ struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instanc
alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1);
}
if (is_hit_region == 1) {
if (is_hit_item == 1) {
if (NULL == mid->compile_mid) {
mid->compile_mid = maat_hierarchy_compile_mid_new(maat_instance->maat_rt->hier, thread_id);
alignment_int64_array_add(maat_instance->compile_mid_cnt, thread_id, 1);
@@ -487,6 +475,140 @@ struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instanc
return mid;
}
void scan_item_hit_wrapper_build(struct scan_item_hit_wrapper* wrapper, struct scan_result* results, size_t n_result,
int district_id, int is_last_item, int virtual_table_id, int Nth_scan)
{
size_t i=0;
struct maat_item_inner *item = NULL;
wrapper->n_wrapped_item = 0;
wrapper->virtual_table_id = 0;
wrapper->virtual_table_ids = NULL;
for (i = 0; i < n_result; i++) {
item = (struct maat_item_inner *)(results[i].tag);
if (item->district_id == district_id || district_id == DISTRICT_ANY) {
wrapper->wrapped_items[wrapper->n_wrapped_item] = item;
wrapper->n_wrapped_item++;
}
}
wrapper->is_last_item = is_last_item;
wrapper->virtual_table_id = virtual_table_id;
wrapper->Nth_scan = Nth_scan;
wrapper->virtual_table_ids = NULL;
}
struct compile_sort_para {
double evaluation_order;
int declared_clause_num;
int compile_id;
void *user;
};
static void compile_sort_para_set(struct compile_sort_para *para,
const struct maat_compile_rule *compile_relation, void *user)
{
para->compile_id=compile_relation->compile_id;
para->evaluation_order=compile_relation->evaluation_order;
para->declared_clause_num=compile_relation->declared_clause_num;
para->user = user;
}
static int compile_sort_para_compare(const struct compile_sort_para *a, const struct compile_sort_para *b)
{
//If both of compile rule's evaluation order are specified, compile rule with small evaluation order is priority.
if(a->evaluation_order!=0 && b->evaluation_order!=0)
{
if(a->evaluation_order - b->evaluation_order <0)
{
return -1;
}
else if(a->evaluation_order - b->evaluation_order >0)
{
return 1;
}
}
//If one of compile rule's evaluation order is zero, compile rule with big evaluation order is priority.
else if(a->evaluation_order + b->evaluation_order!= 0)
{
return (a->evaluation_order - b->evaluation_order >0) ? -1 : 1;
}
//If compile rule's execute sequences are not specified or equal.
if(a->declared_clause_num!=b->declared_clause_num)
{
return (a->declared_clause_num-b->declared_clause_num);
}
else
{
return (b->compile_id-a->compile_id);
}
}
static int compare_compile_rule(const void *a, const void *b)
{
const struct maat_compile_rule *ra=*(const struct Maat_compile_rule **)a;
const struct maat_compile_rule *rb=*(const struct Maat_compile_rule **)b;
struct compile_sort_para sa, sb;
compile_sort_para_set(&sa, ra, NULL);
compile_sort_para_set(&sb, rb, NULL);
return compile_sort_para_compare(&sa, &sb);
}
int item_compile(struct maat *maat_instance, struct maat_hierarchy_compile_mid *compile_mid,
const struct scan_item_hit_wrapper *item_hit_wrapper, int *result,
int size, int thread_id)
{
int is_last_item = item_hit_wrapper->is_last_item;
size_t item_hit_num = item_hit_wrapper->n_wrapped_item;
int scan_ret=0;
int i=0;
struct maat_compile_rule *compile_rule_array[size];
struct maat_compile_rule *compile_rule = NULL;
int virtual_table_id = 0;
struct maat_item_inner *item = NULL;
for (i = 0; (size_t)i < item_hit_num;i++) {
item = item_hit_wrapper->wrapped_items[i];
assert(item->magic_num == ITEM_RULE_MAGIC);
if (item_hit_wrapper->virtual_table_ids) {
virtual_table_id = item_hit_wrapper->virtual_table_ids[i];
} else {
virtual_table_id = item_hit_wrapper->virtual_table_id;
}
maat_hierarchy_compile_mid_update(feather->scanner->hier, compile_mid, item->item_id, virtual_table_id, item_hit_wrapper->Nth_scan, i);
}
scan_ret = maat_hierarchy_item_compile(feather->scanner->hier, compile_mid, is_last_item, (void **)compile_rule_array, size);
//Maat_hierarchy is rwlock protected, it always returns non-NULL compile_rule.
if (scan_ret > 1) {
qsort(compile_rule_array, scan_ret, sizeof(struct maat_compile_rule *),
compare_compile_rule);
}
for (i = 0; i < scan_ret && i < size; i++) {
compile_rule = compile_rule_array[i];
assert(compile_rule->magic_num == COMPILE_RULE_MAGIC);
result[i] = compile_rule->compile_id;
/*
fill_maat_rule(&(result[i]), &(compile_rule->head), compile_rule->service_defined,
compile_rule->head.serv_def_len);*/
}
if (scan_ret > 0) {
alignment_int64_array_add(feather->hit_cnt, thread_id, 1);
}
if (item_hit_num == 0 && scan_ret > 0) {
alignment_int64_array_add(feather->not_grp_hit_cnt, thread_id, 1);
}
return MIN(scan_ret, size);
}
int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
unsigned int intval, int results[], size_t *n_result,
struct maat_state **state)
@@ -494,19 +616,17 @@ int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
return 0;
}
static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4tuple *addr, enum component_table_type child_type)
static int ip_scan_data_set(struct ip_addr *scan_data, struct addr_2tuple *addr, enum component_table_type child_type)
{
switch (addr->type) {
case IP_TYPE_V4:
table_rt_addr->ip_type = IP_TYPE_V4;
scan_data->ip_type = IP_TYPE_V4;
switch (child_type) {
case COMPONENT_TABLE_TYPE_SIP:
table_rt_addr->ipv4 = ntohl(addr->ipv4.sip);
table_rt_addr->port = ntohs(addr->ipv4.sport);
scan_data->ipv4 = ntohl(addr->ipv4.sip);
break;
case COMPONENT_TABLE_TYPE_DIP:
table_rt_addr->ipv4 = ntohl(addr->ipv4.dip);
table_rt_addr->port = ntohs(addr->ipv4.dport);
scan_data->ipv4 = ntohl(addr->ipv4.dip);
break;
default:
assert(0);
@@ -514,17 +634,15 @@ static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4
}
break;
case IP_TYPE_V6:
table_rt_addr->ip_type = IP_TYPE_V6;
scan_data->ip_type = IP_TYPE_V6;
switch (child_type) {
case COMPONENT_TABLE_TYPE_SIP:
memcpy(table_rt_addr->ipv6, addr->ipv6.sip, sizeof(addr->ipv6.sip));
ipv6_ntoh(table_rt_addr->ipv6);
table_rt_addr->port = ntohs(addr->ipv6.sport);
memcpy(scan_data->ipv6, addr->ipv6.sip, sizeof(addr->ipv6.sip));
ipv6_ntoh(scan_data->ipv6);
break;
case COMPONENT_TABLE_TYPE_DIP:
memcpy(table_rt_addr->ipv6, addr->ipv6.dip, sizeof(addr->ipv6.dip));
ipv6_ntoh(table_rt_addr->ipv6);
table_rt_addr->port = ntohs(addr->ipv6.dport);
memcpy(scan_data->ipv6, addr->ipv6.dip, sizeof(addr->ipv6.dip));
ipv6_ntoh(scan_data->ipv6);
break;
default:
assert(0);
@@ -539,12 +657,12 @@ static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4
return 0;
}
static int ip_composition_scan(int thread_id, struct addr_4tuple *addr,
static int ip_composition_scan(int thread_id, struct addr_2tuple *addr,
int parent_table_id, enum component_table_type child_type,
int *virtual_table_id,
struct table_schema_manager *table_schema_mgr,
struct table_runtime_manager *table_rt_mgr,
struct scan_result *region_results, size_t n_result_array)
struct scan_result *item_results, size_t n_result_array)
{
int child_table_id = 0;
@@ -576,12 +694,12 @@ static int ip_composition_scan(int thread_id, struct addr_4tuple *addr,
return 0;
}
struct table_rt_2tuple scan_data;
memset(&scan_data, 0, sizeof(struct table_rt_2tuple));
struct ip_addr scan_data;
memset(&scan_data, 0, sizeof(struct ip_addr));
ip_scan_data_set(&scan_data, addr, child_type);
size_t hit_cnt = 0;
int ret = table_runtime_scan_ip(table_rt, thread_id, &scan_data, region_results, &hit_cnt, n_result_array);
int ret = table_runtime_scan_ip(table_rt, thread_id, &scan_data, item_results, &hit_cnt, n_result_array);
if (ret < 0) {
return -1;
}
@@ -589,7 +707,7 @@ static int ip_composition_scan(int thread_id, struct addr_4tuple *addr,
return hit_cnt;
}
int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct addr_4tuple *addr,
int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct addr_2tuple *addr,
int results[], size_t *n_result, struct maat_state **state)
{
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
@@ -615,49 +733,70 @@ int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct
return -1;
}
int region_ret = 0;
int item_ret = 0;
int virtual_table_id = 0;
struct scan_result *region_result = maat_rt->region_result_buff + thread_id * MAX_SCANNER_HIT_NUM;
int region_hit_cnt = 0;
int region_result_virtual_table_ids[MAX_SCANNER_HIT_NUM];
struct scan_result *item_result = maat_rt->item_result_buff + thread_id * MAX_SCANNER_HIT_NUM;
int item_hit_cnt = 0;
int item_result_virtual_table_ids[MAX_SCANNER_HIT_NUM];
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
maat_runtime_ref_inc(maat_rt, thread_id);
if (table_type == TABLE_TYPE_COMPOSITION) {
region_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_SIP, &virtual_table_id,
/*
item_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_SIP, &virtual_table_id,
maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr,
region_result + region_hit_cnt, MAX_SCANNER_HIT_NUM - region_hit_cnt);
region_hit_cnt += region_ret;
/*enum component_table_type childs[3] = {COMPONENT_TABLE_TYPE_SIP, COMPONENT_TABLE_TYPE_DIP, COMPONENT_TABLE_TYPE_SESSION};
item_result + item_hit_cnt, MAX_SCANNER_HIT_NUM - item_hit_cnt);
item_hit_cnt += item_ret; */
enum component_table_type childs[3] = {COMPONENT_TABLE_TYPE_SIP, COMPONENT_TABLE_TYPE_DIP, COMPONENT_TABLE_TYPE_SESSION};
for (int i = 0; i < 3; i++) {
region_ret = ip_composition_scan(thread_id, addr, table_id, childs[i], &virtual_table_id,
item_ret = ip_composition_scan(thread_id, addr, table_id, childs[i], &virtual_table_id,
maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr,
region_result + region_hit_cnt, MAX_SCANNER_HIT_NUM - region_hit_cnt);
if (region_ret < 0) {
item_result + item_hit_cnt, MAX_SCANNER_HIT_NUM - item_hit_cnt);
if (item_ret < 0) {
maat_instance->scan_err_cnt++;
} else {
for (int j = 0; j < region_ret; j++) {
region_result_virtual_table_ids[region_hit_cnt++] = virtual_table_id;
for (int j = 0; j < item_ret; j++) {
item_result_virtual_table_ids[item_hit_cnt++] = virtual_table_id;
}
}
}
}*/
} else {
region_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_NONE, &virtual_table_id,
item_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_NONE, &virtual_table_id,
maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr,
region_result + region_hit_cnt, MAX_SCANNER_HIT_NUM - region_hit_cnt);
if (region_ret < 0) {
item_result + item_hit_cnt, MAX_SCANNER_HIT_NUM - item_hit_cnt);
if (item_ret < 0) {
maat_instance->scan_err_cnt++;
} else {
region_hit_cnt += region_ret;
item_hit_cnt += item_ret;
}
}
*n_result = region_ret;
for (int i = 0; i < region_ret; i++) {
results[i] = region_result[i].rule_id;
int compile_ret = 0;
struct scan_item_hit_wrapper item_hit_wrapper;
if (item_hit_cnt > 0 || scan_state_should_compile_NOT(mid)) {
mid = grab_mid(state, maat_instance, thread_id, 1);
scan_item_hit_wrapper_build(&item_hit_wrapper, item_result, item_hit_cnt, -1,
mid->is_last_scan, virtual_table_id, mid->scan_cnt);
if (table_type == TABLE_TYPE_COMPOSITION) {
item_hit_wrapper.virtual_table_ids = item_result_virtual_table_ids;
}
return 0;
compile_ret = item_compile(maat_instance, mid->compile_mid,
&item_hit_wrapper,
results, item_hit_cnt,
thread_id);
assert(mid->is_last_scan < 2);
if (mid->is_last_scan == 1) {
mid->is_last_scan = 2;
}
}
maat_runtime_ref_dec(maat_rt, thread_id);
if (compile_ret == 0 && item_hit_cnt > 0) {
return -2;
}
*n_result = compile_ret;
return compile_ret;
}
int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
@@ -705,7 +844,12 @@ int maat_state_get(struct maat *instance, struct maat_state **mid, enum maat_sca
}
void maat_state_reset(struct maat_state **state)
void maat_state_free(struct maat_state **state)
{
}
int maat_matched_compile_id(struct maat *instance, struct maat_matched *matched)
{
}

313
src/maat_group.cpp Normal file
View File

@@ -0,0 +1,313 @@
/*
**********************************************************************************************
* File: maat_group.cpp
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include <assert.h>
#include <pthread.h>
#include "maat_group.h"
#include "utils.h"
#include "maat_utils.h"
#include "uthash/uthash.h"
#include "igraph/igraph.h"
#include "log/log.h"
#define MODULE_GROUP module_name_str("maat.group")
struct maat_group_topology {
struct maat_group *hash_group_by_id; //key: group_id, value: struct maat_group *.
struct maat_group *hash_group_by_vertex; //key: vetex_id, value: struct maat_group *. Multimap (Items with multiple keys).
igraph_t group_graph;
igraph_integer_t group_graph_vcount;
igraph_vector_t dfs_vids;
igraph_integer_t grp_vertex_id_generator;
pthread_rwlock_t rwlock;
struct log_handle *logger;
};
void group_vertex_free(struct maat_group *group)
{
free(group->top_group_ids);
free(group);
}
struct maat_group_topology *maat_group_topology_new(struct log_handle *logger)
{
struct maat_group_topology *group_topo = ALLOC(struct maat_group_topology, 1);
UNUSED int ret = 0;
group_topo->hash_group_by_id = NULL;
group_topo->hash_group_by_vertex = NULL;
ret = igraph_empty(&group_topo->group_graph, 0, IGRAPH_DIRECTED);
assert(ret == IGRAPH_SUCCESS);
ret = pthread_rwlock_init(&group_topo->rwlock, NULL);
assert(ret == 0);
group_topo->logger = logger;
return group_topo;
}
void maat_group_topology_free(struct maat_group_topology *group_topo)
{
struct maat_group *group = NULL, *tmp_group = NULL;
HASH_CLEAR(hh_vertex_id, group_topo->hash_group_by_vertex);//No need group memory clean up.
HASH_ITER(hh_group_id, group_topo->hash_group_by_id, group, tmp_group) {
HASH_DELETE(hh_group_id, group_topo->hash_group_by_id, group);
group_vertex_free(group);
}
assert(group_topo->hash_group_by_id == NULL);
igraph_destroy(&group_topo->group_graph);
pthread_rwlock_unlock(&group_topo->rwlock);
pthread_rwlock_destroy(&group_topo->rwlock);
}
size_t print_igraph_vector(igraph_vector_t *v, char *buff, size_t sz) {
long int i;
int printed = 0;
for (i = 0; i < igraph_vector_size(v); i++) {
printed += snprintf(buff + printed, sz - printed, " %li", (long int) VECTOR(*v)[i]);
}
return printed;
}
struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id)
{
struct maat_group *group = ALLOC(struct maat_group, 1);
group->group_id = group_id;
group->vertex_id = group_topo->grp_vertex_id_generator++;
assert(igraph_vcount(&group_topo->group_graph)==group->vertex_id);
igraph_add_vertices(&group_topo->group_graph, 1, NULL); //Add 1 vertice.
HASH_ADD(hh_group_id, group_topo->hash_group_by_id, group_id, sizeof(group->group_id), group);
HASH_ADD(hh_vertex_id, group_topo->hash_group_by_vertex, vertex_id, sizeof(group->vertex_id), group);
return group;
}
void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group)
{
igraph_vector_t v;
char buff[4096] = {0};
assert(group->ref_by_compile_cnt == 0 && group->ref_by_superior_group_cnt == 0);
igraph_vector_init(&v, 8);
igraph_neighbors(&group_topo->group_graph, &v, group->vertex_id, IGRAPH_ALL);
if (igraph_vector_size(&v) > 0) {
print_igraph_vector(&v, buff, sizeof(buff));
log_error(group_topo->logger, MODULE_GROUP, "Del group %d exception, still reached by %s.",
group->vertex_id, buff);
assert(0);
}
igraph_vector_destroy(&v);
assert(group->top_group_ids==NULL);
//We should not call igraph_delete_vertices, because this is function changes the ids of the vertices.
//igraph_delete_vertices(&hier->group_graph, igraph_vss_1(group->vertex_id));
HASH_DELETE(hh_group_id, group_topo->hash_group_by_id, group);
HASH_DELETE(hh_vertex_id, group_topo->hash_group_by_vertex, group);
group_vertex_free(group);
}
struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id)
{
struct maat_group *group = NULL;
HASH_FIND(hh_group_id, group_topo->hash_group_by_id, &group_id, sizeof(group_id), group);
return group;
}
int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id)
{
int ret = 0;
igraph_integer_t edge_id;
struct maat_group *group = NULL, *superior_group = NULL;
group = maat_group_topology_find_group(group_topo, group_id);
if (NULL == group) {
group = maat_group_topology_add_group(group_topo, group_id);
}
superior_group = maat_group_topology_find_group(group_topo, superior_group_id);
if (NULL == superior_group) {
superior_group = maat_group_topology_add_group(group_topo, superior_group_id);
}
ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0);
//No duplicated edges between two groups.
if (edge_id > 0) {
log_error(group_topo->logger, MODULE_GROUP,
"Add group %d to group %d failed, relation already exisited.",
group->group_id, superior_group->group_id);
ret = -1;
} else {
igraph_add_edge(&group_topo->group_graph, group->vertex_id, superior_group->vertex_id);
group->ref_by_superior_group_cnt++;
superior_group->ref_by_subordinate_group_cnt++;
ret = 0;
}
return ret;
}
int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id)
{
int ret = 0;
struct maat_group *group = NULL, *superior_group = NULL;
//No hash write operation, LOCK protection is unnecessary.
group = maat_group_topology_find_group(group_topo, group_id);
if (NULL == group) {
log_error(group_topo->logger, MODULE_GROUP,
"Del group %d from group %d failed, group %d not exisited.",
group_id, superior_group_id, group_id);
return -1;
}
superior_group = maat_group_topology_find_group(group_topo, superior_group_id);
if (NULL == superior_group) {
log_error(group_topo->logger, MODULE_GROUP,
"Del group %d from group %d failed, superior group %d not exisited.",
group_id, superior_group_id, superior_group_id);
return -1;
}
igraph_es_t es;
igraph_integer_t edge_num_before = 0, edge_num_after = 0;
edge_num_before = igraph_ecount(&group_topo->group_graph);
// The edges between the given pairs of vertices will be included in the edge selection.
//The vertex pairs must be given as the arguments of the function call, the third argument
//is the first vertex of the first edge, the fourth argument is the second vertex of the
//first edge, the fifth is the first vertex of the second edge and so on. The last element
//of the argument list must be -1 to denote the end of the argument list.
//https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small
ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1);
assert(ret==IGRAPH_SUCCESS);
// ignore no such edge to abort().
igraph_set_error_handler(igraph_error_handler_ignore);
ret = igraph_delete_edges(&group_topo->group_graph, es);
edge_num_after = igraph_ecount(&group_topo->group_graph);
igraph_es_destroy(&es);
if (ret != IGRAPH_SUCCESS || edge_num_before - edge_num_after != 1) {
assert(0);
return -1;
}
group->ref_by_superior_group_cnt--;
superior_group->ref_by_subordinate_group_cnt--;
return 0;
}
static size_t effective_vertices_count(igraph_vector_t *vids)
{
size_t i = 0;
int tmp_vid = 0;
for (i = 0; i < (size_t)igraph_vector_size(vids); i++) {
tmp_vid = (int) VECTOR(*vids)[i];
if (tmp_vid < 0) {
break;
}
}
return i;
}
int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo)
{
struct maat_group *group = NULL, *tmp = NULL;
struct maat_group *superior_group = NULL;
int tmp_vid=0;
size_t i=0, top_group_cnt=0;
int* temp_group_ids=NULL;
igraph_bool_t is_dag;
igraph_is_dag(&(group_topo->group_graph), &is_dag);
if (!is_dag) {
log_error(group_topo->logger, MODULE_GROUP, "Sub group cycle detected!");
return -1;
}
group_topo->group_graph_vcount = igraph_vcount(&group_topo->group_graph);
igraph_vector_init(&(group_topo->dfs_vids), group_topo->group_graph_vcount);
HASH_ITER (hh_group_id, group_topo->hash_group_by_id, group, tmp) {
top_group_cnt = 0;
temp_group_ids = NULL;
//Orphan, Not reference by any one, free it.
if (0 == group->ref_by_compile_cnt
&& 0 == group->ref_by_superior_group_cnt
&& 0 == group->ref_by_subordinate_group_cnt
&& 0 == group->ref_by_item_cnt) {
FREE(group->top_group_ids);
maat_group_topology_remove_group(group_topo, group);
continue;
}
//A group is need to build top groups when it has items and referenced by superior groups or compiles.
if (group->ref_by_item_cnt > 0 &&
(group->ref_by_compile_cnt > 0 || group->ref_by_superior_group_cnt > 0)) {
if (0 == group->ref_by_superior_group_cnt) {
//fast path, group is only referenced by compile rules.
top_group_cnt = 1;
temp_group_ids = ALLOC(int, top_group_cnt);
temp_group_ids[0] = group->group_id;
} else {
igraph_vector_t *vids = &(group_topo->dfs_vids);
igraph_dfs(&group_topo->group_graph, group->vertex_id, IGRAPH_OUT,
0, vids, NULL, NULL, NULL, NULL, NULL, NULL);
temp_group_ids = ALLOC(int, effective_vertices_count(vids));
for (size_t i = 0; i < (size_t)igraph_vector_size(vids); i++) {
tmp_vid = (int) VECTOR(*vids)[i];
if (tmp_vid < 0) {
break;
}
HASH_FIND(hh_vertex_id, group_topo->hash_group_by_vertex, &tmp_vid, sizeof(tmp_vid), superior_group);
//including itself
if (superior_group->ref_by_compile_cnt > 0) {
temp_group_ids[top_group_cnt] = superior_group->group_id;
top_group_cnt++;
}
}
}
}
free(group->top_group_ids);
group->top_group_cnt = top_group_cnt;
group->top_group_ids = ALLOC(int, group->top_group_cnt);
memcpy(group->top_group_ids, temp_group_ids, sizeof(int)*group->top_group_cnt);
FREE(temp_group_ids);
}
igraph_vector_destroy(&group_topo->dfs_vids);
return 0;
}

740
src/maat_hierarchy.cpp
View File

File diff suppressed because it is too large Load Diff

60
src/maat_rule.cpp
View File

@@ -30,50 +30,6 @@
#define MODULE_MAAT_RULE module_name_str("maat.rule")
void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head,
const char *srv_def, int srv_def_len)
{
memcpy(rule, rule_head, sizeof(struct maat_rule_head));
memcpy(rule->service_defined, srv_def, MIN(srv_def_len, MAX_SERVICE_DEFINE_LEN));
}
void rule_ex_data_free(const struct maat_rule_head *rule_head, const char *srv_def, void *ex_data,
const struct compile_ex_data_schema *ex_schema)
{
struct maat_rule rule;
memset(&rule, 0, sizeof(rule));
fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1);
ex_schema->free_func(ex_schema->idx, &rule, srv_def, ex_data, ex_schema->argl, ex_schema->argp);
}
void destroy_compile_rule(struct maat_compile_rule *compile_rule)
{
size_t n_compile_ex_schema = table_schema_compile_rule_ex_data_schema_count(compile_rule->ref_table);
assert(compile_rule->magic_num == COMPILE_RULE_MAGIC);
for (size_t i = 0; i < n_compile_ex_schema; i++) {
struct compile_ex_data_schema *compile_ex_schema = table_schema_get_compile_rule_ex_data_schema(compile_rule->ref_table, i);
rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ex_data+i, compile_ex_schema);
void *ex_data = compile_rule->ex_data + 1;
ex_data = NULL;
}
free(compile_rule->ex_data);
compile_rule->is_valid = 0;
compile_rule->declared_clause_num = -1;
FREE(compile_rule->service_defined);
FREE(compile_rule);
}
void maat_region_inner_free(struct maat_region_inner *region)
{
assert(region->magic_num == REGION_RULE_MAGIC);
assert(region->expr_id_cnt == 0 ||region->expr_id_cnt == (region->expr_id_ub - region->expr_id_lb + 1));
region->magic_num = 0;
FREE(region);
}
struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance)
{
struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1);
@@ -81,17 +37,14 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in
maat_rt->version = version;
maat_rt->table_rt_mgr = table_runtime_manager_create(maat_instance->table_schema_mgr,
maat_instance->nr_worker_thread,
maat_instance->garbage_bin);
maat_instance->garbage_bin,
maat_instance->logger);
maat_rt->max_table_num = table_schema_manager_get_size(maat_instance->table_schema_mgr);
maat_rt->max_thread_num = maat_instance->nr_worker_thread;
maat_rt->hier = maat_hierarchy_new(maat_instance->nr_worker_thread, maat_instance->garbage_bin,
maat_instance->logger);
maat_hierarchy_set_compile_user_data_free_func(maat_rt->hier, (void (*)(void*))destroy_compile_rule);
maat_hierarchy_set_region_user_data_free_func(maat_rt->hier, (void (*)(void*))maat_region_inner_free);
maat_rt->logger = maat_instance->logger;
maat_rt->ref_garbage_bin = maat_instance->garbage_bin;
maat_rt->region_result_buff = ALLOC(struct scan_result, MAX_SCANNER_HIT_NUM * maat_instance->nr_worker_thread);
maat_rt->item_result_buff = ALLOC(struct scan_result, MAX_SCANNER_HIT_NUM * maat_instance->nr_worker_thread);
maat_rt->ref_cnt = alignment_int64_array_alloc(maat_instance->nr_worker_thread);
return maat_rt;
@@ -105,7 +58,7 @@ void maat_runtime_commit(struct maat_runtime *maat_rt, struct log_handle *logger
continue;
}
table_runtime_commit(table_rt, maat_rt->max_thread_num, logger);
table_runtime_commit(table_rt, maat_rt->version, maat_rt->max_thread_num, logger);
}
maat_rt->last_update_time = time(NULL);
@@ -180,11 +133,12 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param)
maat_rt = maat_instance->maat_rt;
}
struct table_item *table_item = table_schema_line_to_item(line, table_schema, maat_instance->logger);
struct table_item *table_item = table_schema_line_to_item(line, table_schema, maat_instance->accept_tags,
maat_instance->n_accept_tag, maat_instance->logger);
if (table_item != NULL) {
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id);
table_runtime_update(table_rt, table_schema, line, table_item, maat_instance->logger);
FREE(table_item);
table_item_free(table_item);
}
return 0;

553
src/maat_table_runtime.cpp
View File

@@ -16,7 +16,9 @@
#include "utils.h"
#include "log/log.h"
#include "cJSON/cJSON.h"
#include "maat_utils.h"
#include "maat_table_schema.h"
#include "maat_table_runtime.h"
#include "uthash/uthash.h"
#include "maat_ex_data.h"
@@ -24,6 +26,10 @@
#include "rcu_hash.h"
#include "IPMatcher.h"
#include "alignment.h"
#include "maat_hierarchy.h"
#include "maat_rule.h"
#include "igraph/igraph.h"
#include "maat_group.h"
#define MODULE_TABLE_RUNTIME module_name_str("maat.table_runtime")
@@ -40,6 +46,20 @@ struct ex_container_ctx {
struct ex_data_schema *ex_schema;
};
struct compile_runtime {
struct maat_hierarchy *hier;
};
struct group2compile_runtime {
long long not_flag_group;
struct compile_runtime *ref_compile_rt;
struct group2group_runtime *ref_g2g_rt;
};
struct group2group_runtime {
struct maat_group_topology *group_topo;
};
struct expr_runtime {
enum hs_scan_mode scan_mode;
struct adapter_hs *hs;
@@ -62,8 +82,10 @@ struct table_runtime {
uint32_t updating_rule_num;
enum table_type table_type;
union {
struct compile_runtime compile_rt;
struct group2compile_runtime g2c_rt;
struct group2group_runtime g2g_rt;
struct expr_runtime expr_rt;
//struct ip_plugin_runtime ip_rt;
struct ip_plugin_runtime ip_plus_rt;
struct plugin_runtime plugin_rt;
struct ip_plugin_runtime ip_plugin_rt;
@@ -82,6 +104,7 @@ struct table_runtime_manager {
struct table_runtime **table_rt;
size_t n_table_rt;
struct maat_item *hash_item_by_id;
struct maat_garbage_bin *garbage_bin;
};
@@ -116,34 +139,108 @@ void expr_rule_free(and_expr_t *expr_rule)
for (size_t i = 0; i < expr_rule->n_patterns; i++) {
FREE(expr_rule->patterns[i].pat);
}
FREE(expr_rule);
}
void expr_ex_data_free(void *user_ctx, void *data)
{
and_expr_t *expr_rule = (and_expr_t *)data;
expr_rule_free(expr_rule);
FREE(data);
}
struct table_runtime *table_runtime_new(struct table_schema *table_schema, int max_thread_num, struct maat_garbage_bin* bin)
void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head,
const char *srv_def, int srv_def_len)
{
memcpy(rule, rule_head, sizeof(struct maat_rule_head));
memcpy(rule->service_defined, srv_def, MIN(srv_def_len, MAX_SERVICE_DEFINE_LEN));
}
void *rule_ex_data_new(const struct maat_rule_head *rule_head, const char *srv_def, const struct compile_ex_data_schema *ex_schema)
{
void *ex_data = NULL;
struct maat_rule rule;
fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1);
ex_schema->new_func(ex_schema->idx, &rule, srv_def, &ex_data, ex_schema->argl, ex_schema->argp);
return ex_data;
}
void rule_ex_data_free(const struct maat_rule_head *rule_head, const char *srv_def, void *ex_data,
const struct compile_ex_data_schema *ex_schema)
{
struct maat_rule rule;
memset(&rule, 0, sizeof(rule));
fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1);
ex_schema->free_func(ex_schema->idx, &rule, srv_def, ex_data, ex_schema->argl, ex_schema->argp);
}
void destroy_compile_rule(struct maat_compile_rule *compile_rule)
{
size_t n_compile_ex_schema = table_schema_compile_rule_ex_data_schema_count(compile_rule->ref_table);
assert(compile_rule->magic_num == COMPILE_RULE_MAGIC);
for (size_t i = 0; i < n_compile_ex_schema; i++) {
struct compile_ex_data_schema *compile_ex_schema = table_schema_get_compile_rule_ex_data_schema(compile_rule->ref_table, i);
rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ex_data+i, compile_ex_schema);
void *ex_data = compile_rule->ex_data + 1;
ex_data = NULL;
}
free(compile_rule->ex_data);
compile_rule->is_valid = 0;
compile_rule->declared_clause_num = -1;
FREE(compile_rule->service_defined);
FREE(compile_rule);
}
void maat_item_inner_free(struct maat_item_inner *item)
{
assert(item->magic_num == ITEM_RULE_MAGIC);
assert(item->expr_id_cnt == 0 || item->expr_id_cnt == (item->expr_id_ub - item->expr_id_lb + 1));
item->magic_num = 0;
FREE(item);
}
struct table_runtime *table_runtime_new(struct table_schema *table_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
struct ex_container_ctx *ex_container_ctx = NULL;
int table_id = table_schema_get_table_id(table_schema);
struct table_runtime *table_rt = ALLOC(struct table_runtime, 1);
table_rt->ref_garbage_bin = bin;
table_rt->ref_garbage_bin = garbage_bin;
table_rt->table_type = table_schema_get_table_type(table_schema);
switch (table_rt->table_type) {
case TABLE_TYPE_COMPILE:
table_rt->compile_rt.hier = maat_hierarchy_new(max_thread_num, garbage_bin, logger);
maat_hierarchy_set_compile_user_data_free_func(table_rt->compile_rt.hier, (void (*)(void*))destroy_compile_rule);
maat_hierarchy_set_item_user_data_free_func(table_rt->compile_rt.hier, (void (*)(void*))maat_item_inner_free);
break;
case TABLE_TYPE_GROUP2COMPILE:
break;
case TABLE_TYPE_GROUP2GROUP:
table_rt->g2g_rt.group_topo = maat_group_topology_new(logger);
break;
case TABLE_TYPE_EXPR:
table_rt->expr_rt.htable = rcu_hash_new(expr_ex_data_free);
table_rt->expr_rt.scan_mode = expr_table_schema_get_scan_mode(table_schema);
break;
case TABLE_TYPE_IP_PLUS:
table_rt->ip_plus_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free);
ex_container_ctx = ALLOC(struct ex_container_ctx, 1);
ex_container_ctx->custom_data_free = free;
//ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx);
break;
case TABLE_TYPE_PLUGIN:
table_rt->plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free);
break;
case TABLE_TYPE_IP_PLUGIN:
table_rt->ip_plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free);
ex_container_ctx = ALLOC(struct ex_container_ctx, 1);
ex_container_ctx->custom_data_free = free;
//ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx);
break;
default:
break;
@@ -163,6 +260,14 @@ void table_runtime_free(struct table_runtime * table_rt)
}
switch (table_rt->table_type) {
case TABLE_TYPE_COMPILE:
maat_hierarchy_free(table_rt->compile_rt.hier);
break;
case TABLE_TYPE_GROUP2COMPILE:
break;
case TABLE_TYPE_GROUP2GROUP:
maat_group_topology_free(table_rt->group2group_rt.group_topo);
break;
case TABLE_TYPE_EXPR:
adapter_hs_destroy(table_rt->expr_rt.hs);
rcu_hash_free(table_rt->expr_rt.htable);
@@ -183,7 +288,7 @@ void table_runtime_free(struct table_runtime * table_rt)
struct table_runtime_manager *
table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num,
struct maat_garbage_bin* garbage_bin)
struct maat_garbage_bin *garbage_bin, struct log_handle *logger)
{
if (NULL == table_schema_mgr) {
return NULL;
@@ -193,12 +298,33 @@ table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int
table_rt_mgr->n_table_rt = table_schema_manager_get_size(table_schema_mgr);
table_rt_mgr->table_rt = ALLOC(struct table_runtime *, table_rt_mgr->n_table_rt);
for (size_t i = 0; i < table_rt_mgr->n_table_rt; i++) {
size_t i = 0;
for (i = 0; i < table_rt_mgr->n_table_rt; i++) {
struct table_schema *table_schema = table_schema_get(table_schema_mgr, i);
if (NULL == table_schema) {
continue;
}
table_rt_mgr->table_rt[i] = table_runtime_new(table_schema, max_thread_num, garbage_bin);
table_rt_mgr->table_rt[i] = table_runtime_new(table_schema, max_thread_num, garbage_bin, logger);
}
/* group2compile table_rt depends on associated compile table_rt,
must make sure associated compile table_rt already exist */
for (i = 0; i < table_rt_mgr->n_table_rt; i++) {
struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, i);
if (NULL == table_rt) {
continue;
}
enum table_type table_type = table_runtime_get_type(table_rt);
if (table_type != TABLE_TYPE_GROUP2COMPILE) {
continue;
}
struct table_schema *table_schema = table_schema_get(table_schema_mgr, i);
int associated_compile_table_id = table_schema_get_associated_table_id(table_schema);
struct table_runtime *compile_table_rt = table_runtime_get(table_rt_mgr, associated_compile_table_id);
table_rt->group2compile_rt.ref_compile_rt = &compile_table_rt->compile_rt;
assert(table_rt->group2compile_rt.ref_compile_rt != NULL);
}
return table_rt_mgr;
@@ -294,7 +420,7 @@ enum IP_TYPE ip_type_transform(enum ip_type type)
}
}
int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data,
int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct ip_addr *scan_data,
struct scan_result *results, size_t *n_result, size_t n_result_array)
{
if (NULL == table_rt) {
@@ -306,11 +432,11 @@ int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct
struct scan_result scan_results[n_result_array] = {0};
struct ip_data ip;
ip.type = ip_type_transform(data->ip_type);
ip.type = ip_type_transform(scan_data->ip_type);
if (ip.type == IPv4) {
ip.ipv4 = data->ipv4;
ip.ipv4 = scan_data->ipv4;
} else {
memcpy(ip.ipv6, data->ipv6, sizeof(data->ipv6));
memcpy(ip.ipv6, scan_data->ipv6, sizeof(scan_data->ipv6));
}
n_hit_result = ip_matcher_match(table_rt->ip_plus_rt.ip_matcher, &ip, scan_results, n_result_array);
@@ -319,12 +445,7 @@ int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct
}
int index = 0;
/* filter rule_id by port */
for (i = 0; i < n_hit_result; i++) {
struct port_range *port_range = (struct port_range *)scan_results[i].tag;
if (data->port < port_range->min_port || data->port > port_range->max_port) {
continue;
}
results[index++].rule_id = scan_results[i].rule_id;
}
*n_result = index;
@@ -394,7 +515,7 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handl
}
if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
log_error(logger, MODULE_TABLE_RUNTIME, "item_id:%d too many patterns",
log_error(logger, MODULE_TABLE_RUNTIME, "expr item_id:%d too many patterns",
expr_item->item_id);
return NULL;
}
@@ -429,16 +550,128 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handl
return expr_rule;
}
struct maat_item_inner* maat_item_inner_new(int group_id, int item_id, int district_id)
{
struct maat_item_inner *item = ALLOC(struct maat_item_inner, 1);
item->magic_num = ITEM_RULE_MAGIC;
item->item_id = item_id;
item->group_id = group_id;
item->district_id = district_id;
return item;
}
void maat_item_inner_free(struct maat_item_inner *item)
{
assert(item->magic_num == ITEM_RULE_MAGIC);
assert(item->expr_id_cnt == 0 || item->expr_id_cnt == item->expr_id_ub - item->expr_id_lb + 1);
item->magic_num = 0;
free(item);
}
int compile_runtime_update_row(struct compile_runtime *compile_rt, const char *table_name, int compile_id,
struct maat_compile_rule *compile_rule, struct maat_garbage_bin *garbage_bin,
int is_valid, struct log_handle *logger)
{
int ret = -1;
struct maat_hierarchy *maat_hier = compile_rt->hier;
if (0 == is_valid) {
// delete
struct maat_compile_rule *p_compile = (struct maat_compile_rule *)maat_hierarchy_compile_dettach_user_data(compile_rt->hier, compile_id);
if (p_compile != NULL) {
ret = maat_hierarchy_compile_remove(compile_rt->hier, compile_id);
assert(ret == 0);
maat_garbage_bagging(garbage_bin, p_compile, (void (*)(void*))destroy_compile_rule);
}
} else {
// add
ret = maat_hierarchy_compile_add(compile_rt->hier, compile_id, compile_rule->declared_clause_num, compile_rule);
if (ret != 0) {
log_info(logger, MODULE_TABLE_RUNTIME, "duplicate config of compile table %s compile_id %d",
table_name, compile_id);
destroy_compile_rule(compile_rule);
}
}
return ret;
}
int group2compile_runtime_update_row(struct group2compile_runtime *g2c_rt,
int compile_id, struct group2compile_item *g2c_item,
int is_valid, struct log_handle *logger)
{
int ret = -1;
struct compile_runtime *compile_rt = g2c_rt->ref_compile_rt;
struct group2group_runtime *g2g_rt = g2c_rt->ref_g2g_rt;
struct maat_group *group = NULL;
if (0 == is_valid) {
/*TODO: by luis*/
group = maat_group_topology_find_group(g2g_rt->group_topo, g2c_item->group_id);
if (NULL == group) {
log_error(logger, MODULE_TABLE_RUNTIME,
"Remove group %d from compile %d failed, group is not exisited.",
g2c_item->group_id, compile_id);
return ret;
}
ret = maat_hierarchy_remove_group_from_compile(compile_rt->hier, g2c_item->group_id, g2c_item->virtual_table_id,
g2c_item->not_flag, g2c_item->clause_index, g2c_item->compile_id);
if (0 == ret) {
group->ref_by_compile_cnt--;
if (g2c_item->not_flag) {
g2c_rt->not_flag_group--;
}
}
} else {
group = maat_group_topology_find_group(g2g_rt->group_topo, g2c_item->group_id);
if (NULL == group) {
group = maat_group_topology_add_group(g2g_rt->group_topo, g2c_item->group_id);
}
ret = maat_hierarchy_add_group_to_compile(compile_rt->hier, g2c_item->group_id, g2c_item->virtual_table_id,
g2c_item->not_flag, g2c_item->clause_index, g2c_item->compile_id);
if (0 == ret) {
group->ref_by_compile_cnt++;
if (g2c_item->not_flag) {
g2c_rt->not_flag_group++;
}
}
}
return ret;
}
int group2group_runtime_update_row(struct group2group_runtime *g2g_rt, int group_id,
struct group2group_item *g2g_item,
int is_valid, struct log_handle *logger)
{
int ret = -1;
if (0 == is_valid) {
//delete
ret = maat_group_topology_add_group_to_group(g2g_rt->group_topo, group_id, g2g_item->superior_group_id);
} else {
//add
ret = maat_group_topology_remove_group_from_group(g2g_rt->group_topo, group_id, g2g_item->superior_group_id);
}
return ret;
}
void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len,
and_expr_t *expr_rule, int is_valid, struct log_handle *logger)
{
void *data = NULL;
if (is_valid == 0) {
if (0 == is_valid) {
//delete
data = rcu_hash_find(expr_rt->htable, key, key_len);
if (NULL == data) {
log_error(logger, MODULE_TABLE_RUNTIME, "the key:%s not exist, so can't be deleted", key);
log_error(logger, MODULE_TABLE_RUNTIME,
"the key of expr rule not exist, can't be deleted, expr_id:%d", expr_rule->expr_id);
return;
}
rcu_hash_del(expr_rt->htable, key, key_len);
@@ -446,36 +679,28 @@ void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key
//add
data = rcu_hash_find(expr_rt->htable, key, key_len);
if (data != NULL) {
log_error(logger, MODULE_TABLE_RUNTIME, "the key:%s already exist, so can't be added", key);
log_error(logger, MODULE_TABLE_RUNTIME,
"the key of expr rule already exist, can't be added, expr_id:%d", expr_rule->expr_id);
return;
}
and_expr_t *data = ALLOC(and_expr_t, 1);
memcpy(data, expr_rule, sizeof(and_expr_t));
for (size_t i = 0; i < expr_rule->n_patterns; i++) {
data->patterns[i].pat = ALLOC(char, expr_rule->patterns[i].pat_len);
memcpy(data->patterns[i].pat, expr_rule->patterns[i].pat, expr_rule->patterns[i].pat_len);
}
rcu_hash_add(expr_rt->htable, key, key_len, (void *)data);
rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule);
}
}
void ip_plus_runtime_update_row(struct ip_plugin_runtime *ip_plus_rt, struct table_schema *table_schema,
const char *row, char *key, size_t key_len, struct ip_plus_item *ip_plus_item,
int is_valid, struct log_handle *logger)
void ip_plus_runtime_update_row(struct ip_plugin_runtime *ip_plus_rt, char *key, size_t key_len,
struct ip_plus_item *ip_plus_item, int is_valid,
struct log_handle *logger)
{
struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1);
struct ex_data_runtime *ex_data_rt = ip_plus_rt->ex_data_rt;
ex_container->ex_data = NULL;
ex_container->custom_data = ip_plus_item;
struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1);
ctx->custom_data_free = free;
ex_data_runtime_set_ex_container_ctx(ex_data_rt, ctx);
if (0 == is_valid) {
// delete
ex_data_runtime_del_ex_container(ex_data_rt, key, key_len, logger);
} else {
// add
struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1);
ex_container->ex_data = NULL;
ex_container->custom_data = ip_plus_item;
ex_data_runtime_add_ex_container(ip_plus_rt->ex_data_rt, key, key_len, ex_container);
}
}
@@ -516,13 +741,7 @@ void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct
const char *row, char *key, size_t key_len, struct ip_plugin_item *ip_plugin_item,
int is_valid, struct log_handle *logger)
{
struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1);
struct ex_data_runtime *ex_data_rt = ip_plugin_rt->ex_data_rt;
ex_container->ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len);
ex_container->custom_data = ip_plugin_item;
struct ex_container_ctx *ctx = ex_data_runtime_get_ex_container_ctx(ex_data_rt);
ctx->custom_data_free = free;
int set_flag = table_schema_ex_data_schema_flag(table_schema);
if (1 == set_flag) {
@@ -531,6 +750,9 @@ void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct
ex_data_runtime_del_ex_container(ex_data_rt, key, key_len, logger);
} else {
//add
struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1);
ex_container->ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len);
ex_container->custom_data = ip_plugin_item;
ex_data_runtime_add_ex_container(ip_plugin_rt->ex_data_rt, key, key_len, ex_container);
}
} else {
@@ -538,50 +760,226 @@ void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct
}
}
struct maat_compile_rule *compile_item_to_compile_rule(struct compile_item *compile_item, struct table_schema *table_schema, struct log_handle *logger)
{
struct maat_rule_head rule_head;
rule_head.config_id = compile_item->compile_id;
rule_head.service_id = compile_item->service_id;
rule_head.action = compile_item->action;
rule_head.do_blacklist = compile_item->do_blacklist;
rule_head.do_log = compile_item->do_log;
struct maat_compile_rule *p = ALLOC(struct maat_compile_rule, 1);
p->magic_num = COMPILE_RULE_MAGIC;
p->head = rule_head;
p->declared_clause_num = compile_item->clause_num;
p->ex_data = ALLOC(void *, MAX_COMPILE_EX_DATA_NUM);
p->ref_table = table_schema;
p->head.serv_def_len = strlen(compile_item->user_region);
p->service_defined = ALLOC(char, p->head.serv_def_len);
memcpy(p->service_defined, compile_item->user_region, p->head.serv_def_len);
p->evaluation_order = compile_item->evaluation_order;
size_t n_rule_ex_schema = table_schema_compile_rule_ex_data_schema_count(table_schema);
for (size_t i = 0; i < n_rule_ex_schema; i++) {
struct compile_ex_data_schema *ex_schema = table_schema_get_compile_rule_ex_data_schema(table_schema, i);
p->ex_data[i] = rule_ex_data_new(&p->head, p->service_defined, ex_schema);
}
p->is_valid = 1;
p->compile_id = compile_item->compile_id;
pthread_rwlock_init(&p->rwlock, NULL);
return p;
}
int compile_runtime_update(struct compile_runtime *compile_rt, struct compile_item *compile_item,
struct table_schema *table_schema, const char *table_name,
struct maat_garbage_bin *garbage_bin, int is_valid, struct log_handle *logger)
{
int ret = -1;
if (NULL == compile_rt || NULL == compile_item) {
return ret;
}
struct maat_compile_rule *compile_rule = NULL;
if (1 == is_valid) {
compile_rule = compile_item_to_compile_rule(compile_item, table_schema, logger);
if (NULL == compile_rule) {
log_error(logger, MODULE_TABLE_RUNTIME,
"transform compile table:%s item to compile_rule failed, compile_id:%d",
table_name, compile_item->compile_id);
return ret;
}
}
ret = compile_runtime_update_row(compile_rt, table_name, compile_item->compile_id, compile_rule, garbage_bin, is_valid, logger);
return ret;
}
int group2compile_runtime_update(struct group2compile_runtime *g2c_rt, struct group2compile_item *g2c_item,
const char *table_name, int is_valid, struct log_handle *logger)
{
int ret = -1;
if (NULL == g2c_rt || NULL == g2c_item) {
return ret;
}
ret = group2compile_runtime_update_row(g2c_rt, g2c_item->compile_id, g2c_item, is_valid, logger);
return ret;
}
int group2group_runtime_update(struct group2group_runtime *g2g_rt, struct group2group_item *g2g_item,
const char *table_name, int is_valid, struct log_handle *logger)
{
int ret = -1;
if (NULL == g2g_rt || NULL == g2g_item) {
return ret;
}
ret = group2group_runtime_update_row(g2g_rt, g2g_item->group_id, g2g_item, is_valid, logger);
return ret;
}
int expr_runtime_update(struct expr_runtime *expr_rt, struct expr_item *expr_item,
const char *table_name, int is_valid, struct log_handle *logger)
{
if (NULL == expr_rt || NULL == expr_item) {
return -1;
}
/* add item to group */
struct maat_item_inner *u_para = NULL;
u_para = maat_item_inner_new(expr_item->group_id, expr_item->item_id, -1);
int ret = maat_hierarchy_add_item_to_group(scanner->hier, expr_item->group_id, expr_item->item_id, u_para);
if (ret != 0) {
maat_item_inner_free(u_para);
u_para = NULL;
return -1;
}
and_expr_t *expr_rule = NULL;
if (1 == is_valid) {
expr_rule = expr_item_to_expr_rule(expr_item, logger);
if (NULL == expr_rule) {
log_error(logger, MODULE_TABLE_RUNTIME, "transform expr table:%s item to expr_rule failed, item_id:%d",
table_name, expr_item->item_id);
return -1;
}
}
char *key = (char *)&(expr_item->item_id);
expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule, is_valid, logger);
return 0;
}
int ip_plus_runtime_update(struct ip_plugin_runtime *ip_plus_rt, struct ip_plus_item *ip_plus_item,
const char *table_name, int is_valid, struct log_handle *logger)
{
if (NULL == ip_plus_rt || NULL == ip_plus_item) {
return -1;
}
struct ip_plus_item *item = NULL;
if (1 == is_valid) {
item = ALLOC(struct ip_plus_item, 1);
memcpy(item, ip_plus_item, sizeof(struct ip_plus_item));
}
char *key = (char *)&(ip_plus_item->item_id);
ip_plus_runtime_update_row(ip_plus_rt, key, sizeof(int), ip_plus_item, is_valid, logger);
return 0;
}
int plugin_runtime_update(struct plugin_runtime *plugin_rt, struct plugin_item *plugin_item,
struct table_schema *table_schema, const char *row, int is_valid,
struct log_handle *logger)
{
if (NULL == plugin_rt || NULL == plugin_item) {
return -1;
}
char *key = plugin_item->key;
size_t key_len = plugin_item->key_len;
plugin_runtime_update_row(plugin_rt, table_schema, row, key, key_len, is_valid, logger);
return 0;
}
int ip_plugin_runtime_update(struct ip_plugin_runtime *ip_plugin_rt,
struct ip_plugin_item *ip_plugin_item,
struct table_schema *table_schema, const char *row,
int is_valid, struct log_handle *logger)
{
if (NULL == ip_plugin_rt || NULL == ip_plugin_item) {
return -1;
}
struct ip_plugin_item *item = NULL;
if (1 == is_valid) {
item = ALLOC(struct ip_plugin_item, 1);
memcpy(item, ip_plugin_item, sizeof(struct ip_plugin_item));
}
char *key = (char *)&(ip_plugin_item->item_id);
ip_plugin_runtime_update_row(ip_plugin_rt, table_schema, row, key, sizeof(int),
item, is_valid, logger);
return 0;
}
void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema,
const char *row, struct table_item *table_item, struct log_handle *logger)
{
int ret = -1;
int is_valid = -1;
char *key = NULL;
size_t key_len = 0;
and_expr_t *expr_rule = NULL;
struct ip_plus_item *ip_plus_item = NULL;
struct ip_plugin_item *ip_plugin_item = NULL;
const char *table_name = table_schema_get_updating_name(table_schema);
int table_id = table_schema_get_table_id(table_schema);
switch (table_rt->table_type) {
case TABLE_TYPE_COMPILE:
is_valid = table_item->compile_item.is_valid;
ret = compile_runtime_update(&(table_rt->compile_rt), &(table_item->compile_item), table_schema, table_name, table_rt->ref_garbage_bin, is_valid, logger);
break;
case TABLE_TYPE_GROUP2COMPILE:
is_valid = table_item->g2c_item.is_valid;
ret = group2compile_runtime_update(&(table_rt->g2c_rt), &(table_item->g2c_item), table_name, is_valid, logger);
break;
case TABLE_TYPE_GROUP2GROUP:
is_valid = table_item->g2g_item.is_valid;
ret = group2group_runtime_update(&(table_rt->g2g_rt), &(table_item->g2g_item), table_name, is_valid, logger);
break;
case TABLE_TYPE_EXPR:
is_valid = table_item->expr_item.is_valid;
expr_rule = expr_item_to_expr_rule(&table_item->expr_item, logger);
key = (char *)&(table_item->expr_item.item_id);
expr_runtime_update_row(&(table_rt->expr_rt), key, sizeof(int), expr_rule, is_valid, logger);
expr_rule_free(expr_rule);
ret = expr_runtime_update(&(table_rt->expr_rt), &(table_item->expr_item), table_name, is_valid, logger);
break;
case TABLE_TYPE_IP_PLUS:
is_valid = table_item->ip_plus_item.is_valid;
ip_plus_item = ALLOC(struct ip_plus_item, 1);
memcpy(ip_plus_item, &(table_item->ip_plus_item), sizeof(table_item->ip_plus_item));
key = (char *)&(table_item->ip_plus_item.item_id);
ip_plus_runtime_update_row(&(table_rt->ip_plus_rt), table_schema, row, key, sizeof(int),
ip_plus_item, is_valid, logger);
ret = ip_plus_runtime_update(&(table_rt->ip_plus_rt), &(table_item->ip_plus_item), table_name, is_valid, logger);
break;
case TABLE_TYPE_PLUGIN:
is_valid = table_item->plugin_item.is_valid;
key = table_item->plugin_item.key;
key_len = table_item->plugin_item.key_len;
plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, key, key_len, is_valid, logger);
ret = plugin_runtime_update(&(table_rt->plugin_rt), &(table_item->plugin_item), table_schema, row, is_valid, logger);
break;
case TABLE_TYPE_IP_PLUGIN:
is_valid = table_item->ip_plugin_item.is_valid;
ip_plugin_item = ALLOC(struct ip_plugin_item, 1);
memcpy(ip_plugin_item, &(table_item->ip_plugin_item), sizeof(table_item->ip_plugin_item));
key = (char *)&(table_item->ip_plugin_item.item_id);
ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, row, key, sizeof(int),
ip_plugin_item, is_valid, logger);
ret = ip_plugin_runtime_update(&(table_rt->ip_plugin_rt), &(table_item->ip_plugin_item), table_schema, row, is_valid, logger);
break;
default:
break;
}
if (ret < 0) {
return;
}
if (is_valid == 0) {
table_rt->rule_num--;
} else {
@@ -589,6 +987,22 @@ void table_runtime_update(struct table_runtime *table_rt, struct table_schema *t
}
}
int compile_runtime_commit(struct table_runtime *table_rt, long long version, struct log_handle *logger)
{
int ret = -1;
ret = maat_hierarchy_rebuild(table_rt->compile_rt.hier);
if (ret != 0) {
log_error(logger, MODULE_TABLE_RUNTIME, "version %lld hierarchy rebuild failed.",
version);
return -1;
} else {
log_info(logger, MODULE_TABLE_RUNTIME, "version %lld hierarchy rebuild success.",
version);
return 0;
}
}
int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread,
struct log_handle *logger)
{
@@ -763,10 +1177,15 @@ int table_runtime_updating_flag(struct table_runtime *table_rt)
return updating_flag;
}
void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread,
void table_runtime_commit(struct table_runtime *table_rt, long long version, size_t nr_worker_thread,
struct log_handle *logger)
{
switch (table_rt->table_type) {
case TABLE_TYPE_COMPILE:
case TABLE_TYPE_GROUP2COMPILE:
case TABLE_TYPE_GROUP2GROUP:
compile_runtime_commit(table_rt, version, logger);
break;
case TABLE_TYPE_EXPR:
expr_runtime_commit(table_rt, nr_worker_thread, logger);
break;
@@ -840,7 +1259,7 @@ struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table
}
void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema,
int nr_worker_thread, struct log_handle *logger)
int nr_worker_thread, long long version, struct log_handle *logger)
{
struct ex_data_schema *ex_data_schema = table_schema_get_ex_data_schema(table_schema);
struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt);
@@ -867,5 +1286,5 @@ void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct
}
ex_data_runtime_clear_row_cache(ex_data_rt);
table_runtime_commit(table_rt, nr_worker_thread, logger);
table_runtime_commit(table_rt, version, nr_worker_thread, logger);
}

736
src/maat_table_schema.cpp
View File

@@ -25,12 +25,9 @@
#define MODULE_TABLE_SCHEMA module_name_str("maat.table_schema")
#define MAX_TABLE_LINE_SIZE (1024 * 16)
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_CONJUNCTION_TABLE_NUM 8
#define MAX_COMPILE_EX_DATA_NUM 2
enum user_region_encode {
USER_REGION_ENCODE_NONE=0,
USER_REGION_ENCODE_ESCAPE,
@@ -38,11 +35,37 @@ enum user_region_encode {
};
struct compile_table_schema {
int compile_id_column;
int service_id_column;
int action_column;
int do_blacklist_column;
int do_log_column;
int tags_column;
int user_region_column;
int is_valid_column;
int clause_num_column;
int evaluation_order_column;
enum user_region_encode user_region_encoding;
size_t n_ex_schema;
struct compile_ex_data_schema ex_schema[MAX_COMPILE_EX_DATA_NUM];
};
struct group2compile_table_schema {
int group_id_column;
int compile_id_column;
int is_valid_column;
int not_flag_column;
int virtual_table_name_column;
int clause_index_column;
char associated_compile_table_id;
};
struct group2group_table_schema {
int group_id_column;
int superior_group_id_column;
int is_valid_column;
};
struct expr_table_schema {
int item_id_column;
int group_id_column;
@@ -117,6 +140,8 @@ struct table_schema {
enum table_type table_type;
union {
struct compile_table_schema compile;
struct group2compile_table_schema group2compile;
struct group2group_table_schema group2group;
struct expr_table_schema expr;
struct ip_plus_table_schema ip_plus;
struct plugin_table_schema plugin;
@@ -124,12 +149,16 @@ struct table_schema {
struct virtual_table_schema virtual_table;
struct composition_table_schema composition;
};
uint64_t update_err_cnt;
uint64_t unmatched_tag_cnt;
};
struct table_schema_manager {
struct table_schema *schema_table[MAX_TABLE_NUM];
size_t n_schema_table;
size_t n_active_plugin_table;
char default_compile_table[NAME_MAX];
struct maat_kv_store *tablename2id_map;
struct log_handle *logger;
};
@@ -147,6 +176,239 @@ void table_schema_free(struct table_schema *ptable)
FREE(ptable);
}
int read_compile_table_schema(cJSON *root, struct table_schema *ptable,
struct maat_kv_store* reserved_word_map,
struct log_handle *logger)
{
int read_cnt = 0;
cJSON *json = NULL;
json = cJSON_GetObjectItem(root, "table_id");
if (json != NULL && json->type == cJSON_Number) {
ptable->table_id = json->valueint;
read_cnt++;
}
json = cJSON_GetObjectItem(root, "table_name");
if (json != NULL && json->type == cJSON_String) {
if (strlen(json->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE_SCHEMA,
"compile table name %s length too long", json->valuestring);
return -1;
}
memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring));
read_cnt++;
}
json = cJSON_GetObjectItem(root, "custom");
if (json == NULL || json->type != cJSON_Object) {
log_error(logger, MODULE_TABLE_SCHEMA, "table %s has no custom column",
ptable->table_name[0]);
return -1;
}
cJSON *item = cJSON_GetObjectItem(json, "compile_id");
if (item != NULL && item->type == cJSON_Number) {
ptable->compile.compile_id_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "service_id");
if (item != NULL && item->type == cJSON_Number) {
ptable->compile.service_id_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "action");
if (item != NULL && item->type == cJSON_Number) {
ptable->compile.action_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "do_blacklist");
if (item != NULL && item->type == cJSON_Number) {
ptable->compile.do_blacklist_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "do_log");
if (item != NULL && item->type == cJSON_Number) {
ptable->compile.do_log_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "tags");
if (item != NULL && item->type == cJSON_Number) {
ptable->compile.tags_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "user_region");
if (item != NULL && item->type == cJSON_Number) {
ptable->compile.user_region_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "is_valid");
if (item != NULL && item->type == cJSON_Number) {
ptable->compile.is_valid_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "clause_num");
if (item != NULL && item->type == cJSON_Number) {
ptable->compile.clause_num_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "evaluation_order");
if (item != NULL && item->type == cJSON_Number) {
ptable->compile.evaluation_order_column = item->valueint;
read_cnt++;
}
if (read_cnt < 12) {
return -1;
}
return 0;
}
int read_group2compile_table_schema(cJSON *root, struct table_schema *ptable,
struct maat_kv_store* reserved_word_map,
struct log_handle *logger)
{
int read_cnt = 0;
cJSON *json = NULL;
json = cJSON_GetObjectItem(root, "table_id");
if (json != NULL && json->type == cJSON_Number) {
ptable->table_id = json->valueint;
read_cnt++;
}
json = cJSON_GetObjectItem(root, "table_name");
if (json != NULL && json->type == cJSON_String) {
if (strlen(json->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE_SCHEMA,
"group2compile table name %s length too long", json->valuestring);
return -1;
}
memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring));
read_cnt++;
}
json = cJSON_GetObjectItem(root, "associated_compile_table_id");
if (json != NULL && json->type == cJSON_Number) {
ptable->group2compile.associated_compile_table_id = json->valueint;
read_cnt++;
}
json = cJSON_GetObjectItem(root, "custom");
if (json == NULL || json->type != cJSON_Object) {
log_error(logger, MODULE_TABLE_SCHEMA, "table %s has no custom column",
ptable->table_name[0]);
return -1;
}
cJSON *item = cJSON_GetObjectItem(json, "group_id");
if (item != NULL && item->type == cJSON_Number) {
ptable->group2compile.group_id_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "compile_id");
if (item != NULL && item->type == cJSON_Number) {
ptable->group2compile.compile_id_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "is_valid");
if (item != NULL && item->type == cJSON_Number) {
ptable->group2compile.is_valid_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "not_flag");
if (item != NULL && item->type == cJSON_Number) {
ptable->group2compile.not_flag_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "virtual_table_name");
if (item != NULL && item->type == cJSON_Number) {
ptable->group2compile.virtual_table_name_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "clause_index");
if (item != NULL && item->type == cJSON_Number) {
ptable->group2compile.clause_index_column = item->valueint;
read_cnt++;
}
if (read_cnt < 8) {
return -1;
}
return 0;
}
int read_group2group_table_schema(cJSON *root, struct table_schema *ptable,
struct maat_kv_store* reserved_word_map,
struct log_handle *logger)
{
int read_cnt = 0;
cJSON *json = NULL;
json = cJSON_GetObjectItem(root, "table_id");
if (json != NULL && json->type == cJSON_Number) {
ptable->table_id = json->valueint;
read_cnt++;
}
json = cJSON_GetObjectItem(root, "table_name");
if (json != NULL && json->type == cJSON_String) {
if (strlen(json->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE_SCHEMA,
"group2group table name %s length too long", json->valuestring);
return -1;
}
memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring));
read_cnt++;
}
json = cJSON_GetObjectItem(root, "custom");
if (json == NULL || json->type != cJSON_Object) {
log_error(logger, MODULE_TABLE_SCHEMA, "table %s has no custom column",
ptable->table_name[0]);
return -1;
}
cJSON *item = cJSON_GetObjectItem(json, "group_id");
if (item != NULL && item->type == cJSON_Number) {
ptable->group2group.group_id_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "superior_group_id");
if (item != NULL && item->type == cJSON_Number) {
ptable->group2group.superior_group_id_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "is_valid");
if (item != NULL && item->type == cJSON_Number) {
ptable->group2group.is_valid_column = item->valueint;
read_cnt++;
}
if (read_cnt < 5) {
return -1;
}
return 0;
}
int read_expr_table_schema(cJSON *root, struct table_schema *ptable,
struct maat_kv_store* reserved_word_map,
struct log_handle *logger)
@@ -706,47 +968,42 @@ int table_schema_populate(cJSON *json, struct table_schema **table_array,
}
switch (ptable->table_type) {
case TABLE_TYPE_COMPILE:
ret = read_compile_table_schema(json, ptable, reserved_word_map, logger);
break;
case TABLE_TYPE_GROUP2COMPILE:
ret = read_group2compile_table_schema(json, ptable, reserved_word_map, logger);
break;
case TABLE_TYPE_GROUP2GROUP:
ret = read_group2group_table_schema(json, ptable, reserved_word_map, logger);
break;
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
ret = read_expr_table_schema(json, ptable, reserved_word_map, logger);
if (ret < 0) {
return -1;
}
break;
case TABLE_TYPE_IP_PLUS:
ret = read_ip_plus_table_schema(json, ptable, logger);
if (ret < 0) {
return -1;
}
break;
case TABLE_TYPE_PLUGIN:
ret = read_plugin_table_schema(json, ptable, logger);
if (ret < 0) {
return -1;
}
break;
case TABLE_TYPE_IP_PLUGIN:
ret = read_ip_plugin_table_schema(json, ptable, logger);
if (ret < 0) {
return -1;
}
break;
case TABLE_TYPE_VIRTUAL:
ret = read_virtual_table_schema(json, table_array, ptable, tablename2id_map, logger);
if (ret < 0) {
return -1;
}
break;
case TABLE_TYPE_COMPOSITION:
ret = read_composition_table_schema(json, ptable, tablename2id_map, logger);
if (ret < 0) {
return -1;
}
break;
default:
break;
}
if (ret < 0) {
return -1;
}
return 0;
}
@@ -783,6 +1040,9 @@ table_schema_manager_create(const char *table_info_path, struct log_handle *logg
struct maat_kv_store* reserved_word_map = maat_kv_store_new();
/* register table type reserved word */
maat_kv_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE);
maat_kv_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE);
maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP);
maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR);
maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS);
maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS);
@@ -1092,6 +1352,15 @@ int table_schema_get_table_id(struct table_schema *table_schema)
return table_schema->table_id;
}
int table_schema_get_associated_table_id(struct table_schema *table_schema)
{
if (NULL == table_schema || table_schema->table_type != TABLE_TYPE_GROUP2COMPILE) {
return -1;
}
return table_schema->group2compile.associated_compile_table_id;
}
enum scan_type table_schema_get_scan_type(struct table_schema *table_schema)
{
enum scan_type ret = SCAN_TYPE_INVALID;
@@ -1146,6 +1415,15 @@ int table_schema_get_valid_flag_column(struct table_schema *table_schema)
}
switch (table_schema->table_type) {
case TABLE_TYPE_COMPILE:
valid_flag_column = table_schema->compile.is_valid_column;
break;
case TABLE_TYPE_GROUP2COMPILE:
valid_flag_column = table_schema->group2compile.is_valid_column;
break;
case TABLE_TYPE_GROUP2GROUP:
valid_flag_column = table_schema->group2group.is_valid_column;
break;
case TABLE_TYPE_EXPR:
valid_flag_column = table_schema->expr.is_valid_column;
break;
@@ -1155,13 +1433,6 @@ int table_schema_get_valid_flag_column(struct table_schema *table_schema)
case TABLE_TYPE_IP_PLUGIN:
valid_flag_column = table_schema->ip_plugin.is_valid_column;
break;
/*
case TABLE_TYPE_FQDN_PLUGIN:
valid_flag_column = table_schema->fqdn_plugin.valid_flag_column;
break;
case TABLE_TYPE_BOOL_PLUGIN:
valid_flag_column = table_schema->bool_plugin.valid_flag_column;
break;*/
default:
valid_flag_column = -1;
break;
@@ -1185,6 +1456,326 @@ void table_schema_set_updating_name(struct table_schema *table_schema, const cha
assert(i <= table_schema->conj_cnt);
}
const char *table_schema_get_updating_name(struct table_schema *table_schema)
{
return table_schema->table_name[table_schema->updating_name];
}
static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, int n_accept_tag)
{
cJSON *tab_name_obj = cJSON_GetObjectItem(tag_obj, "tag");
if (NULL == tab_name_obj || tab_name_obj->type != cJSON_String) {
goto error;
}
const char *tag_name = tab_name_obj->valuestring;
cJSON *tag_vals_array = cJSON_GetObjectItem(tag_obj, "value");
if (NULL == tag_vals_array || tag_vals_array->type != cJSON_Array) {
goto error;
}
int name_matched = 0;
int n_val = cJSON_GetArraySize(tag_vals_array);
for (int i = 0; i < n_accept_tag; i++) {
if (0 != strcmp(accept_tags[i].tag_name, tag_name)) {
continue;
}
name_matched++;
for (int j = 0; j < n_val; j++) {
cJSON *tag_val_obj = cJSON_GetArrayItem(tag_vals_array, j);
if (NULL == tag_val_obj || tag_val_obj->type != cJSON_String) {
goto error;
}
const char *tag_val = tag_val_obj->valuestring;
// compare a/b/c with a/b/c/d is a miss.
if (strlen(accept_tags[i].tag_val) < strlen(tag_val)) {
continue;
}
// compare a1a2/b1/c1 with a1a2/b/ is a miss.
//make sure the overlap is ended with a '/'
size_t compare_len =
if (0 == strncmp(accept_tags[i].tag_val, tag_val, strlen(tag_val)) &&
(strlen(accept_tags[i].tag_val) == strlen(tag_val) || accept_tags[i].tag_val[strlen(tag_val)] == '/')) {
return 1;
}
}
}
//no matched name is considered as a
if (name_matched > 0) {
return 0;
} else {
return 1;
}
error:
return -1;
}
//@param tag_set likes [{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}]
static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_tags, int n_accept_tag)
{
int matched = 0;
int n_tag = cJSON_GetArraySize(tag_set);
for (int i = 0; i < n_tag; i++) {
cJSON *tag_obj = cJSON_GetArrayItem(tag_set, i);
if (NULL == tag_obj || tag_obj->type != cJSON_Object) {
goto error;
}
int ret = compare_each_tag(tag_obj, accept_tags, n_accept_tag);
if (ret < 0) {
return -1;
}
if(1 == ret) {
matched++;
}
}
if (matched == n_tag) {
return 1;
} else {
return 0;
}
error:
return -1;
}
//@param value {"tag_sets":[[{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}],[{"tag":"location","value":["北京"]},{"tag":"isp","value":["联通"]}]]}
//@return 1 on match, 0 on not match, -1 on error.
static int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, int n_tag)
{
int ret = -1;
int n_set = 0;
cJSON *tag_set = NULL;
cJSON *tag_set_array = NULL;
cJSON *root = cJSON_Parse(value);
if (NULL == root) {
goto next;
}
tag_set_array = cJSON_GetObjectItem(root, "tag_sets");
if (NULL == tag_set_array || tag_set_array->type != cJSON_Array) {
goto next;
}
n_set = cJSON_GetArraySize(tag_set_array);
for (int i = 0; i < n_set; i++) {
tag_set = cJSON_GetArrayItem(tag_set_array, i);
if (NULL == tag_set || tag_set->type != cJSON_Array) {
goto next;
}
ret = compare_each_tag_set(tag_set, accept_tags, n_tag);
//match or error occurs.
if (ret != 0) {
break;
}
}
error:
cJSON_Delete(root);
return ret;
}
int populate_compile_table_item(const char *line, struct table_schema *table_schema, struct table_item *table_item,
const char *table_name, const struct rule_tag *accept_tags, int n_accept_tag,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
struct compile_table_schema *compile_schema = &(table_schema->compile);
struct compile_item *compile_item = &(table_item->compile_item);
int ret = get_column_pos(line, compile_schema->compile_id_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
compile_item->compile_id = atoi(line + column_offset);
ret = get_column_pos(line, compile_schema->service_id_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
compile_item->service_id = atoi(line + column_offset);
ret = get_column_pos(line, compile_schema->action_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
compile_item->action = atoi(line + column_offset);
ret = get_column_pos(line, compile_schema->do_blacklist_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
compile_item->do_blacklist = atoi(line + column_offset);
ret = get_column_pos(line, compile_schema->do_log_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
compile_item->do_log = atoi(line + column_offset);
ret = get_column_pos(line, compile_schema->tags_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
char tag_str[MAX_TABLE_LINE_SIZE] = {0};
memcpy(tag_str, (line + column_offset), column_len);
if (n_accept_tag > 0 && strlen(tag_str) > 2) {
str_unescape(tag_str);
ret = compare_accept_tag(tag_str, accept_tags, n_accept_tag);
if (ret < 0) {
log_error(logger, MODULE_TABLE_SCHEMA,
"update error, invalid tag format for compile table[%s]:compile_id[%d]",
table_name, compile_item->compile_id);
return -1;
}
if (0 == ret) {
table_schema->unmatched_tag_cnt++;
return -1;
}
}
ret = get_column_pos(line, compile_schema->user_region_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
if (column_len > MAX_TABLE_LINE_SIZE) {
log_error(logger, MODULE_TABLE_SCHEMA,
"update error: compile table[%s]:compile_id[%d] user_region length too long",
table_name, compile_item->compile_id);
return -1;
}
memcpy(compile_item->user_region, (line + column_offset), column_len);
switch (compile_schema->user_region_encoding) {
case USER_REGION_ENCODE_ESCAPE:
str_unescape(compile_item->user_region);
break;
default:
break;
}
ret = get_column_pos(line, compile_schema->is_valid_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
compile_item->is_valid = atoi(line + column_offset);
ret = get_column_pos(line, compile_schema->clause_num_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
compile_item->clause_num = atoi(line + column_offset);
ret = get_column_pos(line, compile_schema->evaluation_order_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
compile_item->evaluation_order = atoi(line + column_offset);
return 0;
}
int populate_group2compile_table_item(const char *line, struct group2compile_table_schema *group2compile_schema,
struct table_item *table_item, const char *table_name,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
struct group2compile_item *group2compile_item = &(table_item->group2compile_item);
int ret = get_column_pos(line, group2compile_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
group2compile_item->group_id = atoi(line + column_offset);
ret = get_column_pos(line, group2compile_schema->compile_id_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
group2compile_item->compile_id = atoi(line + column_offset);
ret = get_column_pos(line, group2compile_schema->is_valid_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
group2compile_item->is_valid = atoi(line + column_offset);
ret = get_column_pos(line, group2compile_schema->not_flag_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
group2compile_item->not_flag = atoi(line + column_offset);
ret = get_column_pos(line, group2compile_schema->virtual_table_name_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
if (column_len > NAME_MAX) {
log_error(logger, MODULE_TABLE_SCHEMA,
"update error: group2compile table[%s]:group_id[%d] virtual table name length too long",
table_name, group2compile_item->group_id);
return -1;
}
memcpy(group2compile_item->virtual_table_name, (line + column_offset), column_len);
ret = get_column_pos(line, group2compile_schema->clause_index_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
group2compile_item->clause_index = atoi(line + column_offset);
return 0;
}
int populate_group2group_table_item(const char *line, struct group2group_table_schema *group2group_schema,
struct table_item *table_item, const char *table_name,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
struct group2group_item *group2group_item = &(table_item->group2group_item);
int ret = get_column_pos(line, group2group_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
group2group_item->group_id = atoi(line + column_offset);
ret = get_column_pos(line, group2group_schema->superior_group_id_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
group2group_item->superior_group_id = atoi(line + column_offset);
ret = get_column_pos(line, group2group_schema->is_valid_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
group2group_item->is_valid = atoi(line + column_offset);
return 0;
}
int populate_expr_table_item(const char *line, struct expr_table_schema *expr_schema,
struct table_item *table_item, const char *table_name,
struct log_handle *logger)
@@ -1540,68 +2131,87 @@ int populate_ip_plugin_table_item(const char *line, struct ip_plugin_table_schem
return 0;
}
struct table_item *
table_schema_line_to_item(const char *line, struct table_schema *table_schema, struct log_handle *logger)
void table_item_free(struct table_item *table_item)
{
if (NULL == table_item) {
return;
}
switch (table_item->table_type) {
case TABLE_TYPE_COMPILE:
case TABLE_TYPE_GROUP2COMPILE:
case TABLE_TYPE_GROUP2GROUP:
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
case TABLE_TYPE_IP_PLUS:
case TABLE_TYPE_PLUGIN:
break;
case TABLE_TYPE_IP_PLUGIN:
/* TODO: free ex_data */
break;
default:
break;
}
FREE(table_item);
}
struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema,
struct rule_tag *accept_tags, int n_accept_tag, struct log_handle *logger)
{
if (NULL == line || NULL == table_schema) {
return NULL;
}
int ret = -1;
const char *table_name = table_schema_get_updating_name(table_schema);
struct table_item *table_item = ALLOC(struct table_item, 1);
switch (table_schema->table_type) {
case TABLE_TYPE_COMPILE:
table_item->table_type = table_schema->table_type;
ret = populate_compile_table_item(line, table_schema, table_item, table_name, accept_tags, n_accept_tag, logger);
break;
case TABLE_TYPE_GROUP2COMPILE:
table_item->table_type = table_schema->table_type;
ret = populate_group2compile_table_item(line, &table_schema->group2compile, table_item, table_name, logger);
break;
case TABLE_TYPE_GROUP2GROUP:
table_item->table_type = table_schema->table_type;
ret = populate_group2group_table_item(line, &table_schema->group2group, table_item, table_name, logger);
break;
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
table_item->table_type = table_schema->table_type;
ret = populate_expr_table_item(line, &table_schema->expr, table_item,
table_schema->table_name[table_schema->updating_name], logger);
if (ret < 0) {
log_error(logger, MODULE_TABLE_SCHEMA,
"abandon config: invalid format of expr table %s:%s",
table_schema->table_name[table_schema->updating_name], line);
goto error;
}
ret = populate_expr_table_item(line, &table_schema->expr, table_item, table_name, logger);
break;
case TABLE_TYPE_IP_PLUS:
table_item->table_type = table_schema->table_type;
ret = populate_ip_plus_table_item(line, &table_schema->ip_plus, table_item,
table_schema->table_name[table_schema->updating_name], logger);
if (ret < 0) {
log_error(logger, MODULE_TABLE_SCHEMA,
"abandon config: invalid format of ip_plus table %s:%s",
table_schema->table_name[table_schema->updating_name], line);
goto error;
}
ret = populate_ip_plus_table_item(line, &table_schema->ip_plus, table_item, table_name, logger);
break;
case TABLE_TYPE_PLUGIN:
table_item->table_type = TABLE_TYPE_PLUGIN;
ret = populate_plugin_table_item(line, &table_schema->plugin, table_item,
table_schema->table_name[table_schema->updating_name], logger);
if (ret < 0) {
log_error(logger, MODULE_TABLE_SCHEMA,
"abandon config: invalid format of plugin table %s:%s",
table_schema->table_name[table_schema->updating_name], line);
goto error;
}
ret = populate_plugin_table_item(line, &table_schema->plugin, table_item, table_name, logger);
break;
case TABLE_TYPE_IP_PLUGIN:
table_item->table_type = TABLE_TYPE_IP_PLUGIN;
ret = populate_ip_plugin_table_item(line, &table_schema->ip_plugin, table_item,
table_schema->table_name[table_schema->updating_name], logger);
if (ret < 0) {
log_error(logger, MODULE_TABLE_SCHEMA,
"abandon config: invalid format of ip_plugin table %s:%s",
table_schema->table_name[table_schema->updating_name], line);
goto error;
}
ret = populate_ip_plugin_table_item(line, &table_schema->ip_plugin, table_item, table_name, logger);
break;
default:
break;
}
if (ret < 0) {
log_error(logger, MODULE_TABLE_SCHEMA,
"abandon config: invalid format of table %s:%s, transform line to item failed. ",
table_name, line);
table_schema->update_err_cnt++;
goto error;
}
return table_item;
error:
FREE(table_item);
table_item_free(table_item);
return NULL;
}

4
src/rcu_hash.cpp
View File

@@ -132,6 +132,10 @@ void rcu_hash_free(struct rcu_hash_table *htable)
}
rcu_hash_garbage_queue_free(&(htable->garbage_q));
if (htable->user_ctx != NULL) {
FREE(htable->user_ctx);
}
pthread_mutex_destroy(&htable->update_mutex);
FREE(htable);

9
test/maat_framework_gtest.cpp
View File

@@ -365,12 +365,6 @@ int main(int argc, char ** argv)
{
int ret=0;
::testing::InitGoogleTest(&argc, argv);
/*
struct maat_options *opts = maat_options_new();
maat_options_set_json_file(opts, json_path);
g_maat_instance = maat_new(opts, table_info_path);
EXPECT_NE(g_maat_instance, nullptr);*/
char json_iris_path[128] = {0};
char redis_ip[64] = "127.0.0.1";
@@ -426,8 +420,7 @@ int main(int argc, char ** argv)
redisFree(c);
struct maat_options *opts = maat_options_new();
maat_options_set_redis_ip(opts, redis_ip);
maat_options_set_redis_port(opts, redis_port);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, logger);
g_maat_instance = maat_new(opts, table_info_path);

6
test/maat_input_mode_gtest.cpp
View File

@@ -67,8 +67,7 @@ TEST(iris_mode, maat_scan_string) {
snprintf(tmp_iris_inc_idx_path, sizeof(tmp_iris_inc_idx_path), "%s/index", json_iris_path);
struct maat_options *opts = maat_options_new();
maat_options_set_iris_full_index_dir(opts, tmp_iris_full_idx_path);
maat_options_set_iris_inc_index_dir(opts, tmp_iris_inc_idx_path);
maat_options_set_iris(opts, tmp_iris_full_idx_path, tmp_iris_inc_idx_path);
struct maat *maat_instance = maat_new(opts, table_info_path);
EXPECT_NE(maat_instance, nullptr);
@@ -187,8 +186,7 @@ TEST(redis_mode, maat_scan_string) {
redisFree(c);
struct maat_options *opts = maat_options_new();
maat_options_set_redis_ip(opts, redis_ip);
maat_options_set_redis_port(opts, redis_port);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, logger);
struct maat *maat_instance = maat_new(opts, table_info_path);

72
test/table_info.conf
View File

@@ -90,5 +90,77 @@
"dest": "COMPOSITION_IP_DEST",
"session": "COMPOSITION_IP_SESSION"
}
},
{
"table_id":9,
"table_name":"COMPILE_1",
"table_type":"compile",
"custom": {
"compile_id":1,
"service_id":2,
"action":3,
"do_blacklist":4,
"do_log":5,
"tags":6,
"user_region":7,
"is_valid":8,
"clause_num":9,
"evaluation_order":10
}
},
{
"table_id":10,
"table_name":"GROUP2COMPILE_1",
"table_type":"group2compile",
"associated_compile_table_id":9,
"custom": {
"group_id":1,
"compile_id":2,
"is_valid":3,
"not_flag":4,
"virtual_table_name":5,
"clause_index":6
}
},
{
"table_id":11,
"table_name":"COMPILE_2",
"table_type":"compile",
"custom": {
"compile_id":1,
"service_id":2,
"action":3,
"do_blacklist":4,
"do_log":5,
"tags":6,
"user_region":7,
"is_valid":8,
"clause_num":9,
"evaluation_order":10
}
},
{
"table_id":12,
"table_name":"GROUP2COMPILE_2",
"table_type":"group2compile",
"associated_compile_table_id":11,
"custom": {
"group_id":1,
"compile_id":2,
"is_valid":3,
"not_flag":4,
"virtual_table_name":5,
"clause_index":6
}
},
{
"table_id":13,
"table_name":"GROUP2GROUP",
"table_type":"group2group",
"custom": {
"group_id":1,
"superior_group_id":2,
"is_valid":3
}
}
]