gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add dynamic config unit-test and hierarchy unfinished

Browse Source
This commit is contained in:
liuwentan
2022-12-14 15:28:21 +08:00
parent 95b2123b5f
commit 9778267b48
26 changed files with 2411 additions and 692 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
include/maat/maat.h
View File

@@ -27,15 +27,19 @@ extern "C"
/* maat instance handle */ /* maat instance handle */
struct maat; struct maat;
struct maat_rule {
int rule_id;
};
enum ip_type { enum ip_type {
IP_TYPE_V4, IP_TYPE_V4,
IP_TYPE_V6 IP_TYPE_V6
}; };
enum maat_scan_opt
{
MAAT_SET_SCAN_DISTRICT = 1, //VALUE is a const char*, SIZE= strlen(string). DEFAULT: no default.
MAAT_SET_SCAN_LAST_REGION, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination.
MAAT_GET_SCAN_HIT_PATH //VALUE is struct Maat_hit_path_t*, an array of struct Maat_hit_path_t, SIZE= sizeof(struct Maat_hit_path_t)*N,
//Maat_get_scan_status returns actual got number.
};
/* network order */ /* network order */
struct ipv4_4tuple { struct ipv4_4tuple {
uint32_t sip; uint32_t sip;
@@ -70,6 +74,12 @@ typedef void maat_plugin_ex_new_func_t(int table_id, const char *key, const char
typedef void maat_plugin_ex_free_func_t(int table_id, void **ad, long argl, void *argp); typedef void maat_plugin_ex_free_func_t(int table_id, void **ad, long argl, void *argp);
typedef void maat_plugin_ex_dup_func_t(int table_id, void **to, void **from, long argl, void *argp); typedef void maat_plugin_ex_dup_func_t(int table_id, void **to, void **from, long argl, void *argp);
typedef void maat_rule_ex_new_func_t(int idx, const struct maat_rule *rule, const char *srv_def_large,
void *ex_data, long argl, void *argp);
typedef void maat_rule_ex_free_func_t(int idx, const struct maat_rule *rule, const char *srv_def_large,
void *ex_data, long argl, void *argp);
typedef void maat_rule_ex_dup_func_t(int idx, void *to, void *from, long argl, void *argp);
/* maat_instance options API */ /* maat_instance options API */
struct maat_options; struct maat_options;
struct maat_options* maat_options_new(void); struct maat_options* maat_options_new(void);
@@ -112,25 +122,30 @@ void *maat_plugin_table_dup_ex_data(struct maat *instance, int table_id,
struct maat_state; struct maat_state;
int maat_scan_integer(struct maat *instance, int table_id, int thread_id, int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
unsigned int intval, int results[], size_t *n_result, unsigned int intval, int results[], size_t *n_result,
struct maat_state *state); struct maat_state **state);
int maat_scan_ip(struct maat *instance, int table_id, int thread_id, int maat_scan_ip(struct maat *instance, int table_id, int thread_id,
struct addr_4tuple *addr, int results[], size_t *n_result, struct addr_4tuple *addr, int results[], size_t *n_result,
struct maat_state *state); struct maat_state **state);
int maat_scan_string(struct maat *instance, int table_id, int thread_id, int maat_scan_string(struct maat *instance, int table_id, int thread_id,
const char *data, size_t data_len, int results[], size_t *n_result, const char *data, size_t data_len, int results[], size_t *n_result,
struct maat_state *state); struct maat_state **state);
struct maat_stream; struct maat_stream;
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id); struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id);
int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* data, int data_len, int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* data, int data_len,
int results[], size_t *n_result, struct maat_state *state); int results[], size_t *n_result, struct maat_state **state);
void maat_scan_stream_close(struct maat_stream **stream); void maat_scan_stream_close(struct maat_stream **stream);
void maat_state_reset(struct maat_state *state); int maat_state_set(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, const void *value, int size);
//return >=0 if success, return -1 when failed;
int maat_state_get(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, void *value, int size);
void maat_state_reset(struct maat_state **state);
#ifdef __cpluscplus #ifdef __cpluscplus
} }

4
src/CMakeLists.txt
View File

@@ -10,7 +10,7 @@ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC) add_definitions(-fPIC)
set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp
maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp
maat_command.cpp maat_redis_monitor.cpp) maat_command.cpp maat_redis_monitor.cpp maat_hierarchy.cpp)
set(LIB_SOURCE_FILES set(LIB_SOURCE_FILES
${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c ${PROJECT_SOURCE_DIR}/deps/log/log.c) ${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c ${PROJECT_SOURCE_DIR}/deps/log/log.c)
@@ -26,7 +26,7 @@ set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX)
set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe) set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe)
set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
target_link_libraries(maat_frame_static adapter-static hiredis-static pthread crypto z) target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static pthread crypto z)
# Shared Library Output # Shared Library Output
#add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES}) #add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES})

79
src/inc_internal/alignment.h Normal file
View File

@@ -0,0 +1,79 @@
/*
**********************************************************************************************
* File: alignment.h
* Description: alignment statistics
* Authors: Zheng Chao <zhengchao@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _ALIGNMENT_H_
#define _ALIGNMENT_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include <stdlib.h>
#include <string.h>
#define CPU_CACHE_ALIGMENT 64
inline long long *alignment_int64_array_alloc(int size)
{
return (long long *)calloc(CPU_CACHE_ALIGMENT, size);
}
inline long long alignment_int64_array_sum(long long *array, int size)
{
int offset = 0;
long long sum = 0;
for (int i = 0; i < size; i++) {
offset = (CPU_CACHE_ALIGMENT / sizeof(long long)) * i;
sum += array[offset];
}
return sum;
}
inline long long alignment_int64_array_add(long long *array, int offset, long long op_val)
{
int idx = (CPU_CACHE_ALIGMENT / sizeof(long long)) * offset;
array[idx] += op_val;
return array[idx];
}
inline long long alignment_int64_array_cnt(long long *array, int size)
{
int offset = 0;
int cnt = 0;
for (int i = 0; i < size; i++) {
offset = (CPU_CACHE_ALIGMENT / sizeof(long long)) * i;
if (array[offset] > 0) {
cnt++;
}
}
return cnt;
}
inline void alignment_int64_array_reset(long long *array, int size)
{
memset(array, 0, CPU_CACHE_ALIGMENT * size);
}
inline void alignment_int64_array_free(long long *array)
{
free(array);
}
#ifdef __cpluscplus
}
#endif
#endif

3
src/inc_internal/json2iris.h
View File

@@ -18,8 +18,7 @@ extern "C"
#include "hiredis/hiredis.h" #include "hiredis/hiredis.h"
int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, int json2iris(const char *json_buff, const char *json_filename, redisContext *redis_write_ctx,
const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx,
char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo, char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo,
struct log_handle *logger); struct log_handle *logger);

7
src/inc_internal/maat_command.h
View File

@@ -30,6 +30,13 @@ struct maat_cmd_line
int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout. int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout.
}; };
/**
* @brief write one line to redis
*
* @retval
* success: number of successfully updated rule.
* failed: -1
*/
int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule); int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule);
#ifdef __cpluscplus #ifdef __cpluscplus

70
src/inc_internal/maat_hierarchy.h Normal file
View File

@@ -0,0 +1,70 @@
/*
**********************************************************************************************
* File: maat_hierarchy.h
* Description:
* Authors: Zheng Chao <zhengchao@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_HIERARCHY_H_
#define _MAAT_HIERARCHY_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "maat_garbage_collection.h"
/* maat hierarchy API */
struct maat_hierarchy;
struct maat_hierarchy *maat_hierarchy_new(int thread_num, struct maat_garbage_bin *bin, struct log_handle *logger);
void maat_hierarchy_free(struct maat_hierarchy *hier);
int maat_hierarchy_rebuild(struct maat_hierarchy *hier);
size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
struct maat_hit_path_t *hit_paths, size_t n_path);
void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
void maat_hierarchy_set_region_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
/* maat hierarchy compile mid API */
struct maat_hierarchy_compile_mid;
struct maat_hierarchy_compile_mid *maat_hierarchy_compile_mid_new(struct maat_hierarchy *hier, int thread_id);
void maat_hierarchy_compile_mid_free(struct maat_hierarchy_compile_mid *mid);
void maat_hierarchy_compile_mid_update(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result);
int maat_hierarchy_compile_mid_has_NOT_clause(struct maat_hierarchy_compile_mid *mid);
/* maat hierarchy compile API */
int maat_hierarchy_compile_add(struct maat_hierarchy *hier, int compile_id, int declared_clause_num, void* user_data);
int maat_hierarchy_compile_remove(struct maat_hierarchy *hier, int compile_id);
void *maat_hierarchy_compile_dettach_user_data(struct maat_hierarchy *hier, int compile_id);
void *maat_hierarchy_compile_read_user_data(struct maat_hierarchy *hier, int compile_id);
void maat_hierarchy_compile_user_data_iterate(struct maat_hierarchy *hier, void (*callback)(void *user_data, void *param), void *param);
int maat_hierarchy_region_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
int is_last_compile, void **user_data_array, size_t ud_array_sz);
void *maat_hierarchy_region_dettach_user_data(struct maat_hierarchy *hier, int region_id);
/* maat hierarchy region2group API */
int maat_hierarchy_add_region_to_group(struct maat_hierarchy *hier, int group_id, int region_id, int table_id, void* user_data);
int maat_hierarchy_remove_region_from_group(struct maat_hierarchy *hier, int group_id, int region_id);
/* maat hierarchy group2group API */
int maat_hierarchy_add_group_to_group(struct maat_hierarchy *hier, int group_id, int superior_group_id);
int maat_hierarchy_remove_group_from_group(struct maat_hierarchy *hier, int group_id, int superior_group_id);
/* maat hierarchy group2compile API */
int maat_hierarchy_add_group_to_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag,
int clause_index, int compile_id);
int maat_hierarchy_remove_group_from_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag,
int clause_index, int compile_id);
#ifdef __cpluscplus
}
#endif
#endif

64
src/inc_internal/maat_rule.h
View File

@@ -29,6 +29,57 @@ extern "C"
#include "uthash/uthash.h" #include "uthash/uthash.h"
#include "maat_table_schema.h" #include "maat_table_schema.h"
#include "maat_command.h" #include "maat_command.h"
#include "IPMatcher.h"
struct maat_rule_head {
int config_id;
int service_id;
char do_log;
char do_blacklist;
char action;
char resevered;
int serv_def_len;
};
#define MAX_SERVICE_DEFINE_LEN 128
struct maat_rule {
int config_id;
int service_id;
uint8_t do_log;
uint8_t do_blacklist;
uint8_t action;
uint8_t reserved;
int serv_def_len;
char service_defined[MAX_SERVICE_DEFINE_LEN];
};
#define REGION_RULE_MAGIC 0x4d3c2b1a
struct maat_region_inner
{
long long magic_num;
int region_id;
int group_id;
int district_id;
int table_id;
int expr_id_cnt;
int expr_id_lb; //low boundary
int expr_id_ub; //up boundary
};
#define COMPILE_RULE_MAGIC 0x1a2b3c4d
struct maat_compile_rule
{
long long magic_num;
struct maat_rule_head head;// fix len of Maat_rule_t
char *service_defined;
int is_valid;
int declared_clause_num;
double evaluation_order;
struct table_schema *ref_table;
void *ex_data;
int compile_id;
pthread_rwlock_t rwlock;
};
struct maat_runtime { struct maat_runtime {
/* maat_runtime can be created and destroy dynamic, so need version info */ /* maat_runtime can be created and destroy dynamic, so need version info */
@@ -36,11 +87,18 @@ struct maat_runtime {
time_t last_update_time; time_t last_update_time;
long long *ref_cnt;
struct table_runtime_manager *table_rt_mgr; struct table_runtime_manager *table_rt_mgr;
size_t max_table_num; size_t max_table_num;
size_t max_thread_num; size_t max_thread_num;
uint32_t rule_num; uint32_t rule_num;
struct maat_hierarchy *hier;
struct maat_garbage_bin *ref_garbage_bin;
struct scan_result *region_result_buff;
struct log_handle *logger;
}; };
enum data_source { enum data_source {
@@ -148,6 +206,12 @@ struct maat {
/* statistics */ /* statistics */
long long line_cmd_acc_num; long long line_cmd_acc_num;
long long *outer_mid_cnt;
long long *compile_mid_cnt;
long long *thread_call_cnt;
long long scan_err_cnt;
}; };
void maat_start_cb(long long new_version, int update_type, void *u_para); void maat_start_cb(long long new_version, int update_type, void *u_para);

6
src/inc_internal/maat_table_runtime.h
View File

@@ -57,7 +57,7 @@ void table_runtime_update(struct table_runtime *table_rt, struct table_schema *t
*/ */
int table_runtime_updating_flag(struct table_runtime *table_rt); int table_runtime_updating_flag(struct table_runtime *table_rt);
void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread); void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread, struct log_handle *logger);
/* table runtime scan API */ /* table runtime scan API */
int table_runtime_scan_string(struct table_runtime *table_rt, int thread_id, const char *data, size_t data_len, int table_runtime_scan_string(struct table_runtime *table_rt, int thread_id, const char *data, size_t data_len,
@@ -67,7 +67,7 @@ void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id);
int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, int results[], size_t *n_result); int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, int results[], size_t *n_result);
void table_runtime_stream_close(struct table_runtime *table_rt); void table_runtime_stream_close(struct table_runtime *table_rt);
int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data, int results[], size_t *n_result); int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data, struct scan_result *results, size_t *n_result, size_t n_result_array);
/* table runtime cached row API */ /* table runtime cached row API */
size_t table_runtime_cached_row_count(struct table_runtime *table_rt); size_t table_runtime_cached_row_count(struct table_runtime *table_rt);
@@ -78,7 +78,7 @@ const char* table_runtime_get_cached_row(struct table_runtime *table_rt, size_t
struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt); struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt);
void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema, void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema,
struct log_handle *logger); int nr_worker_num, struct log_handle *logger);
#ifdef __cpluscplus #ifdef __cpluscplus
} }

54
src/inc_internal/maat_table_schema.h
View File

@@ -27,17 +27,18 @@ extern "C"
#define MAX_KEYWORDS_STR 1024 #define MAX_KEYWORDS_STR 1024
#define MAX_FOREIGN_CLMN_NUM 8 #define MAX_FOREIGN_CLMN_NUM 8
enum table_composition_type { enum component_table_type {
COMPOSITION_TYPE_SIP = 0, COMPONENT_TABLE_TYPE_NONE = -1,
COMPOSITION_TYPE_DIP, COMPONENT_TABLE_TYPE_SIP = 0,
COMPOSITION_TYPE_SESSION, COMPONENT_TABLE_TYPE_DIP,
COMPOSITION_TYPE_MAX COMPONENT_TABLE_TYPE_SESSION,
COMPONENT_TABLE_TYPE_MAX
}; };
enum table_type { enum table_type {
TABLE_TYPE_INVALID = -1,
TABLE_TYPE_EXPR = 0, TABLE_TYPE_EXPR = 0,
TABLE_TYPE_EXPR_PLUS, TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_IP,
TABLE_TYPE_IP_PLUS, TABLE_TYPE_IP_PLUS,
TABLE_TYPE_INTERVAL, TABLE_TYPE_INTERVAL,
TABLE_TYPE_INTERVAL_PLUS, TABLE_TYPE_INTERVAL_PLUS,
@@ -53,8 +54,7 @@ enum table_type {
TABLE_TYPE_COMPILE, TABLE_TYPE_COMPILE,
TABLE_TYPE_GROUP, TABLE_TYPE_GROUP,
TABLE_TYPE_GROUP2GROUP, TABLE_TYPE_GROUP2GROUP,
TABLE_TYPE_GROUP2COMPILE, TABLE_TYPE_GROUP2COMPILE
TABLE_TYPE_MAX
}; };
enum expr_type { enum expr_type {
@@ -172,6 +172,16 @@ struct ex_data_schema
int set_flag; int set_flag;
}; };
struct compile_ex_data_schema {
maat_rule_ex_new_func_t *new_func;
maat_rule_ex_free_func_t *free_func;
maat_rule_ex_dup_func_t *dup_func;
long argl;
void *argp;
int idx;
int table_id;
};
struct table_schema; struct table_schema;
struct table_schema_manager; struct table_schema_manager;
@@ -185,7 +195,7 @@ int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_
* @brief get composition table's child table(specified by type) id * @brief get composition table's child table(specified by type) id
*/ */
int table_schema_manager_get_child_table_id(struct table_schema_manager *table_schema_mgr, int parent_table_id, int table_schema_manager_get_child_table_id(struct table_schema_manager *table_schema_mgr, int parent_table_id,
enum table_composition_type type); enum component_table_type type);
enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int table_id); enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int table_id);
@@ -216,23 +226,33 @@ void table_schema_set_updating_name(struct table_schema *table_schema, const cha
/* expr table schema API */ /* expr table schema API */
enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema); enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);
/* plugin table schema API */ /* table schema ex API */
int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema, int table_schema_set_ex_data_schema(struct table_schema *table_schema,
maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func, maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp, long argl, void *argp,
struct log_handle *logger); struct log_handle *logger);
struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schema *table_schema); struct ex_data_schema *table_schema_get_ex_data_schema(struct table_schema *table_schema);
/* table schema compile rule ex API */
int table_schema_set_compile_rule_ex_data_schema(struct table_schema *table_schema,
maat_rule_ex_new_func_t *new_func,
maat_rule_ex_free_func_t *free_func,
maat_rule_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger);
struct compile_ex_data_schema *table_schema_get_compile_rule_ex_data_schema(struct table_schema *table_schema, size_t idx);
size_t table_schema_compile_rule_ex_data_schema_count(struct table_schema *table_schema);
/** /**
* @brief if plugin table schema's ex data schema set * @brief if plugin table schema's ex data schema set
* *
* @retval 1(already Set) 0(Not set yet) * @retval 1(already Set) 0(Not set yet)
*/ */
int plugin_table_schema_ex_data_schema_flag(struct table_schema *table_schema); int table_schema_ex_data_schema_flag(struct table_schema *table_schema);
int plugin_table_schema_add_callback(struct table_schema_manager *table_schema_mgr, int table_id, int table_schema_add_callback(struct table_schema_manager *table_schema_mgr, int table_id,
maat_start_callback_t *start, maat_start_callback_t *start,
maat_update_callback_t *update, maat_update_callback_t *update,
maat_finish_callback_t *finish, maat_finish_callback_t *finish,
@@ -240,11 +260,11 @@ int plugin_table_schema_add_callback(struct table_schema_manager *table_schema_m
/** /**
* @brief the number of callback function stored in plugin table schema * @brief the number of callback function stored in plugin table schema
*/ */
size_t plugin_table_schema_callback_count(struct table_schema *table_schema); size_t table_schema_callback_count(struct table_schema *table_schema);
void plugin_table_schema_all_cb_update(struct table_schema *table_schema, const char *row); void table_schema_all_cb_update(struct table_schema *table_schema, const char *row);
int plugin_table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns); int table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns);
#ifdef __cpluscplus #ifdef __cpluscplus
} }

2
src/inc_internal/maat_utils.h
View File

@@ -31,6 +31,8 @@ extern "C"
#define MIN(a, b) (((a) < (b)) ? (a) : (b)) #define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif #endif
#define MAX_SCANNER_HIT_NUM 4096
enum maat_ip_format { enum maat_ip_format {
IP_FORMAT_RANGE, IP_FORMAT_RANGE,
IP_FORMAT_MASK, IP_FORMAT_MASK,

106
src/json2iris.cpp
View File

@@ -130,7 +130,6 @@ int set_iris_descriptor(const char *json_file, cJSON *json, const char *encrypt_
maat_kv_register(iris_cfg->str2int_map, "yes", 1); maat_kv_register(iris_cfg->str2int_map, "yes", 1);
maat_kv_register(iris_cfg->str2int_map, "no", 0); maat_kv_register(iris_cfg->str2int_map, "no", 0);
maat_kv_register(iris_cfg->str2int_map, "ip", TABLE_TYPE_IP);
maat_kv_register(iris_cfg->str2int_map, "ip_plus", TABLE_TYPE_IP_PLUS); maat_kv_register(iris_cfg->str2int_map, "ip_plus", TABLE_TYPE_IP_PLUS);
maat_kv_register(iris_cfg->str2int_map, "string", TABLE_TYPE_EXPR); maat_kv_register(iris_cfg->str2int_map, "string", TABLE_TYPE_EXPR);
maat_kv_register(iris_cfg->str2int_map, "expr", TABLE_TYPE_EXPR); maat_kv_register(iris_cfg->str2int_map, "expr", TABLE_TYPE_EXPR);
@@ -433,94 +432,6 @@ int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);
} }
int write_ip_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table,
struct log_handle *logger)
{
struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0;
memset(json_cmd, 0, sizeof(json_cmd));
json_cmd[cmd_cnt].json_string = "region_id";
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "group_id";
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "addr_type";
json_cmd[cmd_cnt].json_type = cJSON_String;
json_cmd[cmd_cnt].str2int_flag = 1;
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "src_ip";
json_cmd[cmd_cnt].json_type = cJSON_String;
json_cmd[cmd_cnt].empty_allowed = 1;
json_cmd[cmd_cnt].default_string = "0.0.0.0";
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "mask_src_ip";
json_cmd[cmd_cnt].json_type = cJSON_String;
json_cmd[cmd_cnt].empty_allowed = 1;
json_cmd[cmd_cnt].default_string = "255.255.255.255";
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "src_port";
json_cmd[cmd_cnt].json_type = cJSON_String;
json_cmd[cmd_cnt].empty_allowed = 1;
json_cmd[cmd_cnt].default_string = "0";
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "mask_src_port";
json_cmd[cmd_cnt].json_type = cJSON_String;
json_cmd[cmd_cnt].empty_allowed = 1;
json_cmd[cmd_cnt].default_string = "65535";
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "dst_ip";
json_cmd[cmd_cnt].json_type = cJSON_String;
json_cmd[cmd_cnt].empty_allowed = 1;
json_cmd[cmd_cnt].default_string = "0.0.0.0";
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "mask_dst_ip";
json_cmd[cmd_cnt].json_type = cJSON_String;
json_cmd[cmd_cnt].empty_allowed = 1;
json_cmd[cmd_cnt].default_string = "255.255.255.255";
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "dst_port";
json_cmd[cmd_cnt].json_type = cJSON_String;
json_cmd[cmd_cnt].empty_allowed = 1;
json_cmd[cmd_cnt].default_string = "0";
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "mask_dst_port";
json_cmd[cmd_cnt].json_type = cJSON_String;
json_cmd[cmd_cnt].empty_allowed = 1;
json_cmd[cmd_cnt].default_string = "65535";
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "protocol";
json_cmd[cmd_cnt].json_type = cJSON_Number;
json_cmd[cmd_cnt].empty_allowed = 1;
json_cmd[cmd_cnt].default_int = 0;
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "direction";
json_cmd[cmd_cnt].json_type = cJSON_String;
json_cmd[cmd_cnt].str2int_flag = 1;
json_cmd[cmd_cnt].empty_allowed = 1;
json_cmd[cmd_cnt].default_string = "double";
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "is_valid";
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);
}
int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table,
struct log_handle *logger) struct log_handle *logger)
{ {
@@ -779,9 +690,6 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i
case TABLE_TYPE_EXPR_PLUS: case TABLE_TYPE_EXPR_PLUS:
ret = write_expr_line(table_content, p_iris, table_info, logger); ret = write_expr_line(table_content, p_iris, table_info, logger);
break; break;
case TABLE_TYPE_IP:
ret = write_ip_line(table_content, p_iris, table_info, logger);
break;
case TABLE_TYPE_IP_PLUS: case TABLE_TYPE_IP_PLUS:
ret = write_ip_plus_line(table_content, p_iris, table_info, logger); ret = write_ip_plus_line(table_content, p_iris, table_info, logger);
break; break;
@@ -1163,13 +1071,15 @@ int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle *
return 0; return 0;
} }
int json2iris(const char *json_buff, const char *json_filename, const char *compile_tn, int json2iris(const char *json_buff, const char *json_filename, redisContext *redis_write_ctx,
const char *group2compile_tn, const char *group2group_tn, redisContext *redis_write_ctx,
char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo, char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo,
struct log_handle *logger) struct log_handle *logger)
{ {
int ret = -1; int ret = -1;
cJSON *tmp_obj = NULL; cJSON *tmp_obj = NULL;
const char *compile_tbl_name = NULL;
const char *group2compile_tbl_name = NULL;
const char *group2group_tbl_name = NULL;
struct iris_description iris_cfg; struct iris_description iris_cfg;
memset(&iris_cfg, 0, sizeof(iris_cfg)); memset(&iris_cfg, 0, sizeof(iris_cfg));
@@ -1182,21 +1092,21 @@ int json2iris(const char *json_buff, const char *json_filename, const char *comp
tmp_obj = cJSON_GetObjectItem(json, "compile_table"); tmp_obj = cJSON_GetObjectItem(json, "compile_table");
if (tmp_obj) { if (tmp_obj) {
compile_tn = tmp_obj->valuestring; compile_tbl_name = tmp_obj->valuestring;
} }
tmp_obj = cJSON_GetObjectItem(json, "group2compile_table"); tmp_obj = cJSON_GetObjectItem(json, "group2compile_table");
if (tmp_obj) { if (tmp_obj) {
group2compile_tn = tmp_obj->valuestring; group2compile_tbl_name = tmp_obj->valuestring;
} }
tmp_obj = cJSON_GetObjectItem(json, "group2group_table"); tmp_obj = cJSON_GetObjectItem(json, "group2group_table");
if (tmp_obj) { if (tmp_obj) {
group2group_tn = tmp_obj->valuestring; group2group_tbl_name = tmp_obj->valuestring;
} }
ret = set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, ret = set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo,
compile_tn, group2compile_tn, group2group_tn, compile_tbl_name, group2compile_tbl_name, group2group_tbl_name,
redis_write_ctx, &iris_cfg); redis_write_ctx, &iris_cfg);
if (ret < 0) { if (ret < 0) {
goto error_out; goto error_out;

218
src/maat_api.cpp
View File

@@ -25,9 +25,25 @@
#include "maat_table_runtime.h" #include "maat_table_runtime.h"
#include "maat_config_monitor.h" #include "maat_config_monitor.h"
#include "maat_redis_monitor.h" #include "maat_redis_monitor.h"
#include "maat_hierarchy.h"
#include "alignment.h"
#define MODULE_MAAT_API module_name_str("maat.api") #define MODULE_MAAT_API module_name_str("maat.api")
#define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2
struct maat_state
{
struct maat *maat_instance;
int16_t thread_id;
unsigned char is_set_district;
unsigned char is_last_scan;
int district_id; //-1: Any District; -2: Unkonwn District;
int scan_cnt;
struct maat_hierarchy_compile_mid *compile_mid;
};
struct maat_options* maat_options_new(void) struct maat_options* maat_options_new(void)
{ {
struct maat_options *options = ALLOC(struct maat_options, 1); struct maat_options *options = ALLOC(struct maat_options, 1);
@@ -267,6 +283,11 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
garbage_gc_timeout_s = (maat_instance->rule_effect_interval_ms / 1000) + garbage_gc_timeout_s = (maat_instance->rule_effect_interval_ms / 1000) +
(maat_instance->gc_timeout_ms / 1000); (maat_instance->gc_timeout_ms / 1000);
maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s); maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s);
maat_instance->outer_mid_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->compile_mid_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->thread_call_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
pthread_mutex_init(&(maat_instance->background_update_mutex), NULL); pthread_mutex_init(&(maat_instance->background_update_mutex), NULL);
if (0 == maat_instance->deferred_load) { if (0 == maat_instance->deferred_load) {
@@ -303,6 +324,26 @@ int maat_table_get_id(struct maat *maat_instance, const char *table_name)
return table_id; return table_id;
} }
inline void maat_runtime_ref_inc(struct maat_runtime *maat_rt, int thread_id)
{
alignment_int64_array_add(maat_rt->ref_cnt, thread_id, 1);
}
inline void maat_runtime_ref_dec(struct maat_runtime *maat_rt, int thread_id)
{
alignment_int64_array_add(maat_rt->ref_cnt, thread_id, -1);
}
inline int scan_state_should_compile_NOT(struct maat_state *mid)
{
if (mid && (1 == mid->is_last_scan) && mid->compile_mid &&
maat_hierarchy_compile_mid_has_NOT_clause(mid->compile_mid)) {
return 1;
} else {
return 0;
}
}
int maat_table_callback_register(struct maat *maat_instance, int table_id, int maat_table_callback_register(struct maat *maat_instance, int table_id,
maat_start_callback_t *start, maat_start_callback_t *start,
maat_update_callback_t *update, maat_update_callback_t *update,
@@ -312,7 +353,7 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id,
int ret = -1; int ret = -1;
pthread_mutex_lock(&(maat_instance->background_update_mutex)); pthread_mutex_lock(&(maat_instance->background_update_mutex));
ret = plugin_table_schema_add_callback(maat_instance->table_schema_mgr, table_id, ret = table_schema_add_callback(maat_instance->table_schema_mgr, table_id,
start, update, finish, u_para, maat_instance->logger); start, update, finish, u_para, maat_instance->logger);
if (ret < 0) { if (ret < 0) {
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_instance->background_update_mutex));
@@ -358,7 +399,7 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i
{ {
struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
pthread_mutex_lock(&(maat_instance->background_update_mutex)); pthread_mutex_lock(&(maat_instance->background_update_mutex));
int ret = plugin_table_schema_set_ex_data_schema(table_schema, new_func, free_func, dup_func, int ret = table_schema_set_ex_data_schema(table_schema, new_func, free_func, dup_func,
argl, argp, maat_instance->logger); argl, argp, maat_instance->logger);
if (ret < 0) { if (ret < 0) {
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_instance->background_update_mutex));
@@ -368,7 +409,8 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i
struct table_runtime *table_rt = NULL; struct table_runtime *table_rt = NULL;
if (maat_instance->maat_rt != NULL) { if (maat_instance->maat_rt != NULL) {
table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id); table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id);
table_runtime_commit_ex_data_schema(table_rt, table_schema, maat_instance->logger); table_runtime_commit_ex_data_schema(table_rt, table_schema, maat_instance->nr_worker_thread,
maat_instance->logger);
} }
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_instance->background_update_mutex));
@@ -401,24 +443,68 @@ void *maat_plugin_table_dup_ex_data(struct maat *maat_instance, int table_id,
return ex_data_runtime_dup_ex_data(ex_data_rt, key, key_len); return ex_data_runtime_dup_ex_data(ex_data_rt, key, key_len);
} }
static void scan_count_inc(struct maat_state *mid)
{
mid->scan_cnt++;
}
struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id)
{
struct maat_state *outer_state = NULL;
outer_state = ALLOC(struct maat_state, 1);
outer_state->maat_instance = maat_instance;
outer_state->district_id = DISTRICT_ANY;
outer_state->thread_id = (signed short)thread_id;
return outer_state;
}
struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instance, int thread_id, int is_hit_region)
{
struct maat_state *mid = *state;
if (NULL == mid) {
mid = make_outer_state(maat_instance, thread_id);
*state = mid;
//Maat_set_scan_status calls grap_mid() with thread_num=-1.
if (mid->thread_id >= 0) {
alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1);
}
}
if (mid->thread_id < 0 && thread_id >= 0) {
mid->thread_id = thread_id;
alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1);
}
if (is_hit_region == 1) {
if (NULL == mid->compile_mid) {
mid->compile_mid = maat_hierarchy_compile_mid_new(maat_instance->maat_rt->hier, thread_id);
alignment_int64_array_add(maat_instance->compile_mid_cnt, thread_id, 1);
}
}
return mid;
}
int maat_scan_integer(struct maat *instance, int table_id, int thread_id, int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
unsigned int intval, int results[], size_t n_result, unsigned int intval, int results[], size_t *n_result,
struct maat_state *state) struct maat_state **state)
{ {
return 0; return 0;
} }
static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4tuple *addr, enum table_composition_type child_type) static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4tuple *addr, enum component_table_type child_type)
{ {
switch (addr->type) { switch (addr->type) {
case IP_TYPE_V4: case IP_TYPE_V4:
table_rt_addr->ip_type = IP_TYPE_V4; table_rt_addr->ip_type = IP_TYPE_V4;
switch (child_type) { switch (child_type) {
case COMPOSITION_TYPE_SIP: case COMPONENT_TABLE_TYPE_SIP:
table_rt_addr->ipv4 = ntohl(addr->ipv4.sip); table_rt_addr->ipv4 = ntohl(addr->ipv4.sip);
table_rt_addr->port = ntohs(addr->ipv4.sport); table_rt_addr->port = ntohs(addr->ipv4.sport);
break; break;
case COMPOSITION_TYPE_DIP: case COMPONENT_TABLE_TYPE_DIP:
table_rt_addr->ipv4 = ntohl(addr->ipv4.dip); table_rt_addr->ipv4 = ntohl(addr->ipv4.dip);
table_rt_addr->port = ntohs(addr->ipv4.dport); table_rt_addr->port = ntohs(addr->ipv4.dport);
break; break;
@@ -430,12 +516,12 @@ static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4
case IP_TYPE_V6: case IP_TYPE_V6:
table_rt_addr->ip_type = IP_TYPE_V6; table_rt_addr->ip_type = IP_TYPE_V6;
switch (child_type) { switch (child_type) {
case COMPOSITION_TYPE_SIP: case COMPONENT_TABLE_TYPE_SIP:
memcpy(table_rt_addr->ipv6, addr->ipv6.sip, sizeof(addr->ipv6.sip)); memcpy(table_rt_addr->ipv6, addr->ipv6.sip, sizeof(addr->ipv6.sip));
ipv6_ntoh(table_rt_addr->ipv6); ipv6_ntoh(table_rt_addr->ipv6);
table_rt_addr->port = ntohs(addr->ipv6.sport); table_rt_addr->port = ntohs(addr->ipv6.sport);
break; break;
case COMPOSITION_TYPE_DIP: case COMPONENT_TABLE_TYPE_DIP:
memcpy(table_rt_addr->ipv6, addr->ipv6.dip, sizeof(addr->ipv6.dip)); memcpy(table_rt_addr->ipv6, addr->ipv6.dip, sizeof(addr->ipv6.dip));
ipv6_ntoh(table_rt_addr->ipv6); ipv6_ntoh(table_rt_addr->ipv6);
table_rt_addr->port = ntohs(addr->ipv6.dport); table_rt_addr->port = ntohs(addr->ipv6.dport);
@@ -454,16 +540,17 @@ static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4
} }
static int ip_composition_scan(int thread_id, struct addr_4tuple *addr, static int ip_composition_scan(int thread_id, struct addr_4tuple *addr,
int parent_table_id, enum table_composition_type child_type, int parent_table_id, enum component_table_type child_type,
int *virtual_table_id, int *virtual_table_id,
struct table_schema_manager *table_schema_mgr, struct table_schema_manager *table_schema_mgr,
struct table_runtime_manager *table_rt_mgr, int results[], size_t *n_result) struct table_runtime_manager *table_rt_mgr,
struct scan_result *region_results, size_t n_result_array)
{ {
int child_table_id = 0; int child_table_id = 0;
if (child_type == COMPOSITION_TYPE_MAX) { if (child_type == COMPONENT_TABLE_TYPE_NONE) {
child_table_id = parent_table_id; child_table_id = parent_table_id;
child_type = COMPOSITION_TYPE_SIP; child_type = COMPONENT_TABLE_TYPE_SESSION;
} else { } else {
child_table_id = table_schema_manager_get_child_table_id(table_schema_mgr, parent_table_id, child_type); child_table_id = table_schema_manager_get_child_table_id(table_schema_mgr, parent_table_id, child_type);
} }
@@ -493,51 +580,93 @@ static int ip_composition_scan(int thread_id, struct addr_4tuple *addr,
memset(&scan_data, 0, sizeof(struct table_rt_2tuple)); memset(&scan_data, 0, sizeof(struct table_rt_2tuple));
ip_scan_data_set(&scan_data, addr, child_type); ip_scan_data_set(&scan_data, addr, child_type);
size_t hit_cnt = 0;
return table_runtime_scan_ip(table_rt, thread_id, &scan_data, results, n_result); int ret = table_runtime_scan_ip(table_rt, thread_id, &scan_data, region_results, &hit_cnt, n_result_array);
} if (ret < 0) {
int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct addr_4tuple *addr,
int results[], size_t *n_result, struct maat_state *state)
{
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
(thread_id < 0) || (NULL == addr) || (NULL == results) || (NULL == n_result)) {
return -1; return -1;
} }
struct table_runtime_manager *table_rt_mgr = maat_instance->maat_rt->table_rt_mgr; return hit_cnt;
}
int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct addr_4tuple *addr,
int results[], size_t *n_result, struct maat_state **state)
{
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
(thread_id < 0) || (NULL == addr) || (NULL == results) || (NULL == n_result) ||
(NULL == state)) {
return -1;
}
struct maat_state *mid = NULL;
mid = grab_mid(state, maat_instance, thread_id, 0);
scan_count_inc(mid);
struct maat_runtime *maat_rt = maat_instance->maat_rt;
if (NULL == maat_rt) {
return 0;
}
struct table_runtime_manager *table_rt_mgr = maat_rt->table_rt_mgr;
struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id); struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id);
enum table_type table_type = table_runtime_get_type(table_rt); enum table_type table_type = table_runtime_get_type(table_rt);
if (table_type == TABLE_TYPE_INVALID) {
maat_instance->scan_err_cnt++;
return -1;
}
int region_ret = 0; int region_ret = 0;
int virtual_table_id = 0; int virtual_table_id = 0;
struct scan_result *region_result = maat_rt->region_result_buff + thread_id * MAX_SCANNER_HIT_NUM;
int region_hit_cnt = 0;
int region_result_virtual_table_ids[MAX_SCANNER_HIT_NUM];
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
maat_runtime_ref_inc(maat_rt, thread_id);
if (table_type == TABLE_TYPE_COMPOSITION) { if (table_type == TABLE_TYPE_COMPOSITION) {
/* region_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_SIP, &virtual_table_id,
enum table_composition_type childs[3] = {COMPOSITION_TYPE_SIP, COMPOSITION_TYPE_DIP, COMPOSITION_TYPE_SESSION}; maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr,
region_result + region_hit_cnt, MAX_SCANNER_HIT_NUM - region_hit_cnt);
region_hit_cnt += region_ret;
/*enum component_table_type childs[3] = {COMPONENT_TABLE_TYPE_SIP, COMPONENT_TABLE_TYPE_DIP, COMPONENT_TABLE_TYPE_SESSION};
for (int i = 0; i < 3; i++) { for (int i = 0; i < 3; i++) {
region_ret = ip_composition_scan(table_rt, thread_id, ip, table_id, childs[i], maat_instance->table_schema_mgr, region_ret = ip_composition_scan(thread_id, addr, table_id, childs[i], &virtual_table_id,
maat_instance->maat_rt->table_rt_mgr, results, n_result);
}*/
region_ret = ip_composition_scan(thread_id, addr, table_id, COMPOSITION_TYPE_SIP, &virtual_table_id,
maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr,
results, n_result); region_result + region_hit_cnt, MAX_SCANNER_HIT_NUM - region_hit_cnt);
if (region_ret < 0) {
maat_instance->scan_err_cnt++;
} else { } else {
region_ret = ip_composition_scan(thread_id, addr, table_id, COMPOSITION_TYPE_MAX, &virtual_table_id, for (int j = 0; j < region_ret; j++) {
region_result_virtual_table_ids[region_hit_cnt++] = virtual_table_id;
}
}
}*/
} else {
region_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_NONE, &virtual_table_id,
maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr,
results, n_result); region_result + region_hit_cnt, MAX_SCANNER_HIT_NUM - region_hit_cnt);
if (region_ret < 0) {
maat_instance->scan_err_cnt++;
} else {
region_hit_cnt += region_ret;
}
} }
return region_ret; *n_result = region_ret;
for (int i = 0; i < region_ret; i++) {
results[i] = region_result[i].rule_id;
}
return 0;
} }
int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
const char *data, size_t data_len, int results[], size_t *n_result, const char *data, size_t data_len, int results[], size_t *n_result,
struct maat_state *state) struct maat_state **state)
{ {
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
(thread_id < 0) || (NULL == data) || (0 == data_len) || (NULL == results) || (thread_id < 0) || (NULL == data) || (0 == data_len) || (NULL == results) ||
(NULL == n_result)) { (NULL == n_result) || (NULL == state)) {
return -1; return -1;
} }
@@ -553,7 +682,7 @@ struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, i
} }
int maat_scan_stream(struct maat_stream **stream, int thread_id, const char *data, int data_len, int maat_scan_stream(struct maat_stream **stream, int thread_id, const char *data, int data_len,
int results[], size_t n_result, struct maat_state *state) int results[], size_t *n_result, struct maat_state **state)
{ {
return 0; return 0;
} }
@@ -563,7 +692,20 @@ void maat_scan_stream_close(struct maat_stream **stream)
} }
void maat_state_reset(struct maat_state *state) int maat_state_set(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt,
const void *value, int size)
{
}
//return >=0 if success, return -1 when failed;
int maat_state_get(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt,
void *value, int size)
{
}
void maat_state_reset(struct maat_state **state)
{ {
} }

7
src/maat_command.cpp
View File

@@ -162,9 +162,6 @@ int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type,
case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR:
column_seq = 7; column_seq = 7;
break; break;
case TABLE_TYPE_IP:
column_seq = 14;
break;
case TABLE_TYPE_IP_PLUS: case TABLE_TYPE_IP_PLUS:
column_seq = 18; column_seq = 18;
break; break;
@@ -332,8 +329,8 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
} }
enum table_type table_type = table_schema_get_table_type(table_schema); enum table_type table_type = table_schema_get_table_type(table_schema);
int is_valid = maat_cmd_get_valid_flag_offset(line_rule->table_line, table_type, valid_flag_column); int valid_offset = maat_cmd_get_valid_flag_offset(line_rule->table_line, table_type, valid_flag_column);
int is_valid = atoi(line_rule->table_line+valid_offset);
if (line_rule->expire_after > 0) { if (line_rule->expire_after > 0) {
absolute_expire_time = server_time + line_rule->expire_after; absolute_expire_time = server_time + line_rule->expire_after;
} }

3
src/maat_config_monitor.cpp
View File

@@ -409,8 +409,7 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
} }
} }
ret = json2iris((const char*)json_buff, json_filename, maat_instance->compile_tn, ret = json2iris((const char*)json_buff, json_filename, NULL,
maat_instance->group2compile_tn, maat_instance->group2group_tn, NULL,
maat_instance->json_ctx.iris_file, sizeof(maat_instance->json_ctx.iris_file), maat_instance->json_ctx.iris_file, sizeof(maat_instance->json_ctx.iris_file),
strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL, strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL,
strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL, strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL,

1323
src/maat_hierarchy.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

2
src/maat_redis_monitor.cpp
View File

@@ -112,7 +112,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, in
} }
int foreign_columns[8]; int foreign_columns[8];
int n_foreign_column = plugin_table_schema_get_foreign_column(table_schema, foreign_columns); int n_foreign_column = table_schema_get_foreign_column(table_schema, foreign_columns);
if (0 == n_foreign_column) { if (0 == n_foreign_column) {
continue; continue;
} }

66
src/maat_rule.cpp
View File

@@ -23,12 +23,57 @@
#include "maat_rule.h" #include "maat_rule.h"
#include "maat_config_monitor.h" #include "maat_config_monitor.h"
#include "maat_redis_monitor.h" #include "maat_redis_monitor.h"
#include "maat_table_runtime.h" #include "maat_table_runtime.h"
#include "maat_table_schema.h" #include "maat_table_schema.h"
#include "maat_hierarchy.h"
#include "alignment.h"
#define MODULE_MAAT_RULE module_name_str("maat.rule") #define MODULE_MAAT_RULE module_name_str("maat.rule")
void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head,
const char *srv_def, int srv_def_len)
{
memcpy(rule, rule_head, sizeof(struct maat_rule_head));
memcpy(rule->service_defined, srv_def, MIN(srv_def_len, MAX_SERVICE_DEFINE_LEN));
}
void rule_ex_data_free(const struct maat_rule_head *rule_head, const char *srv_def, void *ex_data,
const struct compile_ex_data_schema *ex_schema)
{
struct maat_rule rule;
memset(&rule, 0, sizeof(rule));
fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1);
ex_schema->free_func(ex_schema->idx, &rule, srv_def, ex_data, ex_schema->argl, ex_schema->argp);
}
void destroy_compile_rule(struct maat_compile_rule *compile_rule)
{
size_t n_compile_ex_schema = table_schema_compile_rule_ex_data_schema_count(compile_rule->ref_table);
assert(compile_rule->magic_num == COMPILE_RULE_MAGIC);
for (size_t i = 0; i < n_compile_ex_schema; i++) {
struct compile_ex_data_schema *compile_ex_schema = table_schema_get_compile_rule_ex_data_schema(compile_rule->ref_table, i);
rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ex_data+i, compile_ex_schema);
void *ex_data = compile_rule->ex_data + 1;
ex_data = NULL;
}
free(compile_rule->ex_data);
compile_rule->is_valid = 0;
compile_rule->declared_clause_num = -1;
FREE(compile_rule->service_defined);
FREE(compile_rule);
}
void maat_region_inner_free(struct maat_region_inner *region)
{
assert(region->magic_num == REGION_RULE_MAGIC);
assert(region->expr_id_cnt == 0 ||region->expr_id_cnt == (region->expr_id_ub - region->expr_id_lb + 1));
region->magic_num = 0;
FREE(region);
}
struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance) struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance)
{ {
struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1); struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1);
@@ -39,11 +84,20 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in
maat_instance->garbage_bin); maat_instance->garbage_bin);
maat_rt->max_table_num = table_schema_manager_get_size(maat_instance->table_schema_mgr); maat_rt->max_table_num = table_schema_manager_get_size(maat_instance->table_schema_mgr);
maat_rt->max_thread_num = maat_instance->nr_worker_thread; maat_rt->max_thread_num = maat_instance->nr_worker_thread;
maat_rt->hier = maat_hierarchy_new(maat_instance->nr_worker_thread, maat_instance->garbage_bin,
maat_instance->logger);
maat_hierarchy_set_compile_user_data_free_func(maat_rt->hier, (void (*)(void*))destroy_compile_rule);
maat_hierarchy_set_region_user_data_free_func(maat_rt->hier, (void (*)(void*))maat_region_inner_free);
maat_rt->logger = maat_instance->logger;
maat_rt->ref_garbage_bin = maat_instance->garbage_bin;
maat_rt->region_result_buff = ALLOC(struct scan_result, MAX_SCANNER_HIT_NUM * maat_instance->nr_worker_thread);
maat_rt->ref_cnt = alignment_int64_array_alloc(maat_instance->nr_worker_thread);
return maat_rt; return maat_rt;
} }
void maat_runtime_commit(struct maat_runtime *maat_rt) void maat_runtime_commit(struct maat_runtime *maat_rt, struct log_handle *logger)
{ {
for (size_t i = 0; i < maat_rt->max_table_num; i++) { for (size_t i = 0; i < maat_rt->max_table_num; i++) {
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, i); struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, i);
@@ -51,7 +105,7 @@ void maat_runtime_commit(struct maat_runtime *maat_rt)
continue; continue;
} }
table_runtime_commit(table_rt, maat_rt->max_thread_num); table_runtime_commit(table_rt, maat_rt->max_thread_num, logger);
} }
maat_rt->last_update_time = time(NULL); maat_rt->last_update_time = time(NULL);
@@ -159,13 +213,13 @@ void maat_finish_cb(void *u_param)
if (maat_instance->creating_maat_rt != NULL) { if (maat_instance->creating_maat_rt != NULL) {
maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt); maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt);
maat_runtime_commit(maat_instance->creating_maat_rt); maat_runtime_commit(maat_instance->creating_maat_rt, maat_instance->logger);
log_info(maat_instance->logger, MODULE_MAAT_RULE, "Full config version %llu load %d entries complete\n", log_info(maat_instance->logger, MODULE_MAAT_RULE, "Full config version %llu load %d entries complete\n",
maat_instance->creating_maat_rt->version, maat_instance->creating_maat_rt->rule_num); maat_instance->creating_maat_rt->version, maat_instance->creating_maat_rt->rule_num);
} else if (maat_instance->maat_rt != NULL) { } else if (maat_instance->maat_rt != NULL) {
maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt); maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt);
maat_instance->maat_rt->version = maat_instance->maat_version; maat_instance->maat_rt->version = maat_instance->maat_version;
maat_runtime_commit(maat_instance->maat_rt); maat_runtime_commit(maat_instance->maat_rt, maat_instance->logger);
log_info(maat_instance->logger, MODULE_MAAT_RULE, "Inc config version %llu load %d entries complete\n", log_info(maat_instance->logger, MODULE_MAAT_RULE, "Inc config version %llu load %d entries complete\n",
maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num); maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num);
} }
@@ -267,7 +321,7 @@ void *rule_monitor_loop(void *arg)
time_t time_window = time(NULL) - maat_instance->maat_rt->last_update_time; time_t time_window = time(NULL) - maat_instance->maat_rt->last_update_time;
if ((updating_flag > 0) && (time_window >= maat_instance->rule_effect_interval_ms / 1000)) { if ((updating_flag > 0) && (time_window >= maat_instance->rule_effect_interval_ms / 1000)) {
maat_runtime_commit(maat_instance->maat_rt); maat_runtime_commit(maat_instance->maat_rt, maat_instance->logger);
log_info(maat_instance->logger,MODULE_MAAT_RULE, log_info(maat_instance->logger,MODULE_MAAT_RULE,
"Actual update config version %u, %d entries load to rulescan after postpone.", "Actual update config version %u, %d entries load to rulescan after postpone.",
maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num); maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num);

65
src/maat_table_runtime.cpp
View File

@@ -23,6 +23,7 @@
#include "adapter_hs.h" #include "adapter_hs.h"
#include "rcu_hash.h" #include "rcu_hash.h"
#include "IPMatcher.h" #include "IPMatcher.h"
#include "alignment.h"
#define MODULE_TABLE_RUNTIME module_name_str("maat.table_runtime") #define MODULE_TABLE_RUNTIME module_name_str("maat.table_runtime")
@@ -69,6 +70,10 @@ struct table_runtime {
}; };
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
long long *scan_cnt;
long long *hit_cnt;
long long *stream_num;
//ex_data_rt //ex_data_rt
//table相关指针 //table相关指针
}; };
@@ -144,6 +149,10 @@ struct table_runtime *table_runtime_new(struct table_schema *table_schema, int m
break; break;
} }
table_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
table_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
table_rt->stream_num = alignment_int64_array_alloc(max_thread_num);
return table_rt; return table_rt;
} }
@@ -232,7 +241,7 @@ size_t table_runtime_rule_count(struct table_runtime *table_rt)
enum table_type table_runtime_get_type(struct table_runtime* table_rt) enum table_type table_runtime_get_type(struct table_runtime* table_rt)
{ {
if (NULL == table_rt) { if (NULL == table_rt) {
return TABLE_TYPE_MAX; return TABLE_TYPE_INVALID;
} }
return table_rt->table_type; return table_rt->table_type;
@@ -286,7 +295,7 @@ enum IP_TYPE ip_type_transform(enum ip_type type)
} }
int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data, int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data,
int results[], size_t *n_result) struct scan_result *results, size_t *n_result, size_t n_result_array)
{ {
if (NULL == table_rt) { if (NULL == table_rt) {
return -1; return -1;
@@ -294,7 +303,7 @@ int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct
int i = 0; int i = 0;
int n_hit_result = 0; int n_hit_result = 0;
struct scan_result scan_results[table_rt->rule_num] = {0}; struct scan_result scan_results[n_result_array] = {0};
struct ip_data ip; struct ip_data ip;
ip.type = ip_type_transform(data->ip_type); ip.type = ip_type_transform(data->ip_type);
@@ -304,7 +313,11 @@ int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct
memcpy(ip.ipv6, data->ipv6, sizeof(data->ipv6)); memcpy(ip.ipv6, data->ipv6, sizeof(data->ipv6));
} }
n_hit_result = ip_matcher_match(table_rt->ip_plus_rt.ip_matcher, &ip, scan_results, table_rt->rule_num); n_hit_result = ip_matcher_match(table_rt->ip_plus_rt.ip_matcher, &ip, scan_results, n_result_array);
if (n_hit_result > (int)n_result_array) {
n_hit_result = n_result_array;
}
int index = 0; int index = 0;
/* filter rule_id by port */ /* filter rule_id by port */
for (i = 0; i < n_hit_result; i++) { for (i = 0; i < n_hit_result; i++) {
@@ -312,7 +325,7 @@ int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct
if (data->port < port_range->min_port || data->port > port_range->max_port) { if (data->port < port_range->min_port || data->port > port_range->max_port) {
continue; continue;
} }
results[index++] = scan_results[i].rule_id; results[index++].rule_id = scan_results[i].rule_id;
} }
*n_result = index; *n_result = index;
@@ -473,8 +486,8 @@ void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_sc
{ {
struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1);
ex_container->ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len); ex_container->ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len);
int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema); int set_flag = table_schema_ex_data_schema_flag(table_schema);
size_t cb_count = plugin_table_schema_callback_count(table_schema); size_t cb_count = table_schema_callback_count(table_schema);
/* already set plugin_table_schema's ex_data_schema */ /* already set plugin_table_schema's ex_data_schema */
if (1 == set_flag) { if (1 == set_flag) {
@@ -489,7 +502,7 @@ void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_sc
/* plugin table schema has callback */ /* plugin table schema has callback */
if (cb_count > 0) { if (cb_count > 0) {
plugin_table_schema_all_cb_update(table_schema, row); table_schema_all_cb_update(table_schema, row);
} }
if ((0 == set_flag) && (0 == cb_count)) { if ((0 == set_flag) && (0 == cb_count)) {
@@ -510,7 +523,7 @@ void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct
struct ex_container_ctx *ctx = ex_data_runtime_get_ex_container_ctx(ex_data_rt); struct ex_container_ctx *ctx = ex_data_runtime_get_ex_container_ctx(ex_data_rt);
ctx->custom_data_free = free; ctx->custom_data_free = free;
int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema); int set_flag = table_schema_ex_data_schema_flag(table_schema);
if (1 == set_flag) { if (1 == set_flag) {
if (0 == is_valid) { if (0 == is_valid) {
@@ -576,7 +589,8 @@ void table_runtime_update(struct table_runtime *table_rt, struct table_schema *t
} }
} }
int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread) int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread,
struct log_handle *logger)
{ {
struct expr_runtime *expr_rt = &(table_rt->expr_rt); struct expr_runtime *expr_rt = &(table_rt->expr_rt);
void **ex_data_array = NULL; void **ex_data_array = NULL;
@@ -594,8 +608,12 @@ int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread)
struct adapter_hs *new_adapter_hs = NULL; struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL; struct adapter_hs *old_adapter_hs = NULL;
if (rule_cnt > 0) { if (rule_cnt > 0) {
log_info(logger, MODULE_TABLE_RUNTIME,
"committing %zu expr rules for rebuilding adapter_hs engine", rule_cnt);
new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, nr_worker_thread, rules, rule_cnt); new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, nr_worker_thread, rules, rule_cnt);
if (NULL == new_adapter_hs) { if (NULL == new_adapter_hs) {
log_error(logger, MODULE_TABLE_RUNTIME,
"rebuild adapter_hs engine failed when update %zu expr rules", rule_cnt);
ret = -1; ret = -1;
} }
} }
@@ -615,7 +633,7 @@ int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread)
return ret; return ret;
} }
int ip_plus_runtime_commit(struct table_runtime *table_rt) int ip_plus_runtime_commit(struct table_runtime *table_rt, struct log_handle *logger)
{ {
struct ex_data_container **ex_container = NULL; struct ex_data_container **ex_container = NULL;
struct ip_rule *rules = NULL; struct ip_rule *rules = NULL;
@@ -636,8 +654,12 @@ int ip_plus_runtime_commit(struct table_runtime *table_rt)
size_t mem_used = 0; size_t mem_used = 0;
if (rule_cnt > 0) { if (rule_cnt > 0) {
log_info(logger, MODULE_TABLE_RUNTIME,
"committing %zu ip_plus rules for rebuilding ip_matcher engine", rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) { if (NULL == new_ip_matcher) {
log_error(logger, MODULE_TABLE_RUNTIME,
"rebuild ip_matcher engine failed when update %zu ip_plus rules", rule_cnt);
ret = -1; ret = -1;
} }
} }
@@ -662,7 +684,7 @@ int plugin_runtime_commit(struct table_runtime *table_rt)
return 0; return 0;
} }
int ip_plugin_runtime_commit(struct table_runtime *table_rt) int ip_plugin_runtime_commit(struct table_runtime *table_rt, struct log_handle *logger)
{ {
struct ex_data_container **ex_container = NULL; struct ex_data_container **ex_container = NULL;
struct ip_rule *rules = NULL; struct ip_rule *rules = NULL;
@@ -683,8 +705,12 @@ int ip_plugin_runtime_commit(struct table_runtime *table_rt)
size_t mem_used = 0; size_t mem_used = 0;
if (rule_cnt > 0) { if (rule_cnt > 0) {
log_info(logger, MODULE_TABLE_RUNTIME,
"committing %zu ip_plugin rules for rebuilding ip_matcher engine", rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) { if (NULL == new_ip_matcher) {
log_error(logger, MODULE_TABLE_RUNTIME,
"rebuild ip_matcher engine failed when update %zu ip_plugin rules", rule_cnt);
ret = -1; ret = -1;
} }
} }
@@ -737,20 +763,21 @@ int table_runtime_updating_flag(struct table_runtime *table_rt)
return updating_flag; return updating_flag;
} }
void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread) void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread,
struct log_handle *logger)
{ {
switch (table_rt->table_type) { switch (table_rt->table_type) {
case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR:
expr_runtime_commit(table_rt, nr_worker_thread); expr_runtime_commit(table_rt, nr_worker_thread, logger);
break; break;
case TABLE_TYPE_IP_PLUS: case TABLE_TYPE_IP_PLUS:
ip_plus_runtime_commit(table_rt); ip_plus_runtime_commit(table_rt, logger);
break; break;
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
plugin_runtime_commit(table_rt); plugin_runtime_commit(table_rt);
break; break;
case TABLE_TYPE_IP_PLUGIN: case TABLE_TYPE_IP_PLUGIN:
ip_plugin_runtime_commit(table_rt); ip_plugin_runtime_commit(table_rt, logger);
break; break;
default: default:
break; break;
@@ -813,9 +840,9 @@ struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table
} }
void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema, void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema,
struct log_handle *logger) int nr_worker_thread, struct log_handle *logger)
{ {
struct ex_data_schema *ex_data_schema = plugin_table_schema_get_ex_data_schema(table_schema); struct ex_data_schema *ex_data_schema = table_schema_get_ex_data_schema(table_schema);
struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt); struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt);
ex_data_runtime_set_schema(ex_data_rt, ex_data_schema); ex_data_runtime_set_schema(ex_data_rt, ex_data_schema);
struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1); struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1);
@@ -840,5 +867,5 @@ void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct
} }
ex_data_runtime_clear_row_cache(ex_data_rt); ex_data_runtime_clear_row_cache(ex_data_rt);
table_runtime_commit(table_rt, 0); table_runtime_commit(table_rt, nr_worker_thread, logger);
} }

271
src/maat_table_schema.cpp
View File

@@ -29,6 +29,20 @@
#define MAX_FOREIGN_CLMN_NUM 8 #define MAX_FOREIGN_CLMN_NUM 8
#define MAX_CONJUNCTION_TABLE_NUM 8 #define MAX_CONJUNCTION_TABLE_NUM 8
#define MAX_COMPILE_EX_DATA_NUM 2
enum user_region_encode {
USER_REGION_ENCODE_NONE=0,
USER_REGION_ENCODE_ESCAPE,
USER_REGION_ENCODE_BASE64
};
struct compile_table_schema {
enum user_region_encode user_region_encoding;
size_t n_ex_schema;
struct compile_ex_data_schema ex_schema[MAX_COMPILE_EX_DATA_NUM];
};
struct expr_table_schema { struct expr_table_schema {
int item_id_column; int item_id_column;
int group_id_column; int group_id_column;
@@ -41,23 +55,6 @@ struct expr_table_schema {
enum hs_scan_mode scan_mode; /* adapter_hs scan mode */ enum hs_scan_mode scan_mode; /* adapter_hs scan mode */
}; };
struct ip_table_schema {
int item_id_column;
int group_id_column;
int addr_type_column;
int sip_column;
int sip_mask_column;
int sport_column;
int sport_mask_column;
int dip_column;
int dip_mask_column;
int dport_column;
int dport_mask_column;
int proto_column;
int direction_column;
int is_valid_column;
};
struct ip_plus_table_schema { struct ip_plus_table_schema {
int item_id_column; int item_id_column;
int group_id_column; int group_id_column;
@@ -109,7 +106,7 @@ struct virtual_table_schema {
struct composition_table_schema struct composition_table_schema
{ {
int component_table_id[COMPOSITION_TYPE_MAX]; int component_table_id[COMPONENT_TABLE_TYPE_MAX];
}; };
struct table_schema { struct table_schema {
@@ -119,8 +116,8 @@ struct table_schema {
char table_name[MAX_CONJUNCTION_TABLE_NUM][NAME_MAX]; char table_name[MAX_CONJUNCTION_TABLE_NUM][NAME_MAX];
enum table_type table_type; enum table_type table_type;
union { union {
struct compile_table_schema compile;
struct expr_table_schema expr; struct expr_table_schema expr;
struct ip_table_schema ip;
struct ip_plus_table_schema ip_plus; struct ip_plus_table_schema ip_plus;
struct plugin_table_schema plugin; struct plugin_table_schema plugin;
struct ip_plugin_table_schema ip_plugin; struct ip_plugin_table_schema ip_plugin;
@@ -247,127 +244,7 @@ int read_expr_table_schema(cJSON *root, struct table_schema *ptable,
return 0; return 0;
} }
#if 0
int read_ip_table_schema(cJSON *root, struct table_schema *ptable, struct log_handle *logger)
{
size_t read_cnt = 0;
cJSON *json = NULL;
json = cJSON_GetObjectItem(root, "table_id");
if (json != NULL && json->type == cJSON_Number) {
ptable->table_id = json->valueint;
read_cnt++;
}
json = cJSON_GetObjectItem(root, "table_name");
if (json != NULL && json->type == cJSON_String) {
if (strlen(json->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE_SCHEMA, "ip table name %s length too long",
json->valuestring);
return -1;
}
memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring));
read_cnt++;
}
json = cJSON_GetObjectItem(root, "item_id");
if (json != NULL && json->type == cJSON_Number) {
ptable->ip.item_id_column = json->valueint;
read_cnt++;
}
json = cJSON_GetObjectItem(root, "group_id");
if (json != NULL && json->type == cJSON_Number) {
ptable->ip.group_id_column = json->valueint;
read_cnt++;
}
json = cJSON_GetObjectItem(root, "custom");
if (NULL == json || json->type != cJSON_Object) {
log_error(logger, MODULE_TABLE_SCHEMA, "ip table %s has no custom column",
ptable->table_name[0]);
return -1;
}
cJSON *item = cJSON_GetObjectItem(json, "addr_type");
if (item != NULL && item->type == cJSON_Number) {
ptable->ip.addr_type_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "sip");
if (item != NULL && item->type == cJSON_Number) {
ptable->ip.sip_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "sip_mask");
if (item != NULL && item->type == cJSON_Number) {
ptable->ip.sip_mask_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "sport");
if (item != NULL && item->type == cJSON_Number) {
ptable->ip.sport_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "sport_mask");
if (item != NULL && item->type == cJSON_Number) {
ptable->ip.sport_mask_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "dip");
if (item != NULL && item->type == cJSON_Number) {
ptable->ip.dip_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "dip_mask");
if (item != NULL && item->type == cJSON_Number) {
ptable->ip.dip_mask_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "dport");
if (item != NULL && item->type == cJSON_Number) {
ptable->ip.dport_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "dport_mask");
if (item != NULL && item->type == cJSON_Number) {
ptable->ip.dport_mask_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "proto");
if (item != NULL && item->type == cJSON_Number) {
ptable->ip.proto_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "direction");
if (item != NULL && item->type == cJSON_Number) {
ptable->ip.direction_column = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "is_valid");
if (item != NULL && item->type == cJSON_Number) {
ptable->ip.is_valid_column = item->valueint;
read_cnt++;
}
if (read_cnt < 16) {
return -1;
}
return 0;
}
#endif
int read_ip_plus_table_schema(cJSON *root, struct table_schema *ptable, int read_ip_plus_table_schema(cJSON *root, struct table_schema *ptable,
struct log_handle *logger) struct log_handle *logger)
{ {
@@ -606,7 +483,8 @@ int read_composition_table_schema(cJSON *root, struct table_schema *ptable,
cJSON *item = cJSON_GetObjectItem(json, "source"); cJSON *item = cJSON_GetObjectItem(json, "source");
if (item != NULL && item->type == cJSON_String) { if (item != NULL && item->type == cJSON_String) {
/* physical table should already exist */ /* physical table should already exist */
ret = maat_kv_read(tablename2id_map, item->valuestring, &composition_schema->component_table_id[COMPOSITION_TYPE_SIP]); ret = maat_kv_read(tablename2id_map, item->valuestring,
&composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SIP]);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_TABLE_SCHEMA, log_error(logger, MODULE_TABLE_SCHEMA,
"Child table %s of table %s (id=%d) are not defined", "Child table %s of table %s (id=%d) are not defined",
@@ -618,7 +496,8 @@ int read_composition_table_schema(cJSON *root, struct table_schema *ptable,
item = cJSON_GetObjectItem(json, "dest"); item = cJSON_GetObjectItem(json, "dest");
if (item != NULL && item->type == cJSON_String) { if (item != NULL && item->type == cJSON_String) {
/* physical table should already exist */ /* physical table should already exist */
ret = maat_kv_read(tablename2id_map, item->valuestring, &composition_schema->component_table_id[COMPOSITION_TYPE_DIP]); ret = maat_kv_read(tablename2id_map, item->valuestring,
&composition_schema->component_table_id[COMPONENT_TABLE_TYPE_DIP]);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_TABLE_SCHEMA, log_error(logger, MODULE_TABLE_SCHEMA,
"Child table %s of table %s (id=%d) are not defined", "Child table %s of table %s (id=%d) are not defined",
@@ -630,7 +509,8 @@ int read_composition_table_schema(cJSON *root, struct table_schema *ptable,
item = cJSON_GetObjectItem(json, "session"); item = cJSON_GetObjectItem(json, "session");
if (item != NULL && item->type == cJSON_String) { if (item != NULL && item->type == cJSON_String) {
/* physical table should already exist */ /* physical table should already exist */
ret = maat_kv_read(tablename2id_map, item->valuestring, &composition_schema->component_table_id[COMPOSITION_TYPE_SESSION]); ret = maat_kv_read(tablename2id_map, item->valuestring,
&composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SESSION]);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_TABLE_SCHEMA, log_error(logger, MODULE_TABLE_SCHEMA,
"Child table %s of table %s (id=%d) are not defined", "Child table %s of table %s (id=%d) are not defined",
@@ -833,13 +713,6 @@ int table_schema_populate(cJSON *json, struct table_schema **table_array,
return -1; return -1;
} }
break; break;
/* TODO: delete
case TABLE_TYPE_IP:
ret = read_ip_table_schema(json, ptable);
if (ret < 0) {
return -1;
}
break; */
case TABLE_TYPE_IP_PLUS: case TABLE_TYPE_IP_PLUS:
ret = read_ip_plus_table_schema(json, ptable, logger); ret = read_ip_plus_table_schema(json, ptable, logger);
if (ret < 0) { if (ret < 0) {
@@ -912,7 +785,6 @@ table_schema_manager_create(const char *table_info_path, struct log_handle *logg
/* register table type reserved word */ /* register table type reserved word */
maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR);
maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS);
//maat_kv_register(reserved_word_map, "ip", TABLE_TYPE_IP);
maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS);
maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN);
maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN);
@@ -1023,7 +895,7 @@ int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_
} }
int table_schema_manager_get_child_table_id(struct table_schema_manager *table_schema_mgr, int parent_table_id, int table_schema_manager_get_child_table_id(struct table_schema_manager *table_schema_mgr, int parent_table_id,
enum table_composition_type type) enum component_table_type type)
{ {
struct table_schema* table_schema = table_schema_get(table_schema_mgr, parent_table_id); struct table_schema* table_schema = table_schema_get(table_schema_mgr, parent_table_id);
if (table_schema->table_type != TABLE_TYPE_COMPOSITION) { if (table_schema->table_type != TABLE_TYPE_COMPOSITION) {
@@ -1036,11 +908,11 @@ int table_schema_manager_get_child_table_id(struct table_schema_manager *table_s
enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int id) enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int id)
{ {
if (NULL == table_schema_mgr) { if (NULL == table_schema_mgr) {
return TABLE_TYPE_MAX; return TABLE_TYPE_INVALID;
} }
if (table_schema_mgr->schema_table[id] == NULL) { if (NULL == table_schema_mgr->schema_table[id]) {
return TABLE_TYPE_MAX; return TABLE_TYPE_INVALID;
} }
return table_schema_mgr->schema_table[id]->table_type; return table_schema_mgr->schema_table[id]->table_type;
@@ -1205,7 +1077,7 @@ struct table_schema *table_schema_get_by_scan_type(struct table_schema_manager *
enum table_type table_schema_get_table_type(struct table_schema *table_schema) enum table_type table_schema_get_table_type(struct table_schema *table_schema)
{ {
if (NULL == table_schema) { if (NULL == table_schema) {
return TABLE_TYPE_MAX; return TABLE_TYPE_INVALID;
} }
return table_schema->table_type; return table_schema->table_type;
@@ -1239,7 +1111,6 @@ enum scan_type table_schema_get_scan_type(struct table_schema *table_schema)
case TABLE_TYPE_INTERVAL_PLUS: case TABLE_TYPE_INTERVAL_PLUS:
ret = SCAN_TYPE_INTERVAL; ret = SCAN_TYPE_INTERVAL;
break; break;
//case TABLE_TYPE_IP:
case TABLE_TYPE_IP_PLUS: case TABLE_TYPE_IP_PLUS:
case TABLE_TYPE_COMPOSITION: case TABLE_TYPE_COMPOSITION:
ret = SCAN_TYPE_IP; ret = SCAN_TYPE_IP;
@@ -1530,7 +1401,7 @@ int populate_ip_plus_table_item(const char *line, struct ip_plus_table_schema *i
direction = ip_plus_item->ipv4.direction; direction = ip_plus_item->ipv4.direction;
} else { } else {
//ipv6 //ipv6
ret = ip_format2range(ip_plus_item->addr_type, IP_FORMAT_MASK, sip1_str, sip2_str, ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str,
ip_plus_item->ipv6.min_sip, ip_plus_item->ipv6.max_sip); ip_plus_item->ipv6.min_sip, ip_plus_item->ipv6.max_sip);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
@@ -1747,7 +1618,7 @@ enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_sch
return table_schema->expr.scan_mode; return table_schema->expr.scan_mode;
} }
int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema, int table_schema_set_ex_data_schema(struct table_schema *table_schema,
maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func, maat_plugin_ex_dup_func_t *dup_func,
@@ -1760,7 +1631,7 @@ int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema,
return -1; return -1;
} }
struct ex_data_schema *ex_schema = plugin_table_schema_get_ex_data_schema(table_schema); struct ex_data_schema *ex_schema = table_schema_get_ex_data_schema(table_schema);
if (NULL == ex_schema) { if (NULL == ex_schema) {
log_error(logger, MODULE_TABLE_SCHEMA, log_error(logger, MODULE_TABLE_SCHEMA,
"Error: %s, target table is not a valid plugin table", __FUNCTION__); "Error: %s, target table is not a valid plugin table", __FUNCTION__);
@@ -1784,7 +1655,7 @@ int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema,
return 0; return 0;
} }
struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schema *table_schema) struct ex_data_schema *table_schema_get_ex_data_schema(struct table_schema *table_schema)
{ {
if (NULL == table_schema) { if (NULL == table_schema) {
return NULL; return NULL;
@@ -1806,7 +1677,77 @@ struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schem
return ex_schema; return ex_schema;
} }
int plugin_table_schema_ex_data_schema_flag(struct table_schema *table_schema) int table_schema_set_compile_rule_ex_data_schema(struct table_schema *table_schema,
maat_rule_ex_new_func_t *new_func,
maat_rule_ex_free_func_t *free_func,
maat_rule_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger)
{
if (NULL == table_schema || NULL == new_func || NULL == free_func || NULL == dup_func) {
assert(0);
log_error(logger, MODULE_TABLE_SCHEMA, "%s failed: invalid parameter", __FUNCTION__);
return -1;
}
if (table_schema->table_type != TABLE_TYPE_COMPILE) {
log_error(logger, MODULE_TABLE_SCHEMA,
"table type must TABLE_TYPE_COMPILE when set compile rule ex_data schema");
return -1;
}
struct compile_table_schema* compile_schema = &(table_schema->compile);
if (compile_schema->n_ex_schema == MAX_COMPILE_EX_DATA_NUM) {
log_error(logger, MODULE_TABLE_SCHEMA,
"compile ex schema num reach maxium, can't set anymore");
return -1;
}
int idx = compile_schema->n_ex_schema;
compile_schema->ex_schema[idx].idx = idx;
compile_schema->ex_schema[idx].table_id = table_schema->table_id;
compile_schema->ex_schema[idx].argl = argl;
compile_schema->ex_schema[idx].argp = argp;
compile_schema->ex_schema[idx].new_func = new_func;
compile_schema->ex_schema[idx].free_func = free_func;
compile_schema->ex_schema[idx].dup_func = dup_func;
compile_schema->n_ex_schema++;
return idx;
}
struct compile_ex_data_schema *
table_schema_get_compile_rule_ex_data_schema(struct table_schema *table_schema, size_t idx)
{
if (NULL == table_schema) {
return NULL;
}
if (table_schema->table_type != TABLE_TYPE_COMPILE) {
return NULL;
}
if (idx < table_schema->compile.n_ex_schema) {
return (table_schema->compile.ex_schema + idx);
}
return NULL;
}
size_t table_schema_compile_rule_ex_data_schema_count(struct table_schema *table_schema)
{
if (NULL == table_schema) {
return 0;
}
if (table_schema->table_type != TABLE_TYPE_COMPILE) {
return 0;
}
return table_schema->compile.n_ex_schema;
}
int table_schema_ex_data_schema_flag(struct table_schema *table_schema)
{ {
if (NULL == table_schema) { if (NULL == table_schema) {
return -1; return -1;
@@ -1828,7 +1769,7 @@ int plugin_table_schema_ex_data_schema_flag(struct table_schema *table_schema)
return ex_schema->set_flag; return ex_schema->set_flag;
} }
int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_mgr, int table_id, int table_schema_add_callback(struct table_schema_manager* table_schema_mgr, int table_id,
maat_start_callback_t *start, maat_start_callback_t *start,
maat_update_callback_t *update, maat_update_callback_t *update,
maat_finish_callback_t *finish, maat_finish_callback_t *finish,
@@ -1872,7 +1813,7 @@ int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_m
return 0; return 0;
} }
size_t plugin_table_schema_callback_count(struct table_schema *table_schema) size_t table_schema_callback_count(struct table_schema *table_schema)
{ {
if (NULL == table_schema) { if (NULL == table_schema) {
return 0; return 0;
@@ -1885,7 +1826,7 @@ size_t plugin_table_schema_callback_count(struct table_schema *table_schema)
} }
} }
void plugin_table_schema_all_cb_update(struct table_schema* table_schema, const char *row) void table_schema_all_cb_update(struct table_schema* table_schema, const char *row)
{ {
if (NULL == table_schema || NULL == row) { if (NULL == table_schema || NULL == row) {
return; return;
@@ -1911,7 +1852,7 @@ void plugin_table_schema_all_cb_update(struct table_schema* table_schema, const
} }
} }
int plugin_table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns) int table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns)
{ {
if (NULL == table_schema) { if (NULL == table_schema) {
return -1; return -1;

414
test/maat_framework_gtest.cpp
View File

@@ -1,5 +1,6 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include "utils.h"
#include "maat/maat.h" #include "maat/maat.h"
#include "maat_rule.h" #include "maat_rule.h"
#include "maat_utils.h" #include "maat_utils.h"
@@ -7,175 +8,26 @@
#include "maat_table_runtime.h" #include "maat_table_runtime.h"
#include "maat_command.h" #include "maat_command.h"
#include "IPMatcher.h" #include "IPMatcher.h"
#include "json2iris.h"
#include "maat_config_monitor.h"
struct maat *g_maat_instance = NULL; struct maat *g_maat_instance = NULL;
const char *table_info_path = "./table_info.conf"; const char *table_info_path = "./table_info.conf";
const char *rule_full_path = "./rule/full/index";
const char *rule_inc_path = "./rule/inc/index";
const char *json_path="./maat_json.json"; const char *json_path="./maat_json.json";
const char *iris_file = "./HTTP_URL.000001"; const char *json_filename = "maat_json.json";
#if 0
TEST(maat_scan_string, hit_one_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
char data[64] = "";
int result_array[5] = {0};
size_t n_result_array = 0;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, -1);
char data1[64] = "baidu.com";
ret = maat_scan_string(g_maat_instance, table_id, 0, data1, strlen(data1), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 0);
char data2[64] = "hello";
ret = maat_scan_string(g_maat_instance, table_id, 0, data2, strlen(data2), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 1);
EXPECT_EQ(result_array[0], 101);
/*
memset(result_array, 0, sizeof(result_array));
char data3[64] = "maat";
ret = maat_scan_string(g_maat_instance, table_id, 0, data3, strlen(data3), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 1);
EXPECT_EQ(result_array[0], 102); */
/*
memset(result_array, 0, sizeof(result_array));
char data4[64] = "world";
ret = maat_scan_string(g_maat_instance, table_id, 0, data4, strlen(data4), result_array, &n_result_array, NULL);
EXPECT_EQ(n_result_array, 1);
EXPECT_EQ(result_array[0], 103); */
}
TEST(maat_scan_string, hit_two_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
char data[64] = "hello maat";
int result_array[5] = {0};
size_t n_result_array = 0;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 2);
EXPECT_EQ(result_array[0], 102);
EXPECT_EQ(result_array[1], 101);
memset(result_array, 0, sizeof(result_array));
char data1[64] = "maat world";
ret = maat_scan_string(g_maat_instance, table_id, 0, data1, strlen(data1), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 2);
EXPECT_EQ(result_array[0], 103);
EXPECT_EQ(result_array[1], 102);
}
TEST(maat_scan_string, hit_three_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
char data[64] = "hello world";
int result_array[5] = {0};
size_t n_result_array = 0;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 3);
EXPECT_EQ(result_array[0], 104);
EXPECT_EQ(result_array[1], 103);
EXPECT_EQ(result_array[2], 101);
}
TEST(maat_scan_string, hit_four_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
char data[64] = "9hello world";
int result_array[5] = {0};
size_t n_result_array = 0;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 4);
EXPECT_EQ(result_array[0], 105);
EXPECT_EQ(result_array[1], 104);
EXPECT_EQ(result_array[2], 103);
EXPECT_EQ(result_array[3], 101);
}
TEST(maat_scan_string, hit_five_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
char data[64] = "9hello 8world";
int result_array[5] = {0};
size_t n_result_array = 0;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 5);
EXPECT_EQ(result_array[0], 106);
EXPECT_EQ(result_array[1], 105);
EXPECT_EQ(result_array[2], 104);
EXPECT_EQ(result_array[3], 103);
EXPECT_EQ(result_array[4], 101);
}
TEST(maat_scan_string, config_dynamic_update) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
char data[128] = "www.baidu.com";
int result_array[5] = {0};
size_t n_result_array = 0;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 0);
/* generate inc update config */
system_cmd_mkdir("./rule/inc/index");
char new_rule[128] = "207 7 baidu 0 0 0 1";
FILE *fp = fopen("./rule/inc/HTTP_URL.000002", "w+");
EXPECT_NE(fp, nullptr);
fprintf(fp, "%s\n", "0000000001");
fprintf(fp, "%s\n", new_rule);
fclose(fp);
char new_rule_idx[128] = "HTTP_URL 1 ./rule/inc/HTTP_URL.000002";
fp = fopen("./rule/inc/index/inc_config_index.000002", "w+");
EXPECT_NE(fp, nullptr);
fprintf(fp, "%s\n", new_rule_idx);
fclose(fp);
/* if updated rule already valid */
memset(result_array, 0, sizeof(result_array));
sleep(2);
ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 1);
EXPECT_EQ(result_array[0], 207);
/* if the old rules are still valid */
memset(result_array, 0, sizeof(result_array));
char data1[64] = "9hello 8world";
ret = maat_scan_string(g_maat_instance, table_id, 0, data1, strlen(data1), result_array, &n_result_array, NULL);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 5);
EXPECT_EQ(result_array[0], 106);
EXPECT_EQ(result_array[1], 105);
EXPECT_EQ(result_array[2], 104);
EXPECT_EQ(result_array[3], 103);
EXPECT_EQ(result_array[4], 101);
}
#endif
TEST(maat_scan_string, hit_one_expr) { TEST(maat_scan_string, hit_one_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
char data[128] = "i.ytimg.com"; char data[128] = "i.ytimg.com";
int result_array[5] = {0}; int results[5] = {0};
size_t n_result_array = 0; size_t n_result = 0;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 1); EXPECT_EQ(n_result, 1);
EXPECT_EQ(result_array[0], 30); EXPECT_EQ(results[0], 30);
} }
TEST(maat_scan_string, hit_two_expr) { TEST(maat_scan_string, hit_two_expr) {
@@ -183,13 +35,14 @@ TEST(maat_scan_string, hit_two_expr) {
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
char data[128] = "should hit aaa bbb"; char data[128] = "should hit aaa bbb";
int result_array[5] = {0}; int results[5] = {0};
size_t n_result_array = 0; size_t n_result = 0;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 2); EXPECT_EQ(n_result, 2);
EXPECT_EQ(result_array[0], 28); EXPECT_EQ(results[0], 28);
EXPECT_EQ(result_array[1], 27); EXPECT_EQ(results[1], 27);
} }
TEST(maat_scan_string, hit_three_expr) { TEST(maat_scan_string, hit_three_expr) {
@@ -197,14 +50,15 @@ TEST(maat_scan_string, hit_three_expr) {
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
char data[128] = "should hit aaa bbb C#中国"; char data[128] = "should hit aaa bbb C#中国";
int result_array[5] = {0}; int results[5] = {0};
size_t n_result_array = 0; size_t n_result = 0;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 3); EXPECT_EQ(n_result, 3);
EXPECT_EQ(result_array[0], 28); EXPECT_EQ(results[0], 28);
EXPECT_EQ(result_array[1], 27); EXPECT_EQ(results[1], 27);
EXPECT_EQ(result_array[2], 18); EXPECT_EQ(results[2], 18);
} }
TEST(maat_scan_ipv4, hit_ip_and_port) { TEST(maat_scan_ipv4, hit_ip_and_port) {
@@ -220,14 +74,16 @@ TEST(maat_scan_ipv4, hit_ip_and_port) {
int results[3] = {-1}; int results[3] = {-1};
size_t n_result = 0; size_t n_result = 0;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1); EXPECT_EQ(n_result, 1);
EXPECT_EQ(results[0], 7); EXPECT_EQ(results[0], 7);
port = 20001; port = 20001;
addr.ipv4.sport = htons(port); addr.ipv4.sport = htons(port);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0); EXPECT_EQ(n_result, 0);
} }
@@ -245,7 +101,8 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) {
int results[3] = {-1}; int results[3] = {-1};
size_t n_result = 0; size_t n_result = 0;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1); EXPECT_EQ(n_result, 1);
EXPECT_EQ(results[0], 4); EXPECT_EQ(results[0], 4);
@@ -254,7 +111,8 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) {
addr.ipv4.sport = htons(port); addr.ipv4.sport = htons(port);
memset(results, 0, sizeof(results)); memset(results, 0, sizeof(results));
n_result = 0; n_result = 0;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1); EXPECT_EQ(n_result, 1);
EXPECT_EQ(results[0], 4); EXPECT_EQ(results[0], 4);
@@ -263,7 +121,8 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) {
addr.ipv4.sport = htons(port); addr.ipv4.sport = htons(port);
memset(results, 0, sizeof(results)); memset(results, 0, sizeof(results));
n_result = 0; n_result = 0;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0); EXPECT_EQ(n_result, 0);
} }
@@ -284,7 +143,8 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) {
int results[3] = {-1}; int results[3] = {-1};
size_t n_result = 0; size_t n_result = 0;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1); EXPECT_EQ(n_result, 1);
EXPECT_EQ(results[0], 8); EXPECT_EQ(results[0], 8);
@@ -292,7 +152,8 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) {
ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip); ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
addr.ipv4.sport = htons(port2); addr.ipv4.sport = htons(port2);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1); EXPECT_EQ(n_result, 1);
EXPECT_EQ(results[0], 8); EXPECT_EQ(results[0], 8);
@@ -300,7 +161,8 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) {
ret = inet_pton(AF_INET, ip_str3, &addr.ipv4.sip); ret = inet_pton(AF_INET, ip_str3, &addr.ipv4.sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
addr.ipv4.sport = htons(port2); addr.ipv4.sport = htons(port2);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0); EXPECT_EQ(n_result, 0);
} }
@@ -319,14 +181,16 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) {
int results[3] = {-1}; int results[3] = {-1};
size_t n_result = 0; size_t n_result = 0;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1); EXPECT_EQ(n_result, 1);
EXPECT_EQ(results[0], 50); EXPECT_EQ(results[0], 50);
ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip); ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0); EXPECT_EQ(n_result, 0);
} }
@@ -344,7 +208,8 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) {
int results[3] = {-1}; int results[3] = {-1};
size_t n_result = 0; size_t n_result = 0;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 2); EXPECT_EQ(n_result, 2);
EXPECT_EQ(results[0], 63); EXPECT_EQ(results[0], 63);
@@ -352,7 +217,8 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) {
port = 442; port = 442;
addr.ipv4.sport = htons(port); addr.ipv4.sport = htons(port);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0); EXPECT_EQ(n_result, 0);
} }
@@ -370,31 +236,205 @@ TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) {
int results[3] = {-1}; int results[3] = {-1};
size_t n_result = 0; size_t n_result = 0;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1); EXPECT_EQ(n_result, 1);
EXPECT_EQ(results[0], 47); EXPECT_EQ(results[0], 47);
port = 442; port = 442;
addr.ipv6.sport = htons(port); addr.ipv6.sport = htons(port);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0); EXPECT_EQ(n_result, 0);
} }
TEST(maat_scan_string, dynamic_config) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
char data[128] = "hello world";
int results[5] = {0};
size_t n_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
const char *table_name = "HTTP_URL";
const char *table_line = "9999\t8888\thello world\t0\t0\t0\t1\t";
struct maat_cmd_line line_rule;
line_rule.rule_id = 101;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
ret = maat_cmd_set_line(g_maat_instance, &line_rule);
EXPECT_EQ(ret, 1);
sleep(2);
state = NULL;
ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(results[0], 9999);
}
TEST(maat_scan_ip, dynamic_config) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG");
char ip_str[32] = "10.0.6.201";
uint16_t port = 443;
struct addr_4tuple addr;
addr.type = IP_TYPE_V4;
int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip);
EXPECT_EQ(ret, 1);
addr.ipv4.sport = htons(port);
int results[3] = {-1};
size_t n_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
const char *table_name = "IP_PLUS_CONFIG";
const char *table_line = "9998\t8887\t4\trange\t10.0.6.201\t255.255.0.0\trange\t0\t65535\t6\t0\t1";
struct maat_cmd_line line_rule;
line_rule.rule_id = 101;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
ret = maat_cmd_set_line(g_maat_instance, &line_rule);
EXPECT_EQ(ret, 0);
sleep(2);
state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(results[0], 9998);
}
int count_line_num_cb(const char *table_name, const char *line, void *u_para)
{
(*((unsigned int *)u_para))++;
return 0;
}
int line_idx = 0;
long long absolute_expire_time=0;
int make_serial_rule(const char *table_name, const char *line, void *u_para)
{
struct serial_rule *s_rule=(struct serial_rule *)u_para;
int rule_id = 0;
char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1);
while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') {
buff[strlen(buff) - 1] = '\0';
}
int j = 0;
char *str1 = NULL;
char *token = NULL;
char *saveptr1 = NULL;
for (j = 0,str1 = buff; ; j++, str1 = NULL) {
token = strtok_r(str1, "\t ", &saveptr1);
if (token == NULL)
break;
if (j == 0) {
sscanf(token,"%d", &rule_id);
}
}
memcpy(buff, line, strlen(line)+1);
while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t') {
buff[strlen(buff)-1]='\0';
}
maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, buff, absolute_expire_time);
line_idx++;
FREE(str1);
return 0;
}
int main(int argc, char ** argv) int main(int argc, char ** argv)
{ {
int ret=0; int ret=0;
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
/*
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_json_file(opts, json_path); maat_options_set_json_file(opts, json_path);
g_maat_instance = maat_new(opts, table_info_path); g_maat_instance = maat_new(opts, table_info_path);
EXPECT_NE(g_maat_instance, nullptr); EXPECT_NE(g_maat_instance, nullptr);*/
char json_iris_path[128] = {0};
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
struct log_handle *logger = log_handle_create("./tmp.log", 0);
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename);
redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, logger);
EXPECT_NE(c, nullptr);
redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb");
EXPECT_NE(reply, nullptr);
if ((access(json_iris_path, F_OK)) < 0) {
char tmp_iris_path[128] = {0};
char *json_buff = NULL;
size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz);
EXPECT_NE(ret, -1);
ret = json2iris(json_buff, json_filename, c, tmp_iris_path,
sizeof(tmp_iris_path), NULL, NULL, logger);
EXPECT_NE(ret, -1);
}
size_t total_line_cnt = 0;
char tmp_iris_full_idx_path[128] = {0};
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
long long server_time = maat_cmd_redis_server_time_s(c);
EXPECT_NE(server_time, -1);
absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger);
int success_cnt = 0;
do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, logger);
} while (success_cnt < 0);
EXPECT_EQ(success_cnt, (int)total_line_cnt);
for (size_t i = 0; i < total_line_cnt; i++) {
maat_cmd_clear_rule_cache(s_rule + i);
}
FREE(s_rule);
redisFree(c);
struct maat_options *opts = maat_options_new();
maat_options_set_redis_ip(opts, redis_ip);
maat_options_set_redis_port(opts, redis_port);
maat_options_set_logger(opts, logger);
g_maat_instance = maat_new(opts, table_info_path);
ret=RUN_ALL_TESTS(); ret=RUN_ALL_TESTS();
log_handle_destroy(g_maat_instance->logger);
maat_free(g_maat_instance); maat_free(g_maat_instance);
return ret; return ret;

6
test/maat_input_mode_gtest.cpp
View File

@@ -57,8 +57,7 @@ TEST(iris_mode, maat_scan_string) {
int ret = load_file_to_memory(json_filename, (unsigned char**)&json_buff, &json_buff_sz); int ret = load_file_to_memory(json_filename, (unsigned char**)&json_buff, &json_buff_sz);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, NULL, ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, NULL);
tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, NULL);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
} }
@@ -158,8 +157,7 @@ TEST(redis_mode, maat_scan_string) {
int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz); int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, c, tmp_iris_path, ret = json2iris(json_buff, json_filename, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, logger);
sizeof(tmp_iris_path), NULL, NULL, logger);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
} }

102
test/maat_json.json
View File

@@ -37,18 +37,22 @@
"group_name": "financial-department-ip", "group_name": "financial-department-ip",
"regions": [ "regions": [
{ {
"table_name": "IP_CONFIG", "table_name": "IP_PLUS_CONFIG",
"table_type": "ip", "table_type": "ip_plus",
"table_content": { "table_content": {
"addr_type": "ipv4", "addr_type": "ipv4",
"src_ip": "192.168.40.88", "saddr_format": "mask",
"mask_src_ip": "255.255.255.255", "src_ip1": "192.168.40.88",
"src_port": "0", "src_ip2": "255.255.255.255",
"mask_src_port": "65535", "sport_format": "mask",
"dst_ip": "0.0.0.0", "src_port1": "0",
"mask_dst_ip": "255.255.255.255", "src_port2": "65535",
"dst_port": "0", "daddr_format": "mask",
"mask_dst_port": "65535", "dst_ip1": "0.0.0.0",
"dst_ip2": "255.255.255.255",
"dport_format": "mask",
"dst_port1": "0",
"dst_port2": "65535",
"protocol": 6, "protocol": 6,
"direction": "double" "direction": "double"
} }
@@ -192,35 +196,43 @@
"group_name": "123_IP_group", "group_name": "123_IP_group",
"regions": [ "regions": [
{ {
"table_name": "IP_CONFIG", "table_name": "IP_PLUS_CONFIG",
"table_type": "ip", "table_type": "ip_plus",
"table_content": { "table_content": {
"addr_type": "ipv4", "addr_type": "ipv4",
"src_ip": "10.0.6.201", "saddr_format": "mask",
"mask_src_ip": "255.255.0.0", "src_ip1": "10.0.6.201",
"src_port": "0", "src_ip2": "255.255.0.0",
"mask_src_port": "65535", "sport_format": "mask",
"dst_ip": "0.0.0.0", "src_port1": "0",
"mask_dst_ip": "255.255.255.255", "src_port2": "65535",
"dst_port": "0", "daddr_format":"mask",
"mask_dst_port": "65535", "dst_ip1": "0.0.0.0",
"dst_ip2": "255.255.255.255",
"dport_format":"mask",
"dst_port1": "0",
"dst_port2": "65535",
"protocol": 6, "protocol": 6,
"direction": "double" "direction": "double"
} }
}, },
{ {
"table_name": "IP_CONFIG", "table_name": "IP_PLUS_CONFIG",
"table_type": "ip", "table_type": "ip_plus",
"table_content": { "table_content": {
"addr_type": "ipv6", "addr_type": "ipv6",
"src_ip": "2001:da8:205:1::101", "saddr_format":"mask",
"mask_src_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:0000", "src_ip1": "2001:da8:205:1::101",
"src_port": "0", "src_ip2": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:0000",
"mask_src_port": "65535", "sport_format":"mask",
"dst_ip": "0::0", "src_port1": "0",
"mask_dst_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "src_port2": "65535",
"dst_port": "0", "daddr_format":"mask",
"mask_dst_port": "65535", "dst_ip1": "0::0",
"dst_ip2": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff",
"dport_format":"mask",
"dst_port1": "0",
"dst_port2": "65535",
"protocol": 6, "protocol": 6,
"direction": "double" "direction": "double"
} }
@@ -941,6 +953,7 @@
} }
] ]
}, },
{ {
"compile_id": 150, "compile_id": 150,
"service": 0, "service": 0,
@@ -1999,7 +2012,6 @@
} }
] ]
}, },
{ {
"compile_id": 184, "compile_id": 184,
"user_region": "APP_ID=6006740;Liumengyan-Bugreport-20210515", "user_region": "APP_ID=6006740;Liumengyan-Bugreport-20210515",
@@ -2015,20 +2027,24 @@
"regions": [ "regions": [
{ {
"table_name": "IP_CONFIG", "table_name": "IP_PLUS_CONFIG",
"table_type": "ip", "table_type": "ip_plus",
"table_content": { "table_content": {
"protocol": 0,
"addr_type": "ipv6", "addr_type": "ipv6",
"direction": "double", "saddr_format":"mask",
"src_ip": "::", "src_ip1": "::",
"dst_ip": "2620:100:3000::", "src_ip2": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff",
"src_port": "0", "sport_format":"mask",
"dst_port": "0", "src_port1": "0",
"mask_src_port": "65535", "src_port2": "65535",
"mask_src_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "daddr_format":"mask",
"mask_dst_port": "65535", "dst_ip1": "2620:100:3000::",
"mask_dst_ip": "ffff:ffff:ff00:0000:0000:0000:0000:0000" "dst_ip2": "ffff:ffff:ff00:0000:0000:0000:0000:0000",
"dport_format":"mask",
"dst_port1": "0",
"dst_port2": "65535",
"protocol": 0,
"direction": "double"
} }
} }
] ]

9
tools/maat_redis_tool.cpp
View File

@@ -308,14 +308,13 @@ int main(int argc, char * argv[])
printf("open %s failed.\n", json_file); printf("open %s failed.\n", json_file);
} }
ret = json2iris(json_buff, json_file, NULL, NULL, NULL, c, tmp_iris_path, ret = json2iris(json_buff, json_file, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, logger);
sizeof(tmp_iris_path), NULL, NULL, NULL);
if (ret < 0) { if (ret < 0) {
printf("Invalid json format.\n"); printf("Invalid json format.\n");
} }
size_t total_line_cnt = 0; size_t total_line_cnt = 0;
config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, NULL); config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger);
printf("Serialize %s to %zu lines, write temp file to %s .\n", json_file, total_line_cnt, tmp_iris_path); printf("Serialize %s to %zu lines, write temp file to %s .\n", json_file, total_line_cnt, tmp_iris_path);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
@@ -328,13 +327,13 @@ int main(int argc, char * argv[])
absolute_expire_time = server_time + timeout; absolute_expire_time = server_time + timeout;
} }
config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, NULL); config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, logger);
printf("Timeout = %lld\n", absolute_expire_time); printf("Timeout = %lld\n", absolute_expire_time);
ret = 0; ret = 0;
int success_cnt = 0; int success_cnt = 0;
do { do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, NULL); success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, logger);
} while(success_cnt < 0); } while(success_cnt < 0);
if (success_cnt != (int)total_line_cnt) { if (success_cnt != (int)total_line_cnt) {

17
vendor/CMakeLists.txt vendored
View File

@@ -83,3 +83,20 @@ add_library(hiredis-static STATIC IMPORTED GLOBAL)
add_dependencies(hiredis-static hiredis) add_dependencies(hiredis-static hiredis)
set_property(TARGET hiredis-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhiredisd.a) set_property(TARGET hiredis-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhiredisd.a)
set_property(TARGET hiredis-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include) set_property(TARGET hiredis-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include)
#igraph-0.7.1
ExternalProject_Add(igraph PREFIX igraph
URL ${CMAKE_CURRENT_SOURCE_DIR}/igraph-0.7.1.tar.gz
URL_MD5 4f6e7c16b45fce8ed423516a9786e4e8
CONFIGURE_COMMAND ./configure --prefix=<INSTALL_DIR> --disable-shared --with-pic
BUILD_COMMAND make
INSTALL_COMMAND make install
BUILD_IN_SOURCE 1)
ExternalProject_Get_Property(igraph INSTALL_DIR)
file(MAKE_DIRECTORY ${INSTALL_DIR}/include)
add_library(igraph-static STATIC IMPORTED GLOBAL)
add_dependencies(igraph-static igraph)
set_property(TARGET igraph-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libigraph.a)
set_property(TARGET igraph-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include)

BIN
vendor/igraph-0.7.1.tar.gz vendored Normal file
View File

Binary file not shown.