gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

unfinished work

Browse Source
This commit is contained in:
liuwentan
2023-01-30 21:59:35 +08:00
parent 3d4b833e48
commit 25f944a1d1
49 changed files with 6537 additions and 6149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
deps/log/log.c vendored
View File

@@ -136,6 +136,7 @@ static int log_create_path(const char *file_path)
{
return 0;
}
fclose(fp);
return 1;
}

89
include/maat/maat.h
View File

@@ -27,11 +27,6 @@ extern "C"
/* maat instance handle */
struct maat;
enum ip_type {
IP_TYPE_V4,
IP_TYPE_V6
};
struct maat_hit_path {
int Nth_scan;
int item_id;
@@ -41,7 +36,7 @@ struct maat_hit_path {
int compile_id;
};
struct maat_matched {
struct maat_hit_object {
int virtual_table_id;
int group_id;
};
@@ -49,30 +44,17 @@ struct maat_matched {
enum maat_scan_opt
{
MAAT_SET_SCAN_DISTRICT = 1, //VALUE is a const char*, SIZE= strlen(string). DEFAULT: no default.
MAAT_SET_SCAN_LAST_REGION, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination.
MAAT_SET_SCAN_LAST_ITEM, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination.
MAAT_SET_SCAN_COMPILE_TABLE_ID, //Caller can specify which compile table to use
MAAT_GET_SCAN_HIT_PATH, //VALUE is struct maat_hit_path*, an array of struct maat_hit_path, SIZE= sizeof(struct maat_hit_path)*N,
//Maat_get_scan_status returns actual got number.
MAAT_GET_SCAN_MATCHED //VALUE is struct maat_matched*, an array of struct maat_matched, SIZE= sizeof(struct maat_matched)*N,
MAAT_GET_SCAN_HIT_OBJECTS //VALUE is struct maat_hit_object*, an array of struct maat_hit_object, SIZE= sizeof(struct maat_hit_object)*N,
};
/* network order */
struct ipv4_2tuple {
uint32_t sip;
uint32_t dip;
};
struct ipv6_2tuple {
uint8_t sip[16];
uint8_t dip[16];
};
struct addr_2tuple {
enum ip_type type;
union {
struct ipv4_2tuple ipv4;
struct ipv6_2tuple ipv6;
};
};
#define MAAT_OK 0 //scan but not hit(group or compile)
#define MAAT_ERR -1 //scan error
#define MAAT_HALF_HIT 1 //half hit: hit group, not hit compile
#define MAAT_HIT 2 //scan hit compile
#define MAAT_RULE_UPDATE_TYPE_FULL 1
#define MAAT_RULE_UPDATE_TYPE_INC 2
@@ -93,8 +75,11 @@ typedef void maat_rule_ex_dup_func_t(int idx, void *to, void *from, long argl, v
/* maat_instance options API */
struct maat_options;
struct maat_options* maat_options_new(void);
struct maat_options *maat_options_new(void);
void maat_options_free(struct maat_options *opts);
int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread);
int maat_options_set_accept_tags(struct maat_options *opts, const char *accept_tags);
int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms);
@@ -110,8 +95,9 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path);
void maat_free(struct maat *instance);
/* maat table API */
/* return table_id(>=0) if success,otherwise return -1 */
int maat_table_get_id(struct maat *instance, const char *table_name);
/* return 0 if success, otherwise return -1 */
int maat_table_callback_register(struct maat *instance, int table_id,
maat_start_callback_t *start,
maat_update_callback_t *update,
@@ -129,17 +115,33 @@ void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id,
const char *key, size_t key_len);
/* maat scan API */
struct maat_state;
int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
unsigned int intval, int results[], size_t *n_result,
struct maat_state **state);
int maat_scan_ip(struct maat *instance, int table_id, int thread_id,
struct addr_2tuple *addr, int results[], size_t *n_result,
struct maat_state **state);
/**
* @param instance: maat instance created by maat_new()
* @param table_id: the id of table which to be scanned
* @param thread_id: thread index
* @param results: array to store hit compile id
* @param n_result: the array size
* @param n_hit_result: the number of hit compile id
* @param state: scan mid status
*
* @retval MAAT_ERR/MAAT_OK/MAAT_HALF_HIT
*/
int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
unsigned int intval, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state);
int maat_scan_ipv4(struct maat *instance, int table_id, int thread_id,
uint32_t ip_addr, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state);
int maat_scan_ipv6(struct maat *instance, int table_id, int thread_id,
uint8_t *ip_addr, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state);
int maat_scan_string(struct maat *instance, int table_id, int thread_id,
const char *data, size_t data_len, int results[], size_t *n_result,
struct maat_state **state);
const char *data, size_t data_len, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state);
struct maat_stream;
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id);
@@ -150,15 +152,20 @@ int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* dat
void maat_scan_stream_close(struct maat_stream **stream);
/* maat state API */
int maat_state_set(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, const void *value, int size);
int maat_state_set_scan_district(struct maat *instance, struct maat_state **state, const char *district, size_t district_len);
//return >=0 if success, return -1 when failed;
int maat_state_get(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, void *value, int size);
int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **state);
int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, int compile_table_id);
int maat_state_get_hit_paths(struct maat *instance, struct maat_state **state, struct maat_hit_path *paths, size_t n_path);
int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state, struct maat_hit_object *objs, size_t n_obj);
void maat_state_free(struct maat_state **state);
/* return matched compile_id */
int maat_matched_compile_id(struct maat *instance, struct maat_matched *matched);
/* return hit object compile_id */
int maat_hit_object_compile_id(struct maat *instance, struct maat_hit_object *obj);
#ifdef __cpluscplus
}

1
scanner/CMakeLists.txt
View File

@@ -3,6 +3,7 @@ add_definitions(-fPIC)
include_directories(${PROJECT_SOURCE_DIR}/deps)
include_directories(${PROJECT_SOURCE_DIR}/include)
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
add_library(adapter-static adapter_hs.cpp bool_matcher.cpp)

39
scanner/adapter_hs.cpp
View File

@@ -17,8 +17,11 @@
#include "uthash/utarray.h"
#include "uthash/uthash.h"
#include "utils.h"
#include "maat_utils.h"
#include "bool_matcher.h"
#define MODULE_ADAPTER_HS module_name_str("maat.adapter_hs")
struct adpt_hs_compile_data {
unsigned int *ids;
unsigned int *flags;
@@ -56,7 +59,8 @@ struct adapter_hs_stream {
UT_array *pattern_id_set;
};
int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_threads, int max_pattern_type)
static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_threads, int max_pattern_type,
struct log_handle *logger)
{
hs_database_t *database = NULL;
hs_rt->scratchs = ALLOC(hs_scratch_t *, nr_worker_threads);
@@ -68,7 +72,7 @@ int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_thr
}
if (hs_alloc_scratch(database, &hs_rt->scratchs[0]) != HS_SUCCESS) {
fprintf(stderr, "ERROR: Unable to allocate scratch space. Exiting.\n");
log_error(logger, MODULE_ADAPTER_HS, "ERROR: Unable to allocate scratch space. Exiting.");
hs_free_database(database);
return -1;
}
@@ -76,14 +80,14 @@ int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_thr
for (size_t i = 1; i < nr_worker_threads; i++) {
hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]);
if (err != HS_SUCCESS) {
fprintf(stderr, "Unable to clone scratch prototype\n");
log_error(logger, MODULE_ADAPTER_HS, "Unable to clone scratch prototype");
hs_free_database(database);
return -1;
}
err = hs_scratch_size(hs_rt->scratchs[i], &hs_rt->scratch_size);
if (err != HS_SUCCESS) {
fprintf(stderr, "Unable to query scratch size\n");
log_error(logger, MODULE_ADAPTER_HS, "Unable to query scratch size");
hs_free_database(database);
return -1;
}
@@ -97,10 +101,10 @@ int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_thr
*
* @retval 0(success) -1(failed)
*/
int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
struct adpt_hs_compile_data *literal_cd,
struct adpt_hs_compile_data *regex_cd,
int scan_mode)
int scan_mode, struct log_handle *logger)
{
hs_error_t err;
hs_compile_error_t *compile_err = NULL;
@@ -115,7 +119,7 @@ int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
scan_mode, NULL, &hs_rt->literal_db, &compile_err);
if (err != HS_SUCCESS) {
if (compile_err) {
fprintf(stderr, "%s compile error: %s\n", __func__, compile_err->message);
log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s", __func__, compile_err->message);
}
hs_free_compile_error(compile_err);
@@ -129,7 +133,7 @@ int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
scan_mode, NULL, &hs_rt->regex_db, &compile_err);
if (err != HS_SUCCESS) {
if (compile_err) {
fprintf(stderr, "%s compile error: %s\n", __func__, compile_err->message);
log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s", __func__, compile_err->message);
}
hs_free_compile_error(compile_err);
goto error;
@@ -182,11 +186,12 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patt
FREE(hs_cd);
}
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array)
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array,
struct log_handle *logger)
{
if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) ||
0 == nr_worker_threads || NULL == expr_array || 0 == n_expr_array) {
fprintf(stderr, "%s input parameters illegal!\n", __func__);
log_error(logger, MODULE_ADAPTER_HS, "%s input parameters illegal!", __func__);
return NULL;
}
@@ -196,7 +201,8 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
for (size_t i = 0; i < n_expr_array; i++) {
if (expr_array[i].n_patterns > MAX_EXPR_PATTERN_NUM) {
fprintf(stderr, "the number of patterns in one expression should less than %d\n", MAX_EXPR_PATTERN_NUM);
log_error(logger, MODULE_ADAPTER_HS,
"the number of patterns in one expression should less than %d", MAX_EXPR_PATTERN_NUM);
return NULL;
}
@@ -206,7 +212,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
} else if (expr_array[i].patterns[j].type == PATTERN_TYPE_REG) {
regex_pattern_num++;
} else {
fprintf(stderr, "unknown pattern type: %d\n", expr_array[i].patterns[j].type);
log_error(logger, MODULE_ADAPTER_HS, "unknown pattern type: %d", expr_array[i].patterns[j].type);
return NULL;
}
}
@@ -285,16 +291,17 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
/* create bool matcher */
hs_instance->hs_rt->bm = bool_matcher_new(exprs, n_expr_array, &mem_size);
if (hs_instance->hs_rt->bm != NULL) {
fprintf(stdout, "Adapter_hs module: build bool matcher of %zu expressions with %zu bytes memory\n",
log_info(logger, MODULE_ADAPTER_HS,
"Adapter_hs module: build bool matcher of %zu expressions with %zu bytes memory",
n_expr_array, mem_size);
} else {
fprintf(stderr, "Adapter_hs module: build bool matcher failed\n");
log_error(logger, MODULE_ADAPTER_HS, "Adapter_hs module: build bool matcher failed");
goto error;
}
FREE(exprs);
/* build hs database */
ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, scan_mode);
ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, scan_mode, logger);
if (ret < 0) {
goto error;
}
@@ -314,7 +321,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
max_patterns_type = PATTERN_TYPE_REG;
}
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, nr_worker_threads, max_patterns_type);
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, nr_worker_threads, max_patterns_type, logger);
if (ret < 0) {
goto error;
}

4
scanner/adapter_hs.h
View File

@@ -19,6 +19,8 @@ extern "C"
#include <stddef.h>
#include <stdint.h>
#include "log/log.h"
#define MAX_EXPR_PATTERN_NUM 8
struct adapter_hs;
@@ -63,7 +65,7 @@ typedef struct {
*
* @retval the pointer to adapter_hs instance
*/
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array);
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array, struct log_handle *logger);
/**
* @brief scan input data to match logic AND expression, return all matched expr_id

1
scanner/adapter_hs_gtest.cpp
View File

@@ -2,7 +2,6 @@
#include "utils.h"
#include "adapter_hs.h"
#include "maat_table_schema.h"
int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr)
{

19
src/CMakeLists.txt
View File

@@ -9,8 +9,9 @@ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC)
set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp
maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp
maat_command.cpp maat_redis_monitor.cpp maat_hierarchy.cpp)
maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_utils.cpp maat_command.cpp maat_redis_monitor.cpp
maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp
maat_expr.cpp maat_plugin.cpp maat_ip_plugin.cpp maat_virtual.cpp)
set(LIB_SOURCE_FILES
${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c ${PROJECT_SOURCE_DIR}/deps/log/log.c)
@@ -29,14 +30,14 @@ set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static pthread crypto z)
# Shared Library Output
add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES})
set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX)
set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe)
set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1)
set_target_properties(maat_frame_shared PROPERTIES VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION})
set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION})
#add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES})
#set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX)
#set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe)
#set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1)
#set_target_properties(maat_frame_shared PROPERTIES VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION})
#set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION})
target_link_libraries(maat_frame_shared hyperscan_static)
#target_link_libraries(maat_frame_shared hyperscan_static)
# install
set(CMAKE_INSTALL_PREFIX /opt/MESA/)

4
src/inc_internal/maat_common.h
View File

@@ -23,8 +23,10 @@ extern "C"
struct maat_options {
char instance_name[NAME_MAX];
char compile_tablename[NAME_MAX];
size_t nr_worker_threads;
const char *accept_tags;
int rule_effect_interval_ms;
int rule_update_checking_interval_ms;
int gc_timeout_ms;

90
src/inc_internal/maat_compile.h Normal file
View File

@@ -0,0 +1,90 @@
/*
**********************************************************************************************
* File: maat_compile.h
* Description:
* Authors: Zheng Chao <zhengchao@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_COMPILE_H_
#define _MAAT_COMPILE_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "log/log.h"
#include "cJSON/cJSON.h"
#include "maat.h"
#include "maat_kv.h"
#include "maat_rule.h"
struct compile_ex_data_schema {
maat_rule_ex_new_func_t *new_func;
maat_rule_ex_free_func_t *free_func;
maat_rule_ex_dup_func_t *dup_func;
long argl;
void *argp;
int idx;
int table_id;
};
/* compile schema API */
void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void compile_schema_free(void *compile_schema);
void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void group2compile_schema_free(void *g2c_schema);
int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, int table_id,
maat_rule_ex_new_func_t *new_func,
maat_rule_ex_free_func_t *free_func,
maat_rule_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger);
struct compile_ex_data_schema *
compile_table_get_rule_ex_data_schema(struct compile_schema *compile_schema, size_t idx);
size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema);
/* compile runtime API */
void *compile_runtime_new(void *compile_schema, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void compile_runtime_free(void *compile_runtime);
int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line,
int valid_column);
int compile_runtime_commit(void *compile_runtime);
int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids,
int *compile_ids, size_t compile_ids_size, struct maat_state *state);
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct maat_group_topology *group_topo,
struct maat_compile_state *compile_state,
struct maat_hit_path *hit_paths, size_t hit_path_siz);
/* group2compile runtime API */
void *group2compile_runtime_new(void *g2c_schema, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void group2compile_runtime_free(void *g2c_runtime);
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line,
int valid_column);
/* maat compile state API */
struct maat_compile_state;
struct maat_compile_state *maat_compile_state_new(int thread_id);
void maat_compile_state_free(struct maat_compile_state *compile_state);
void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, int item_id, int group_id, int virtual_table_id,
int Nth_scan, int Nth_item_result);
void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, struct maat_compile **compile_hash,
int group_id, int virtual_table_id);
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state);
#ifdef __cpluscplus
}
#endif
#endif

31
src/inc_internal/maat_ex_data.h
View File

@@ -16,6 +16,7 @@ extern "C"
{
#endif
#include "maat/maat.h"
#include "rcu_hash.h"
struct ex_data_container {
@@ -23,6 +24,12 @@ struct ex_data_container {
void *custom_data;
};
struct ex_container_ctx {
int table_id;
void (*custom_data_free)(void *custom_data);
struct ex_data_schema *ex_schema;
};
struct ex_data_runtime;
/* ex_data_runtime API */
@@ -42,6 +49,12 @@ size_t ex_data_runtime_cached_row_count(struct ex_data_runtime *ex_data_rt);
void ex_data_runtime_clear_row_cache(struct ex_data_runtime *ex_data_rt);
/* set schema API */
struct ex_data_schema *ex_data_schema_new(maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp);
void ex_data_schema_free(struct ex_data_schema *ex_schema);
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema);
/* set user_ctx API */
@@ -49,16 +62,24 @@ void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, st
struct ex_container_ctx *ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt);
struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data);
void ex_data_container_free(void *ctx, void *data);
/* ex_data_runtime ex data API */
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len);
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt,
const char *row, const char *key, size_t key_len);
void ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct ex_data_container *ex_container);
int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len,
struct ex_data_container *ex_container);
void ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct log_handle *logger);
int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len);
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, struct ex_data_container ***ex_container);
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_data_container ***ex_container);
void *ex_data_runtime_dup_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);
void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);

54
src/inc_internal/maat_expr.h Normal file
View File

@@ -0,0 +1,54 @@
/*
**********************************************************************************************
* File: maat_expr.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_EXPR_H_
#define _MAAT_EXPR_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "log/log.h"
#include "cJSON/cJSON.h"
struct expr_runtime;
void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void expr_schema_free(void *expr_schema);
/* expr runtime API */
void *expr_runtime_new(void *expr_schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
void expr_runtime_free(void *expr_runtime);
int expr_runtime_updating_flag(struct expr_runtime *expr_rt);
int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column);
int expr_runtime_commit(void *expr_runtime);
/* expr runtime scan API */
/**
* @brief scan string to get hit group_ids
*
* @retval the num of hit group_id
*/
int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len, int group_ids[],
size_t max_hit_num, int virtual_table_id,
struct maat_state *state);
void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len, int results[], size_t *n_result);
void expr_runtime_stream_close(struct expr_runtime *expr_rt);
#ifdef __cpluscplus
}
#endif
#endif

34
src/inc_internal/maat_group.h
View File

@@ -16,15 +16,10 @@ extern "C"
{
#endif
struct maat_item {
int item_id;
int group_id;
struct maat_group *ref_parent_group;
UT_hash_handle hh;
void *user_data;
};
#include "cJSON/cJSON.h"
#include "uthash/uthash.h"
#include "igraph/igraph.h"
#include "maat_kv.h"
struct maat_group {
igraph_integer_t vertex_id;
@@ -32,16 +27,16 @@ struct maat_group {
int ref_by_compile_cnt;
int ref_by_superior_group_cnt;
int ref_by_subordinate_group_cnt;
int ref_by_item_cnt;
size_t top_group_cnt;
int *top_group_ids;
UT_hash_handle hh_group_id;
UT_hash_handle hh_vertex_id;
};
/* maat group topology API */
struct maat_group_topology;
struct maat_group_topology *maat_group_topology_new(struct log_handle *logger);
void maat_group_topology_free(struct maat_group_topology *group_topo);
@@ -59,6 +54,23 @@ int maat_group_topology_remove_group_from_group(struct maat_group_topology *grou
/* build top groups */
int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo);
/* group2group schema API */
void *group2group_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void group2group_schema_free(void *g2g_schema);
/* group2group runtime API */
struct group2group_runtime *
group2group_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void group2group_runtime_free(void *g2g_runtime);
int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, int valid_column);
int group2group_runtime_commit(void *g2g_runtime);
int group2group_runtime_get_top_groups(struct group2group_runtime *g2g_rt, int *group_ids, size_t n_group_ids,
int *top_group_ids);
#ifdef __cpluscplus
}
#endif

67
src/inc_internal/maat_hierarchy.h
View File

@@ -1,67 +0,0 @@
/*
**********************************************************************************************
* File: maat_hierarchy.h
* Description:
* Authors: Zheng Chao <zhengchao@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_HIERARCHY_H_
#define _MAAT_HIERARCHY_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "maat_garbage_collection.h"
/* maat hierarchy API */
struct maat_hierarchy;
struct maat_hierarchy *maat_hierarchy_new(int thread_num, struct maat_garbage_bin *bin, struct log_handle *logger);
void maat_hierarchy_free(struct maat_hierarchy *hier);
int maat_hierarchy_rebuild(struct maat_hierarchy *hier);
size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
struct maat_hit_path *hit_paths, size_t n_path);
/* maat hierarchy compile mid API */
struct maat_hierarchy_compile_mid;
struct maat_hierarchy_compile_mid *maat_hierarchy_compile_mid_new(struct maat_hierarchy *hier, int thread_id);
void maat_hierarchy_compile_mid_free(struct maat_hierarchy_compile_mid *mid);
void maat_hierarchy_compile_mid_update(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
int item_id, int virtual_table_id, int Nth_scan, int Nth_item_result);
int maat_hierarchy_compile_mid_has_NOT_clause(struct maat_hierarchy_compile_mid *mid);
/* maat hierarchy compile API */
int maat_hierarchy_compile_add(struct maat_hierarchy *hier, int compile_id, int declared_clause_num, void* user_data);
int maat_hierarchy_compile_remove(struct maat_hierarchy *hier, int compile_id);
void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
void *maat_hierarchy_compile_dettach_user_data(struct maat_hierarchy *hier, int compile_id);
void *maat_hierarchy_compile_read_user_data(struct maat_hierarchy *hier, int compile_id);
void maat_hierarchy_compile_user_data_iterate(struct maat_hierarchy *hier, void (*callback)(void *user_data, void *param), void *param);
/* maat hierarchy group2compile API */
int maat_hierarchy_add_group_to_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag,
int clause_index, int compile_id);
int maat_hierarchy_remove_group_from_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag,
int clause_index, int compile_id);
/* maat hierarchy item API */
int maat_hierarchy_item_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid,
int is_last_compile, void **user_data_array, size_t ud_array_sz);
void *maat_hierarchy_item_dettach_user_data(struct maat_hierarchy *hier, int item_id);
void maat_hierarchy_set_item_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *));
/* maat hierarchy item2group API */
int maat_hierarchy_add_item_to_group(struct maat_hierarchy *hier, int group_id, int item_id, void* user_data);
int maat_hierarchy_remove_item_from_group(struct maat_hierarchy *hier, int group_id, int item_id);
#ifdef __cpluscplus
}
#endif
#endif

42
src/inc_internal/maat_ip.h Normal file
View File

@@ -0,0 +1,42 @@
/*
**********************************************************************************************
* File: maat_ip.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_IP_H_
#define _MAAT_IP_H_
#ifdef __cpluscplus
extern "C"
{
#endif
struct ip_plus_runtime;
void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void ip_plus_schema_free(void *ip_plus_schema);
/* ip plus runtime API */
void *ip_plus_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
void ip_plus_runtime_free(void *ip_plus_runtime);
int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line);
int ip_plus_runtime_commit(void *ip_plus_runtime);
struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt);
/* ip runtime scan API */
int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr *data,
int *group_id_array, size_t n_group_id_array, int virtual_table_id,
struct maat_state *state);
#ifdef __cpluscplus
}
#endif
#endif

49
src/inc_internal/maat_ip_plugin.h Normal file
View File

@@ -0,0 +1,49 @@
/*
**********************************************************************************************
* File: maat_ip_plugin.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_IP_PLUGIN_H_
#define _MAAT_IP_PLUGIN_H_
#ifdef __cpluscplus
extern "C"
{
#endif
struct ip_plugin_runtime;
/* ip plugin schema API */
void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void ip_plugin_schema_free(void *ip_plugin_schema);
/* ip plugin table ex data API */
struct ex_data_schema *ip_plugin_table_get_ex_data_schema(void *ip_plugin_schema);
int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp);
/* ip plugin runtime API */
void *ip_plugin_runtime_new(void *ip_plugin_schema, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void ip_plugin_runtime_free(void *ip_plugin_runtime);
int ip_plugin_runtime_updating_flag(struct ip_plugin_runtime *ip_plugin_rt);
int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line);
int ip_plugin_runtime_commit(void *ip_plugin_runtime);
struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime);
#ifdef __cpluscplus
}
#endif
#endif

12
src/inc_internal/maat_kv.h
View File

@@ -18,13 +18,17 @@ extern "C"
struct maat_kv_store;
struct maat_kv_store* maat_kv_store_new(void);
struct maat_kv_store *maat_kv_store_new(void);
void maat_kv_store_free(struct maat_kv_store* store);
void maat_kv_store_free(struct maat_kv_store *store);
int maat_kv_register(struct maat_kv_store* store, const char* key, int value);
int maat_kv_register(struct maat_kv_store *store, const char *key, int value);
int maat_kv_read(struct maat_kv_store* store, const char* key, int* value);
int maat_kv_read(struct maat_kv_store *store, const char *key, int *value);
int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, int *value);
struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *store);
#ifdef __cpluscplus
}

25
src/inc_internal/maat_limits.h Normal file
View File

@@ -0,0 +1,25 @@
/*
**********************************************************************************************
* File: maat_limits.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_LIMITS_H_
#define _MAAT_LIMITS_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#define MAX_KEYWORDS_STR 1024
#ifdef __cpluscplus
}
#endif
#endif

60
src/inc_internal/maat_plugin.h Normal file
View File

@@ -0,0 +1,60 @@
/*
**********************************************************************************************
* File: maat_plugin.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_PLUGIN_H_
#define _MAAT_PLUGIN_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#define MAX_FOREIGN_CLMN_NUM 8
/* plugin schema API */
void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void plugin_schema_free(void *plugin_schema);
struct plugin_schema;
/* plugin table callback API */
int plugin_table_add_callback(void *plugin_schema, int table_id,
maat_start_callback_t *start,
maat_update_callback_t *update,
maat_finish_callback_t *finish,
void *u_para, struct log_handle *logger);
void plugin_table_all_callback_start(struct plugin_schema *plugin_schema, int update_type);
void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema);
int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns);
/* plugin table ex data API */
int plugin_table_set_ex_data_schema(void *custom_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp);
struct ex_data_schema *plugin_table_get_ex_data_schema(void *custom_schema);
/* plugin runtime API */
struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime);
int plugin_runtime_updating_flag(struct plugin_runtime *plugin_rt);
int plugin_runtime_update(struct plugin_runtime *plugin_rt, struct plugin_item *plugin_item,
struct plugin_schema *plugin_schema, const char *row,
struct log_handle *logger);
int plugin_runtime_commit(void *plugin_runtime);
#ifdef __cpluscplus
}
#endif
#endif

62
src/inc_internal/maat_rule.h
View File

@@ -27,9 +27,12 @@ extern "C"
#include "log/log.h"
#include "hiredis/hiredis.h"
#include "uthash/uthash.h"
#include "maat_table_schema.h"
#include "maat_command.h"
#include "IPMatcher.h"
#include "maat_kv.h"
#include "maat_table.h"
#define MAX_TABLE_NUM 256
struct maat_rule_head {
int config_id;
@@ -64,8 +67,15 @@ struct maat_item_inner {
int expr_id_ub; //up boundary
};
struct maat_item {
int item_id;
int group_id;
UT_hash_handle hh;
void *user_data;
};
#define COMPILE_RULE_MAGIC 0x1a2b3c4d
struct maat_compile_rule {
struct compile_rule {
long long magic_num;
int compile_id;
struct maat_rule_head head;// fix len of Maat_rule_t
@@ -73,7 +83,7 @@ struct maat_compile_rule {
int is_valid;
int declared_clause_num;
double evaluation_order;
struct table_schema *ref_table;
struct compile_schema *ref_table;
void **ex_data;
pthread_rwlock_t rwlock;
};
@@ -83,8 +93,9 @@ struct group2compile_rule {
int compile_id;
int is_valid;
int not_flag;
int virtual_table_id;
int vt_id; //virtual_table_id
int clause_index;
int associated_compile_table_id;
};
struct group2group_rule {
@@ -100,14 +111,19 @@ struct maat_runtime {
time_t last_update_time;
long long *ref_cnt;
struct table_runtime_manager *table_rt_mgr;
struct table_manager *ref_tbl_mgr; //share with maat instance
size_t max_table_num;
size_t max_thread_num;
uint32_t rule_num;
struct maat_garbage_bin *ref_garbage_bin;
struct scan_result *item_result_buff;
struct maat_kv_store *district_map;
struct maat_kv_store *tmp_district_map;
unsigned int district_num;
struct log_handle *logger;
};
@@ -179,7 +195,7 @@ struct maat {
struct maat_runtime *maat_rt;
struct maat_runtime *creating_maat_rt;
struct table_schema_manager *table_schema_mgr;
struct table_manager *tbl_mgr;
enum data_source input_mode;
union {
@@ -188,9 +204,6 @@ struct maat {
struct source_redis_ctx mr_ctx;
};
struct rule_tag *accept_tags;
int n_accept_tag;
struct log_handle *logger;
int deferred_load;
@@ -210,10 +223,8 @@ struct maat {
struct maat_garbage_bin *garbage_bin;
char compile_tablename[NAME_MAX];
//char group_tn[NAME_MAX];
//char group2compile_tn[NAME_MAX];
//char group2group_tn[NAME_MAX];
int default_compile_table_id;
int g2g_table_id; //group2group table id
char decrypt_key[NAME_MAX];
char decrypt_algo[NAME_MAX];
@@ -232,6 +243,29 @@ struct maat {
long long scan_err_cnt;
};
struct maat_state {
struct maat *maat_instance;
int16_t thread_id;
int compile_table_id; //caller can select compile table to scan
unsigned char is_set_district;
unsigned char is_last_scan;
int district_id; //-1: Any District; -2: Unkonwn District;
int scan_cnt;
struct maat_compile_state *compile_mid;
};
int parse_accept_tag(const char *value, struct rule_tag **result, void *logger);
int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, int n_tag);
struct maat_item *maat_item_new(int item_id, int group_id, void *user_data);
void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *));
struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int district_id);
void maat_item_inner_free(struct maat_item_inner *item);
void maat_start_cb(long long new_version, int update_type, void *u_para);
int maat_update_cb(const char *table_name, const char *line, void *u_para);

68
src/inc_internal/maat_table.h Normal file
View File

@@ -0,0 +1,68 @@
/*
**********************************************************************************************
* File: maat_table.h
* Description: maat table schema and runtime
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_TABLE_H_
#define _MAAT_TABLE_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include <stddef.h>
#include <cJSON/cJSON.h>
enum table_type {
TABLE_TYPE_INVALID = -1,
TABLE_TYPE_EXPR = 0,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_IP_PLUS,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_INTERVAL_PLUS,
TABLE_TYPE_DIGEST,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_CONJUNCTION,
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN,
TABLE_TYPE_FQDN_PLUGIN,
TABLE_TYPE_BOOL_PLUGIN,
//above are physical table
TABLE_TYPE_VIRTUAL,
TABLE_TYPE_COMPILE,
TABLE_TYPE_GROUP2GROUP,
TABLE_TYPE_GROUP2COMPILE,
TABLE_TYPE_MAX
};
struct table_manager;
struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags,
struct log_handle *logger);
int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *garbage_bin);
int table_manager_deinit(struct table_manager *tbl_mgr);
void table_manager_destroy(struct table_manager *tbl_mgr);
size_t table_manager_table_count(struct table_manager *tbl_mgr);
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name);
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id);
int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id);
int table_manager_accept_tags_match(const char *tags);
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id);
void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id);
void table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line);
void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id);
#ifdef __cpluscplus
}
#endif
#endif

86
src/inc_internal/maat_table_runtime.h
View File

@@ -1,86 +0,0 @@
/*
**********************************************************************************************
* File: maat_table_runtime.h
* Description: maat table runtime entry
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_TABLE_RUNTIME_H_
#define _MAAT_TABLE_RUNTIME_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "maat/maat.h"
#include "maat_table_schema.h"
#include "maat_garbage_collection.h"
struct ip_addr {
enum ip_type ip_type;
union {
uint32_t ipv4;
uint32_t ipv6[4];
};
};
struct table_item;
struct table_runtime;
struct table_runtime_manager;
/* table runtime manager API */
struct table_runtime_manager *
table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num,
struct maat_garbage_bin *bin, struct log_handle *logger);
void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr);
/* table runtime API */
struct table_runtime *table_runtime_get(struct table_runtime_manager *table_rt_mgr, int table_id);
size_t table_runtime_rule_count(struct table_runtime *table_rt);
enum table_type table_runtime_get_type(struct table_runtime* table_rt);
void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema,
const char *line, struct table_item *table_item, struct log_handle *logger);
/**
* @brief if table_runtime is updating
*
* @retval 1(yes) 0(no)
*/
int table_runtime_updating_flag(struct table_runtime *table_rt);
void table_runtime_commit(struct table_runtime *table_rt, long long version, size_t nr_worker_thread, struct log_handle *logger);
/* table runtime scan API */
int table_runtime_scan_string(struct table_runtime *table_rt, int thread_id, const char *data, size_t data_len,
int results[], size_t *n_result);
void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id);
int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, int results[], size_t *n_result);
void table_runtime_stream_close(struct table_runtime *table_rt);
int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct ip_addr *data, struct scan_result *results, size_t *n_result, size_t n_result_array);
/* table runtime cached row API */
size_t table_runtime_cached_row_count(struct table_runtime *table_rt);
const char* table_runtime_get_cached_row(struct table_runtime *table_rt, size_t row_seq);
/* table runtime ex data API */
struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt);
void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema,
int nr_worker_thread, long long version, struct log_handle *logger);
#ifdef __cpluscplus
}
#endif
#endif

310
src/inc_internal/maat_table_schema.h
View File

@@ -1,310 +0,0 @@
/*
**********************************************************************************************
* File: maat_table_schema.h
* Description: maat table schema entry
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_TABLE_SCHEMA_H_
#define _MAAT_TABLE_SCHEMA_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include <stddef.h>
#include <limits.h>
#include "maat/maat.h"
#include "adapter_hs.h"
#define MAX_TABLE_NUM 256
#define MAX_DISTRICT_STR 128
#define MAX_IP_STR 128
#define MAX_KEYWORDS_STR 1024
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_TABLE_LINE_SIZE (1024 * 16)
#define MAX_COMPILE_EX_DATA_NUM 2
enum component_table_type {
COMPONENT_TABLE_TYPE_NONE = -1,
COMPONENT_TABLE_TYPE_SIP = 0,
COMPONENT_TABLE_TYPE_DIP,
COMPONENT_TABLE_TYPE_SESSION,
COMPONENT_TABLE_TYPE_MAX
};
enum table_type {
TABLE_TYPE_INVALID = -1,
TABLE_TYPE_EXPR = 0,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_IP_PLUS,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_INTERVAL_PLUS,
TABLE_TYPE_DIGEST,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN,
TABLE_TYPE_FQDN_PLUGIN,
TABLE_TYPE_BOOL_PLUGIN,
//above are physical table
TABLE_TYPE_VIRTUAL,
TABLE_TYPE_COMPOSITION,
TABLE_TYPE_COMPILE,
TABLE_TYPE_GROUP,
TABLE_TYPE_GROUP2GROUP,
TABLE_TYPE_GROUP2COMPILE
};
enum expr_type {
EXPR_TYPE_STRING = 0,
EXPR_TYPE_AND,
EXPR_TYPE_REGEX,
EXPR_TYPE_MAX
};
enum scan_type {
SCAN_TYPE_INVALID = -1,
SCAN_TYPE_NONE = 0,
SCAN_TYPE_PLUGIN,
SCAN_TYPE_IP_PLUGIN,
SCAN_TYPE_FQDN_PLUGIN,
SCAN_TYPE_BOOL_PLUGIN,
SCAN_TYPE_IP,
SCAN_TYPE_INTERVAL,
SCAN_TYPE_STRING,
SCAN_TYPE_MAX
};
enum match_method {
MATCH_METHOD_SUB = 0,
MATCH_METHOD_RIGHT,
MATCH_METHOD_LEFT,
MATCH_METHOD_COMPLETE,
MATCH_METHOD_MAX
};
struct compile_item {
int compile_id;
int service_id;
int action;
int do_blacklist;
int do_log;
char user_region[MAX_TABLE_LINE_SIZE];
int is_valid;
int clause_num;
int evaluation_order;
};
struct group2compile_item {
int group_id;
int compile_id;
int is_valid;
int not_flag;
int virtual_table_id;
int clause_index;
int associated_compile_table_id;
};
struct group2group_item {
int group_id;
int superior_group_id;
int is_valid;
};
struct expr_item {
int item_id;
int group_id;
char district[MAX_DISTRICT_STR];
char keywords[MAX_KEYWORDS_STR];
enum expr_type expr_type;
enum match_method match_method;
int is_hexbin;
int is_case_sensitive;
int is_valid;
};
struct ipv4_item_rule {
uint32_t min_sip; /* 源地址下界；0表示忽略本字段 */
uint32_t max_sip; /* 源地址上界；0表示固定IP=min_saddr */
uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */
uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */
uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP；0表示忽略本字段 */
uint16_t direction; /* 方向，0表示双向，1表示单向 */
};
struct ipv6_item_rule {
uint32_t min_sip[4]; /* 源地址下界；全0表示忽略本字段 */
uint32_t max_sip[4]; /* 源地址上界；全0表示固定IP=min_saddr */
uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */
uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */
uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP，无限制默认为0 */
uint16_t direction; /* 方向，0表示双向，1表示单向 */
};
struct ip_plus_item {
int item_id;
int group_id;
int addr_type;
union {
struct ipv4_item_rule ipv4;
struct ipv6_item_rule ipv6;
};
int is_valid;
};
struct plugin_item {
char key[MAX_KEYWORDS_STR];
size_t key_len;
int is_valid;
};
struct ip_plugin_item {
int item_id;
int ip_type;
char start_ip[MAX_IP_STR];
char end_ip[MAX_IP_STR];
int is_valid;
int rule_tag;
};
struct table_item {
enum table_type table_type;
union {
struct compile_item compile_item;
struct group2compile_item g2c_item;
struct group2group_item g2g_item;
struct expr_item expr_item;
struct ip_plus_item ip_plus_item;
struct plugin_item plugin_item;
struct ip_plugin_item ip_plugin_item;
};
};
struct plugin_table_callback_schema
{
maat_start_callback_t *start;
maat_update_callback_t *update;
maat_finish_callback_t *finish;
void *u_para;
};
struct ex_data_schema
{
maat_plugin_ex_new_func_t *new_func;
maat_plugin_ex_free_func_t *free_func;
maat_plugin_ex_dup_func_t *dup_func;
//Maat_plugin_EX_key2index_func_t* key2index_func;
long argl;
void *argp;
int set_flag;
};
struct compile_ex_data_schema {
maat_rule_ex_new_func_t *new_func;
maat_rule_ex_free_func_t *free_func;
maat_rule_ex_dup_func_t *dup_func;
long argl;
void *argp;
int idx;
int table_id;
};
struct table_schema;
struct table_schema_manager;
/* table schema manager API */
struct table_schema_manager *table_schema_manager_create(const char *table_info_path, struct log_handle *logger);
void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr);
int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_mgr, const char *table_name);
/**
* @brief get composition table's child table(specified by type) id
*/
int table_schema_manager_get_child_table_id(struct table_schema_manager *table_schema_mgr, int parent_table_id,
enum component_table_type type);
enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int table_id);
size_t table_schema_manager_get_size(struct table_schema_manager* table_schema_mgr);
void table_schema_manager_all_plugin_cb_start(struct table_schema_manager* table_schema_mgr, int update_type);
void table_schema_manager_all_plugin_cb_finish(struct table_schema_manager* table_schema_mgr);
/* table schema generic API */
struct table_schema *table_schema_get(struct table_schema_manager *table_schema_mgr, int table_id);
struct table_schema *table_schema_get_by_scan_type(struct table_schema_manager *table_schema_mgr,
int table_id, enum scan_type type, int *virtual_table_id);
enum table_type table_schema_get_table_type(struct table_schema *table_schema);
int table_schema_get_table_id(struct table_schema *table_schema);
/* get group2compile table's associated compile table id */
int table_schema_get_associated_table_id(struct table_schema *table_schema);
enum scan_type table_schema_get_scan_type(struct table_schema *table_schema);
struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema,
struct rule_tag *accept_tags, int n_accept_tag, struct log_handle *logger);
void table_item_free(struct table_item *table_item);
int table_schema_get_valid_flag_column(struct table_schema *table_schema);
void table_schema_set_updating_name(struct table_schema *table_schema, const char *table_name);
const char *table_schema_get_updating_name(struct table_schema *table_schema);
/* expr table schema API */
enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);
/* table schema ex API */
int table_schema_set_ex_data_schema(struct table_schema *table_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger);
struct ex_data_schema *table_schema_get_ex_data_schema(struct table_schema *table_schema);
/* table schema compile rule ex API */
int table_schema_set_compile_rule_ex_data_schema(struct table_schema *table_schema,
maat_rule_ex_new_func_t *new_func,
maat_rule_ex_free_func_t *free_func,
maat_rule_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger);
struct compile_ex_data_schema *table_schema_get_compile_rule_ex_data_schema(struct table_schema *table_schema, size_t idx);
size_t table_schema_compile_rule_ex_data_schema_count(struct table_schema *table_schema);
/**
* @brief if plugin table schema's ex data schema set
*
* @retval 1(already Set) 0(Not set yet)
*/
int table_schema_ex_data_schema_flag(struct table_schema *table_schema);
int table_schema_add_callback(struct table_schema_manager *table_schema_mgr, int table_id,
maat_start_callback_t *start,
maat_update_callback_t *update,
maat_finish_callback_t *finish,
void *u_para, struct log_handle *logger);
/**
* @brief the number of callback function stored in plugin table schema
*/
size_t table_schema_callback_count(struct table_schema *table_schema);
void table_schema_all_cb_update(struct table_schema *table_schema, const char *row);
int table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns);
#ifdef __cpluscplus
}
#endif
#endif

7
src/inc_internal/maat_utils.h
View File

@@ -31,7 +31,9 @@ extern "C"
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#define MAX_SCANNER_HIT_NUM 4096
#define MAX_SCANNER_HIT_COMPILE_NUM 4096
#define MAX_SCANNER_HIT_GROUP_NUM 4096
#define MAX_SCANNER_HIT_ITEM_NUM 4096
enum maat_ip_format {
IP_FORMAT_RANGE,
@@ -59,6 +61,9 @@ char *maat_strdup(const char *s);
int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len);
/* the column value must be integer */
int get_column_value(const char *line, int column_seq);
int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *out_sz);
char *strtok_r_esc(char *s, const char delim, char **save_ptr);

38
src/inc_internal/maat_virtual.h Normal file
View File

@@ -0,0 +1,38 @@
/*
**********************************************************************************************
* File: maat_virtual.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_VIRTUAL_H_
#define _MAAT_VIRTUAL_H_
#ifdef __cpluscplus
extern "C"
{
#endif
enum scan_type {
SCAN_TYPE_INVALID = -1,
SCAN_TYPE_NONE = 0,
SCAN_TYPE_PLUGIN,
SCAN_TYPE_IP_PLUGIN,
SCAN_TYPE_FQDN_PLUGIN,
SCAN_TYPE_BOOL_PLUGIN,
SCAN_TYPE_IP,
SCAN_TYPE_INTERVAL,
SCAN_TYPE_STRING,
SCAN_TYPE_MAX
};
void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
#ifdef __cpluscplus
}
#endif
#endif

1
src/inc_internal/rcu_hash.h
View File

@@ -51,6 +51,7 @@ void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len
void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_len);
size_t rcu_hash_count(struct rcu_hash_table *htable);
size_t rcu_hash_updating_count(struct rcu_hash_table *htable);
/**
* @brief make add/del effective

1
src/json2iris.cpp
View File

@@ -20,7 +20,6 @@
#include "cJSON/cJSON.h"
#include "maat_kv.h"
#include "maat_utils.h"
#include "maat_table_schema.h"
#include "maat_rule.h"
#include "uthash/uthash.h"

701
src/maat_api.cpp
View File

@@ -21,31 +21,22 @@
#include "maat_kv.h"
#include "maat_command.h"
#include "maat_ex_data.h"
#include "maat_table_schema.h"
#include "maat_table_runtime.h"
#include "maat_table.h"
#include "maat_config_monitor.h"
#include "maat_redis_monitor.h"
#include "maat_hierarchy.h"
#include "maat_compile.h"
#include "alignment.h"
#include "maat_plugin.h"
#include "maat_ip_plugin.h"
#define MODULE_MAAT_API module_name_str("maat.api")
#define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2
struct maat_state {
struct maat *maat_instance;
int16_t thread_id;
unsigned char is_set_district;
unsigned char is_last_scan;
int district_id; //-1: Any District; -2: Unkonwn District;
int scan_cnt;
struct maat_hierarchy_compile_mid *compile_mid;
};
struct scan_item_hit_wrapper {
int Nth_scan;
struct maat_item_inner* wrapped_items[MAX_SCANNER_HIT_NUM];
struct maat_item_inner* wrapped_items[MAX_SCANNER_HIT_COMPILE_NUM];
size_t n_wrapped_item;
int *virtual_table_ids;
@@ -56,7 +47,7 @@ struct scan_item_hit_wrapper {
inline int scan_state_should_compile_NOT(struct maat_state *mid)
{
if (mid && mid->is_last_scan==1 && mid->compile_mid &&
maat_hierarchy_compile_mid_has_NOT_clause(mid->compile_mid)) {
maat_compile_state_has_NOT_clause(mid->compile_mid)) {
return 1;
} else {
return 0;
@@ -78,6 +69,19 @@ struct maat_options* maat_options_new(void)
return options;
}
void maat_options_free(struct maat_options *opts)
{
if (NULL == opts) {
return;
}
if (opts->accept_tags != NULL) {
FREE(opts->accept_tags);
}
FREE(opts);
}
int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread)
{
opts->nr_worker_threads = n_thread;
@@ -85,6 +89,13 @@ int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_th
return 0;
}
int maat_options_set_accept_tags(struct maat_options *opts, const char *accept_tags)
{
opts->accept_tags = maat_strdup(accept_tags);
return 0;
}
int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms)
{
opts->rule_effect_interval_ms = interval_ms;
@@ -245,14 +256,14 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
maat_instance->logger = log_handle_create(log_path, opts->log_level);
}
maat_instance->table_schema_mgr = table_schema_manager_create(table_info_path, maat_instance->logger);
if (NULL == maat_instance->table_schema_mgr) {
maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags, maat_instance->logger);
if (NULL == maat_instance->tbl_mgr) {
goto failed;
}
//TODO: by luis
//maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr);
maat_instance->default_compile_table_id = -1;
if (opts->compile_tablename != NULL) {
memcpy(maat_instance->compile_tablename, opts->compile_tablename, strlen(opts->compile_tablename));
}
maat_instance->input_mode = opts->input_mode;
switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS:
@@ -317,8 +328,8 @@ int maat_table_get_id(struct maat *maat_instance, const char *table_name)
{
int table_id = -1;
struct table_schema_manager *table_schema_mgr = maat_instance->table_schema_mgr;
table_id = table_schema_manager_get_table_id(table_schema_mgr, table_name);
struct table_manager *table_mgr = maat_instance->tbl_mgr;
table_id = table_manager_get_table_id(table_mgr, table_name);
return table_id;
}
@@ -333,6 +344,24 @@ inline void maat_runtime_ref_dec(struct maat_runtime *maat_rt, int thread_id)
alignment_int64_array_add(maat_rt->ref_cnt, thread_id, -1);
}
void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head,
const char *srv_def, int srv_def_len)
{
memcpy(rule, rule_head, sizeof(struct maat_rule_head));
memcpy(rule->service_defined, srv_def, MIN(srv_def_len, MAX_SERVICE_DEFINE_LEN));
}
size_t generic_plugin_runtime_cached_row_count(void *custom_rt, enum table_type table_type)
{
}
const char *generic_plugin_runtime_get_cached_row(void *custom_rt, enum table_type table_type, size_t row_id)
{
}
/* must be plugin table */
int maat_table_callback_register(struct maat *maat_instance, int table_id,
maat_start_callback_t *start,
maat_update_callback_t *update,
@@ -342,8 +371,9 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id,
int ret = -1;
pthread_mutex_lock(&(maat_instance->background_update_mutex));
ret = table_schema_add_callback(maat_instance->table_schema_mgr, table_id,
start, update, finish, u_para, maat_instance->logger);
void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
ret = plugin_table_add_callback(schema, table_id, start, update, finish,
u_para, maat_instance->logger);
if (ret < 0) {
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return -1;
@@ -354,15 +384,17 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id,
return 0;
}
struct table_runtime *table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id);
size_t row_count = table_runtime_cached_row_count(table_rt);
if (row_count > 0) {
void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
size_t row_cnt = generic_plugin_runtime_cached_row_count(runtime, table_type);
if (row_cnt > 0) {
if (start != NULL) {
start(MAAT_RULE_UPDATE_TYPE_FULL, u_para);
}
for (size_t i = 0; i < row_count; i++) {
const char *line = table_runtime_get_cached_row(table_rt, i);
const char *line = generic_plugin_runtime_get_cached_row(runtime, table_type, i);
if (NULL == line) {
break;
}
@@ -380,25 +412,131 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id,
return 0;
}
int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int table_id,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp)
{
if (NULL == tbl_mgr || NULL == new_func || NULL == free_func || NULL == dup_func) {
assert(0);
log_error(tbl_mgr->logger, MODULE_MAAT_API,
"table(table_id:%d) %s failed: invalid parameter", __FUNCTION__);
return -1;
}
void *schema = table_manager_get_schema(tbl_mgr, table_id);
if (NULL == schema) {
log_error(tbl_mgr->logger, MODULE_MAAT_API,
"Error: %s, table(table_id:%d) is not registered", __FUNCTION__, table_id);
return -1;
}
struct ex_data_schema *ex_schema = NULL;
enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
switch (table_type) {
case TABLE_TYPE_PLUGIN:
ex_schema = plugin_table_get_ex_data_schema(schema);
if (NULL == ex_schema) {
log_error(tbl_mgr->logger, MODULE_MAAT_API,
"Error: %s, table(table_id:%d) is not a valid plugin table",
__FUNCTION__, table_id);
return -1;
}
plugin_table_set_ex_data_schema(schema, new_func, free_func,
dup_func, argl, argp, tbl_mgr->logger);
break;
case TABLE_TYPE_IP_PLUGIN:
ex_schema = ip_plugin_table_get_ex_data_schema(schema);
if (NULL == ex_schema) {
log_error(tbl_mgr->logger, MODULE_MAAT_API,
"Error: %s, table(table_id:%d) is not a valid ip_plugin table",
__FUNCTION__, table_id);
return -1;
}
ip_plugin_table_set_ex_data_schema(schema, new_func, free_func,
dup_func, argl, argp);
break;
default:
break;
}
return 0;
}
void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, enum table_type table_type,
int nr_worker_thread, long long version, struct log_handle *logger)
{
struct ex_data_schema *ex_data_schema = NULL;
struct ex_data_runtime *ex_data_rt = NULL;
switch (table_type) {
case TABLE_TYPE_PLUGIN:
ex_data_schema = plugin_table_get_ex_data_schema(schema);
ex_data_rt = plugin_runtime_get_ex_data_rt(runtime);
break;
case TABLE_TYPE_IP_PLUGIN:
ex_data_schema = ip_plugin_table_get_ex_data_schema(schema);
ex_data_rt = ip_plugin_runtime_get_ex_data_rt(runtime);
break;
default:
break;
}
ex_data_runtime_set_schema(ex_data_rt, ex_data_schema);
struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1);
ctx->table_id = table_schema_get_table_id(custom_schema);
ctx->ex_schema = ex_data_schema;
ex_data_runtime_set_ex_container_ctx(ex_data_rt, ctx);
size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt);
for (size_t i = 0; i < n_cached_row; i++) {
const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i);
switch (table_rt->table_type) {
case TABLE_TYPE_PLUGIN:
plugin_runtime_update_row(custom_rt, custom_schema, row, NULL, 0, 1);
break;
case TABLE_TYPE_IP_PLUGIN:
ip_plugin_runtime_update_row(custom_rt, custom_schema, NULL, NULL, 0, NULL, 1);
break;
default:
break;
}
}
ex_data_runtime_clear_row_cache(ex_data_rt);
switch (table_type) {
case TABLE_TYPE_PLUGIN:
plugin_runtime_commit(custom_rt);
break;
case TABLE_TYPE_IP_PLUGIN:
ip_plugin_runtime_commit(custom_rt);
break;
default:
break;
}
}
int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_id,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp)
{
struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
pthread_mutex_lock(&(maat_instance->background_update_mutex));
int ret = table_schema_set_ex_data_schema(table_schema, new_func, free_func, dup_func,
argl, argp, maat_instance->logger);
int ret = generic_plugin_table_ex_schema_register(maat_instance->tbl_mgr, table_id,
new_func, free_func, dup_func, argl, argp);
if (ret < 0) {
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return -1;
}
struct table_runtime *table_rt = NULL;
if (maat_instance->maat_rt != NULL) {
table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id);
table_runtime_commit_ex_data_schema(table_rt, table_schema, maat_instance->nr_worker_thread,
void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
generic_plugin_runtime_commit_ex_schema(runtime, schema, table_type, maat_instance->nr_worker_thread,
maat_instance->maat_rt->version, maat_instance->logger);
}
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
@@ -413,22 +551,29 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, co
return NULL;
}
struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
if (NULL == table_schema) {
enum table_type table_type = table_manager_get_table_type(maat_rt->ref_tbl_mgr, table_id);
void *runtime = table_manager_get_runtime(maat_rt->ref_tbl_mgr, table_id);
if (NULL == runtime) {
return NULL;
}
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id);
if (NULL == table_rt) {
return NULL;
struct ex_data_runtime *ex_data_rt = NULL;
switch (table_type) {
case TABLE_TYPE_PLUGIN:
ex_data_rt = plugin_runtime_get_ex_data_rt(runtime);
break;
case TABLE_TYPE_IP_PLUGIN:
ex_data_rt = ip_plugin_runtime_get_ex_data_rt(runtime);
break;
default:
break;
}
struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt);
if (NULL == ex_data_rt) {
return NULL;
}
return ex_data_runtime_dup_ex_data(ex_data_rt, key, key_len);
return ex_data_runtime_get_ex_data(ex_data_rt, key, key_len);
}
static void scan_count_inc(struct maat_state *mid)
@@ -444,11 +589,12 @@ struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id)
outer_state->maat_instance = maat_instance;
outer_state->district_id = DISTRICT_ANY;
outer_state->thread_id = (signed short)thread_id;
outer_state->compile_table_id = -1; //-1 means caller not specify compile table, use default compile table
return outer_state;
}
struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instance, int thread_id, int is_hit_item)
struct maat_state *grab_state(struct maat_state **state, struct maat *maat_instance, int thread_id)
{
struct maat_state *mid = *state;
@@ -466,12 +612,11 @@ struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instanc
alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1);
}
if (is_hit_item == 1) {
if (NULL == mid->compile_mid) {
mid->compile_mid = maat_hierarchy_compile_mid_new(maat_instance->maat_rt->hier, thread_id);
mid->compile_mid = maat_compile_state_new(thread_id);
alignment_int64_array_add(maat_instance->compile_mid_cnt, thread_id, 1);
}
}
return mid;
}
@@ -497,186 +642,27 @@ void scan_item_hit_wrapper_build(struct scan_item_hit_wrapper* wrapper, struct s
wrapper->virtual_table_ids = NULL;
}
struct compile_sort_para {
double evaluation_order;
int declared_clause_num;
int compile_id;
void *user;
};
static void compile_sort_para_set(struct compile_sort_para *para,
const struct maat_compile_rule *compile_relation, void *user)
{
para->compile_id=compile_relation->compile_id;
para->evaluation_order=compile_relation->evaluation_order;
para->declared_clause_num=compile_relation->declared_clause_num;
para->user = user;
}
static int compile_sort_para_compare(const struct compile_sort_para *a, const struct compile_sort_para *b)
{
//If both of compile rule's evaluation order are specified, compile rule with small evaluation order is priority.
if(a->evaluation_order!=0 && b->evaluation_order!=0)
{
if(a->evaluation_order - b->evaluation_order <0)
{
return -1;
}
else if(a->evaluation_order - b->evaluation_order >0)
{
return 1;
}
}
//If one of compile rule's evaluation order is zero, compile rule with big evaluation order is priority.
else if(a->evaluation_order + b->evaluation_order!= 0)
{
return (a->evaluation_order - b->evaluation_order >0) ? -1 : 1;
}
//If compile rule's execute sequences are not specified or equal.
if(a->declared_clause_num!=b->declared_clause_num)
{
return (a->declared_clause_num-b->declared_clause_num);
}
else
{
return (b->compile_id-a->compile_id);
}
}
static int compare_compile_rule(const void *a, const void *b)
{
const struct maat_compile_rule *ra=*(const struct Maat_compile_rule **)a;
const struct maat_compile_rule *rb=*(const struct Maat_compile_rule **)b;
struct compile_sort_para sa, sb;
compile_sort_para_set(&sa, ra, NULL);
compile_sort_para_set(&sb, rb, NULL);
return compile_sort_para_compare(&sa, &sb);
}
int item_compile(struct maat *maat_instance, struct maat_hierarchy_compile_mid *compile_mid,
const struct scan_item_hit_wrapper *item_hit_wrapper, int *result,
int size, int thread_id)
{
int is_last_item = item_hit_wrapper->is_last_item;
size_t item_hit_num = item_hit_wrapper->n_wrapped_item;
int scan_ret=0;
int i=0;
struct maat_compile_rule *compile_rule_array[size];
struct maat_compile_rule *compile_rule = NULL;
int virtual_table_id = 0;
struct maat_item_inner *item = NULL;
for (i = 0; (size_t)i < item_hit_num;i++) {
item = item_hit_wrapper->wrapped_items[i];
assert(item->magic_num == ITEM_RULE_MAGIC);
if (item_hit_wrapper->virtual_table_ids) {
virtual_table_id = item_hit_wrapper->virtual_table_ids[i];
} else {
virtual_table_id = item_hit_wrapper->virtual_table_id;
}
maat_hierarchy_compile_mid_update(feather->scanner->hier, compile_mid, item->item_id, virtual_table_id, item_hit_wrapper->Nth_scan, i);
}
scan_ret = maat_hierarchy_item_compile(feather->scanner->hier, compile_mid, is_last_item, (void **)compile_rule_array, size);
//Maat_hierarchy is rwlock protected, it always returns non-NULL compile_rule.
if (scan_ret > 1) {
qsort(compile_rule_array, scan_ret, sizeof(struct maat_compile_rule *),
compare_compile_rule);
}
for (i = 0; i < scan_ret && i < size; i++) {
compile_rule = compile_rule_array[i];
assert(compile_rule->magic_num == COMPILE_RULE_MAGIC);
result[i] = compile_rule->compile_id;
/*
fill_maat_rule(&(result[i]), &(compile_rule->head), compile_rule->service_defined,
compile_rule->head.serv_def_len);*/
}
if (scan_ret > 0) {
alignment_int64_array_add(feather->hit_cnt, thread_id, 1);
}
if (item_hit_num == 0 && scan_ret > 0) {
alignment_int64_array_add(feather->not_grp_hit_cnt, thread_id, 1);
}
return MIN(scan_ret, size);
}
int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
unsigned int intval, int results[], size_t *n_result,
struct maat_state **state)
unsigned int intval, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state)
{
return 0;
}
static int ip_scan_data_set(struct ip_addr *scan_data, struct addr_2tuple *addr, enum component_table_type child_type)
static int ip_scan_data_set(struct ip_addr *scan_data, const char *ip_addr)
{
switch (addr->type) {
case IP_TYPE_V4:
scan_data->ip_type = IP_TYPE_V4;
switch (child_type) {
case COMPONENT_TABLE_TYPE_SIP:
scan_data->ipv4 = ntohl(addr->ipv4.sip);
break;
case COMPONENT_TABLE_TYPE_DIP:
scan_data->ipv4 = ntohl(addr->ipv4.dip);
break;
default:
assert(0);
return -1;
}
break;
case IP_TYPE_V6:
scan_data->ip_type = IP_TYPE_V6;
switch (child_type) {
case COMPONENT_TABLE_TYPE_SIP:
memcpy(scan_data->ipv6, addr->ipv6.sip, sizeof(addr->ipv6.sip));
ipv6_ntoh(scan_data->ipv6);
break;
case COMPONENT_TABLE_TYPE_DIP:
memcpy(scan_data->ipv6, addr->ipv6.dip, sizeof(addr->ipv6.dip));
ipv6_ntoh(scan_data->ipv6);
break;
default:
assert(0);
return -1;
}
break;
default:
assert(0);
return -1;
}
return 0;
}
static int ip_composition_scan(int thread_id, struct addr_2tuple *addr,
int parent_table_id, enum component_table_type child_type,
int *virtual_table_id,
static int ip_composition_scan(int thread_id, const char *ip_addr,
int parent_table_id, int *virtual_table_id,
struct table_schema_manager *table_schema_mgr,
struct table_runtime_manager *table_rt_mgr,
struct scan_result *item_results, size_t n_result_array)
int *group_id_array, size_t n_group_id_array,
struct maat_state *state)
{
int child_table_id = 0;
if (child_type == COMPONENT_TABLE_TYPE_NONE) {
child_table_id = parent_table_id;
child_type = COMPONENT_TABLE_TYPE_SESSION;
} else {
child_table_id = table_schema_manager_get_child_table_id(table_schema_mgr, parent_table_id, child_type);
}
if (child_table_id < 0) {
return 0;
}
struct table_schema *real_table = table_schema_get_by_scan_type(table_schema_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id);
if (NULL == real_table) {
return 0;
@@ -688,7 +674,7 @@ static int ip_composition_scan(int thread_id, struct addr_2tuple *addr,
}
int table_id = table_schema_get_table_id(real_table);
struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id);
struct table_runtime *table_rt = table_manager_get_runtime(table_rt_mgr, table_id);
size_t rule_num = table_runtime_rule_count(table_rt);
if (0 == rule_num) {
return 0;
@@ -697,122 +683,150 @@ static int ip_composition_scan(int thread_id, struct addr_2tuple *addr,
struct ip_addr scan_data;
memset(&scan_data, 0, sizeof(struct ip_addr));
ip_scan_data_set(&scan_data, addr, child_type);
size_t hit_cnt = 0;
int ret = table_runtime_scan_ip(table_rt, thread_id, &scan_data, item_results, &hit_cnt, n_result_array);
ip_scan_data_set(&scan_data, ip_addr);
size_t hit_group_cnt = 0;
int ret = table_runtime_scan_ip(table_rt, thread_id, &scan_data, group_id_array, n_group_id_array, *virtual_table_id, state);
if (ret < 0) {
return -1;
}
return hit_cnt;
return hit_group_cnt;
}
int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct addr_2tuple *addr,
int results[], size_t *n_result, struct maat_state **state)
int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id,
uint32_t ip_addr, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state)
{
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
(thread_id < 0) || (NULL == addr) || (NULL == results) || (NULL == n_result) ||
(thread_id < 0) || (NULL == ip_addr) || (NULL == results) || (0 == n_result) ||
(NULL == state)) {
return -1;
}
struct maat_state *mid = NULL;
mid = grab_mid(state, maat_instance, thread_id, 0);
mid = grab_state(state, maat_instance, thread_id);
scan_count_inc(mid);
struct maat_runtime *maat_rt = maat_instance->maat_rt;
if (NULL == maat_rt) {
return 0;
}
struct table_runtime_manager *table_rt_mgr = maat_rt->table_rt_mgr;
struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id);
enum table_type table_type = table_runtime_get_type(table_rt);
if (table_type == TABLE_TYPE_INVALID) {
maat_instance->scan_err_cnt++;
return -1;
}
int item_ret = 0;
int virtual_table_id = 0;
struct scan_result *item_result = maat_rt->item_result_buff + thread_id * MAX_SCANNER_HIT_NUM;
int item_hit_cnt = 0;
int item_result_virtual_table_ids[MAX_SCANNER_HIT_NUM];
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
maat_runtime_ref_inc(maat_rt, thread_id);
struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id,
SCAN_TYPE_IP, &virtual_table_id);
if (NULL == table_schema) {
return MAAT_ERR;
}
if (table_type == TABLE_TYPE_COMPOSITION) {
/*
item_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_SIP, &virtual_table_id,
struct maat_runtime *maat_rt = maat_instance->maat_rt;
int group_result[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int group_hit_cnt = 0;
int group_result_virtual_table_ids[MAX_SCANNER_HIT_GROUP_NUM];
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
int group_ret = -1;
group_ret = ip_composition_scan(thread_id, ip_addr, table_id, &virtual_table_id,
maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr,
item_result + item_hit_cnt, MAX_SCANNER_HIT_NUM - item_hit_cnt);
item_hit_cnt += item_ret; */
enum component_table_type childs[3] = {COMPONENT_TABLE_TYPE_SIP, COMPONENT_TABLE_TYPE_DIP, COMPONENT_TABLE_TYPE_SESSION};
for (int i = 0; i < 3; i++) {
item_ret = ip_composition_scan(thread_id, addr, table_id, childs[i], &virtual_table_id,
maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr,
item_result + item_hit_cnt, MAX_SCANNER_HIT_NUM - item_hit_cnt);
if (item_ret < 0) {
group_result + group_hit_cnt, MAX_SCANNER_HIT_COMPILE_NUM - group_hit_cnt, mid);
if (group_ret < 0) {
maat_instance->scan_err_cnt++;
} else if (0 == group_ret) {
return MAAT_OK;
} else {
for (int j = 0; j < item_ret; j++) {
item_result_virtual_table_ids[item_hit_cnt++] = virtual_table_id;
}
}
}
} else {
item_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_NONE, &virtual_table_id,
maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr,
item_result + item_hit_cnt, MAX_SCANNER_HIT_NUM - item_hit_cnt);
if (item_ret < 0) {
maat_instance->scan_err_cnt++;
} else {
item_hit_cnt += item_ret;
}
group_hit_cnt += group_ret;
}
int compile_ret = 0;
struct scan_item_hit_wrapper item_hit_wrapper;
if (item_hit_cnt > 0 || scan_state_should_compile_NOT(mid)) {
mid = grab_mid(state, maat_instance, thread_id, 1);
scan_item_hit_wrapper_build(&item_hit_wrapper, item_result, item_hit_cnt, -1,
mid->is_last_scan, virtual_table_id, mid->scan_cnt);
if (table_type == TABLE_TYPE_COMPOSITION) {
item_hit_wrapper.virtual_table_ids = item_result_virtual_table_ids;
// come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT
struct table_runtime *g2g_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int n_top_group_ids = group2group_runtime_get_top_groups(g2g_rt, group_result, group_hit_cnt, top_group_ids);
size_t n_all_group_ids = 0;
if (n_top_group_ids > 0) {
n_all_group_ids = group_hit_cnt + n_top_group_ids;
} else {
n_all_group_ids = group_hit_cnt;
}
compile_ret = item_compile(maat_instance, mid->compile_mid,
&item_hit_wrapper,
results, item_hit_cnt,
thread_id);
assert(mid->is_last_scan < 2);
if (mid->is_last_scan == 1) {
mid->is_last_scan = 2;
}
}
maat_runtime_ref_dec(maat_rt, thread_id);
int all_group_ids[n_all_group_ids] = {-1};
// maat state find compile_table_id, if not found, maat_instance->default_compile_table_id
struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id);
int n_compile_ids = compile_runtime_match(compile_table_rt, all_group_ids, n_all_group_ids, results, n_result, mid);
*n_hit_result = n_compile_ids;
if (compile_ret == 0 && item_hit_cnt > 0) {
return -2;
if (n_compile_ids > 0) {
return MAAT_HIT;
} else {
return MAAT_HALF_HIT;
}
*n_result = compile_ret;
return compile_ret;
}
int maat_scan_ipv6(struct maat *instance, int table_id, int thread_id,
uint8_t *ip_addr, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state)
{
return 0;
}
int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
const char *data, size_t data_len, int results[], size_t *n_result,
struct maat_state **state)
const char *data, size_t data_len, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state)
{
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
(thread_id < 0) || (NULL == data) || (0 == data_len) || (NULL == results) ||
(NULL == n_result) || (NULL == state)) {
return -1;
(0 == n_result) || (NULL == state)) {
return MAAT_ERR;
}
struct table_runtime_manager *table_rt_mgr = maat_instance->maat_rt->table_rt_mgr;
struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id);
struct maat_state *mid = NULL;
mid = grab_state(state, maat_instance, thread_id);
scan_count_inc(mid);
return table_runtime_scan_string(table_rt, thread_id, data, data_len, results, n_result);
int virtual_table_id = 0;
struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id,
SCAN_TYPE_STRING, &virtual_table_id);
if (NULL == table_schema) {
return MAAT_ERR;
}
struct table_runtime *table_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int group_hit_cnt = table_runtime_scan_string(table_rt, thread_id, data, data_len, group_ids, MAX_SCANNER_HIT_GROUP_NUM,
virtual_table_id, mid);
if (group_hit_cnt <= 0) {
return MAAT_OK;
}
// come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT
struct table_runtime *g2g_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int n_top_group_ids = group2group_runtime_get_top_groups(g2g_rt, group_ids, group_hit_cnt, top_group_ids);
size_t n_all_group_ids = 0;
if (n_top_group_ids > 0) {
n_all_group_ids = group_hit_cnt + n_top_group_ids;
} else {
n_all_group_ids = group_hit_cnt;
}
int all_group_ids[n_all_group_ids] = {-1};
int i = 0, j = 0, k = 0;
for (i = 0; i < group_hit_cnt; i++) {
all_group_ids[i] = group_ids[i];
}
for (j = i; j < n_all_group_ids; j++, k++) {
all_group_ids[j] = top_group_ids[k];
}
// maat state find compile_table_id, if not found, maat_instance->default_compile_table_id
struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id);
int n_compile_ids = compile_runtime_match(compile_table_rt, all_group_ids, n_all_group_ids, results, n_result, mid);
*n_hit_result = n_compile_ids;
if (n_compile_ids > 0) {
return MAAT_HIT;
} else {
return MAAT_HALF_HIT;
}
}
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id)
@@ -831,25 +845,112 @@ void maat_scan_stream_close(struct maat_stream **stream)
}
int maat_state_set(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt,
const void *value, int size)
int maat_state_set_scan_district(struct maat *maat_instance, struct maat_state **state, const char *district, size_t district_len)
{
if (NULL == maat_instance->maat_rt || NULL == district || district_len <= 0) {
return -1;
}
struct maat_state *mid = grab_state(state, maat_instance, -1);
int map_ret = maat_kv_read_unNull(maat_instance->maat_rt->district_map, district, district_len, &(mid->district_id));
if (map_ret < 0) {
mid->district_id = DISTRICT_UNKNOWN;
}
mid->is_set_district = 1;
return 0;
}
//return >=0 if success, return -1 when failed;
int maat_state_get(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt,
void *value, int size)
int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **state)
{
if (NULL == maat_instance->maat_rt) {
return -1;
}
struct maat_state *mid = grab_state(state, maat_instance, -1);
assert(mid->is_last_scan == 0);
mid->is_last_scan = 1;
return 0;
}
int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, int compile_table_id)
{
if (NULL == maat_instance->maat_rt) {
return -1;
}
struct maat_state *mid = grab_state(state, maat_instance, -1);
mid->compile_table_id = compile_table_id;
return 0;
}
size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *mid, struct maat_hit_path *paths, size_t n_path)
{
int compile_table_id = -1;
if (mid->compile_table_id == -1) {
compile_table_id = maat_instance->default_compile_table_id;
} else {
compile_table_id = mid->compile_table_id;
}
compile_runtime_get_hit_paths(maat_instance->maat_rt->table_rt_mgr, compile_table_id, maat_instance->g2g_table_id, mid, paths, n_path);
return 0;
}
size_t maat_get_hit_objects(struct maat_compile_state *compile_state, struct maat_hit_object *objs, size_t n_objs)
{
return 0;
}
int maat_state_get_hit_paths(struct maat *maat_instance, struct maat_state **state, struct maat_hit_path *paths, size_t n_path)
{
struct maat_state *mid = NULL;
struct maat_hit_path *paths;
mid = grab_state(state, maat_instance, 0);
if (NULL == mid->compile_mid || NULL == maat_instance->maat_rt) {
return 0;
}
int n_read = maat_get_hit_paths(mid->maat_instance, mid, paths, n_path);
return n_read;
}
int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state, struct maat_hit_object *objs, size_t n_obj)
{
return 0;
}
void maat_state_free(struct maat_state **state)
{
struct maat_state *mid = NULL;
if (NULL == *state) {
return;
}
mid = *state;
if (mid->thread_id >= 0) {
alignment_int64_array_add(mid->maat_instance->outer_mid_cnt, mid->thread_id, -1);
}
if (mid->compile_mid != NULL) {
maat_compile_state_free(mid->compile_mid);
mid->compile_mid = NULL;
alignment_int64_array_add(mid->maat_instance->compile_mid_cnt, mid->thread_id, -1);
}
mid->maat_instance = NULL;
free(mid);
*state = NULL;
}
int maat_matched_compile_id(struct maat *instance, struct maat_matched *matched)
int maat_hit_object_compile_id(struct maat *instance, struct maat_hit_object *obj)
{
return 0;
}

19
src/maat_command.cpp
View File

@@ -19,7 +19,6 @@
#include "maat_rule.h"
#include "hiredis/hiredis.h"
#include "maat_config_monitor.h"
#include "maat_table_schema.h"
#define MODULE_MAAT_COMMAND module_name_str("maat.command")
@@ -305,7 +304,7 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
struct serial_rule *s_rule = ALLOC(struct serial_rule, 1);
int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, line_rule->table_name);
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, line_rule->table_name);
if (table_id < 0) {
log_error(maat_instance->logger, MODULE_MAAT_COMMAND, "Command set line id %d failed: unknown table %s",
line_rule->rule_id, line_rule->table_name);
@@ -313,14 +312,8 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
return -1;
}
struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
if (!table_schema) {
FREE(s_rule);
return -1;
}
int valid_flag_column = table_schema_get_valid_flag_column(table_schema);
if (valid_flag_column < 0) {
int valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id);
if (valid_column < 0) {
log_error(maat_instance->logger, MODULE_MAAT_COMMAND,
"Command set line id %d failed: table %s is not a plugin or ip_plugin table",
line_rule->rule_id, line_rule->table_name);
@@ -328,9 +321,9 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
return -1;
}
enum table_type table_type = table_schema_get_table_type(table_schema);
int valid_offset = maat_cmd_get_valid_flag_offset(line_rule->table_line, table_type, valid_flag_column);
int is_valid = atoi(line_rule->table_line+valid_offset);
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
int valid_offset = maat_cmd_get_valid_flag_offset(line_rule->table_line, table_type, valid_column);
int is_valid = atoi(line_rule->table_line + valid_offset);
if (line_rule->expire_after > 0) {
absolute_expire_time = server_time + line_rule->expire_after;
}

1580
src/maat_compile.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

98
src/maat_ex_data.cpp
View File

@@ -17,11 +17,18 @@
#include "rcu_hash.h"
#include "utils.h"
#include "maat_utils.h"
#include "maat_table_schema.h"
#include "maat_ex_data.h"
#define MODULE_EX_DATA module_name_str("maat.ex_data")
struct ex_data_schema {
maat_plugin_ex_new_func_t *new_func;
maat_plugin_ex_free_func_t *free_func;
maat_plugin_ex_dup_func_t *dup_func;
long argl;
void *argp;
};
struct ex_data_runtime {
UT_array *cache_rows;
size_t cache_row_num;
@@ -30,6 +37,8 @@ struct ex_data_runtime {
struct rcu_hash_table *htable;
struct ex_data_schema *ex_schema;
int table_id;
struct log_handle *logger;
};
void cache_row_free(void *p)
@@ -39,13 +48,15 @@ void cache_row_free(void *p)
UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn)
struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn,
struct log_handle *logger)
{
struct ex_data_runtime *ex_data_rt = ALLOC(struct ex_data_runtime, 1);
utarray_new(ex_data_rt->cache_rows, &ut_cache_row_icd);
ex_data_rt->htable = rcu_hash_new(data_free_fn);
ex_data_rt->table_id = table_id;
ex_data_rt->logger = logger;
return ex_data_rt;
}
@@ -117,6 +128,27 @@ void ex_data_runtime_clear_row_cache(struct ex_data_runtime *ex_data_rt)
ex_data_rt->cache_size = 0;
}
struct ex_data_schema *ex_data_schema_new(maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp)
{
struct ex_data_schema *ex_schema = ALLOC(struct ex_data_schema, 1);
ex_schema->new_func = new_func;
ex_schema->free_func = free_func;
ex_schema->dup_func = dup_func;
ex_schema->argl = argl;
ex_schema->argp = argp;
return ex_schema;
}
void ex_data_schema_free(struct ex_data_schema *ex_schema)
{
FREE(ex_schema);
}
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema)
{
ex_data_rt->ex_schema = schema;
@@ -141,29 +173,73 @@ void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char
return ex_data;
}
void ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct ex_data_container *ex_container)
struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data)
{
struct ex_data_container *tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len);
if (tmp_container != NULL) {
struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1);
ex_container->ex_data = ex_data;
ex_container->custom_data = custom_data;
return ex_container;
}
void ex_data_container_free(void *ctx, void *data)
{
if (NULL == ctx || NULL == data) {
return;
}
struct ex_container_ctx *container_ctx = (struct ex_container_ctx *)ctx;
long argl = container_ctx->ex_schema->argl;
void *argp = container_ctx->ex_schema->argp;
struct ex_data_container *ex_container = (struct ex_data_container *)data;
if (ex_container->ex_data != NULL && container_ctx->ex_schema->free_func != NULL) {
container_ctx->ex_schema->free_func(container_ctx->table_id, &(ex_container->ex_data), argl, argp);
}
if (ex_container->custom_data != NULL && container_ctx->custom_data_free != NULL) {
container_ctx->custom_data_free(ex_container->custom_data);
}
FREE(ex_container);
}
int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len,
struct ex_data_container *ex_container)
{
struct ex_data_container *tmp_container = NULL;
tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len);
if (tmp_container != NULL) {
log_error(ex_data_rt->logger, MODULE_EX_DATA,
"ex_data_runtime add ex container error: already exist same key:%s", key);
return -1;
}
rcu_hash_add(ex_data_rt->htable, key, key_len, ex_container);
return 0;
}
void ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len,
struct log_handle *logger)
int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len)
{
struct ex_data_container *tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len);
struct ex_data_container *tmp_container = NULL;
tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len);
if (NULL == tmp_container) {
log_error(logger, MODULE_EX_DATA, "ex data del error: no such key:%s", key);
return;
log_error(ex_data_rt->logger, MODULE_EX_DATA,
"ex_data_runtime del ex container error: no such key:%s", key);
return -1;
}
rcu_hash_del(ex_data_rt->htable, key, key_len);
return 0;
}
void *ex_data_runtime_dup_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len)
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len)
{
struct ex_data_container *ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len);
if (NULL == ex_container) {

699
src/maat_expr.cpp Normal file
View File

@@ -0,0 +1,699 @@
/*
**********************************************************************************************
* File: maat_expr.cpp
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include <string.h>
#include <assert.h>
#include "utils.h"
#include "maat_expr.h"
#include "adapter_hs.h"
#include "maat_utils.h"
#include "maat_kv.h"
#include "maat_limits.h"
#include "rcu_hash.h"
#include "maat_rule.h"
#include "maat_garbage_collection.h"
#define MAX_DISTRICT_STR 128
#define MODULE_EXPR module_name_str("maat.expr")
struct expr_schema {
int item_id_column;
int group_id_column;
int district_column;
int keywords_column;
int expr_type_column;
int match_method_column;
int is_hexbin_column;
enum hs_scan_mode scan_mode; /* adapter_hs scan mode */
};
enum expr_type {
EXPR_TYPE_STRING = 0,
EXPR_TYPE_AND,
EXPR_TYPE_REGEX,
EXPR_TYPE_MAX
};
enum match_method {
MATCH_METHOD_SUB = 0,
MATCH_METHOD_RIGHT,
MATCH_METHOD_LEFT,
MATCH_METHOD_COMPLETE,
MATCH_METHOD_MAX
};
struct expr_item {
int item_id;
int group_id;
char district[MAX_DISTRICT_STR];
char keywords[MAX_KEYWORDS_STR];
enum expr_type expr_type;
enum match_method match_method;
int is_hexbin;
int is_case_sensitive;
};
struct expr_runtime {
enum hs_scan_mode scan_mode;
struct adapter_hs *hs;
struct adapter_hs_stream *hs_stream;
struct rcu_hash_table *htable;
struct group2group_runtime *ref_g2g_rt;
uint32_t rule_num;
uint32_t updating_rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
// long long *scan_cnt;
// long long *hit_cnt;
// long long *not_grp_hit_cnt;
// long long *stream_num;
};
enum expr_type int_to_expr_type(int expr_type)
{
enum expr_type type = EXPR_TYPE_MAX;
switch (expr_type) {
case 0:
type = EXPR_TYPE_STRING;
break;
case 1:
type = EXPR_TYPE_AND;
break;
case 2:
type = EXPR_TYPE_REGEX;
break;
default:
break;
}
return type;
}
enum match_method int_to_match_method_type(int match_method_type)
{
enum match_method type = MATCH_METHOD_MAX;
switch (match_method_type) {
case 0:
type = MATCH_METHOD_SUB;
break;
case 1:
type = MATCH_METHOD_RIGHT;
break;
case 2:
type = MATCH_METHOD_LEFT;
break;
case 3:
type = MATCH_METHOD_COMPLETE;
break;
default:
break;
}
return type;
}
struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schema,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
int db_hexbin = -1;
int expr_type = -1;
int match_method_type = -1;
struct expr_item *expr_item = ALLOC(struct expr_item, 1);
int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
goto error;
}
expr_item->item_id = atoi(line + column_offset);
ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) {
goto error;
}
expr_item->group_id = atoi(line + column_offset);
//TODO
#if 0
if (table_item->table_type == TABLE_TYPE_EXPR_PLUS) {
ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
if (column_len >= MAX_DISTRICT_STR) {
log_error(logger, MODULE_EXPR,
"update error: expr table[%s]:item_id[%d] district length too long",
table_name, expr_item->item_id);
return -1;
}
memcpy(expr_item->district, (line + column_offset), column_len);
}
#endif
ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len);
if (ret < 0) {
goto error;
}
if (column_len >= MAX_KEYWORDS_STR) {
log_error(logger, MODULE_EXPR,
"update error: expr table[%s]:item_id[%d] keywords length too long",
table_name, expr_item->item_id);
goto error;
}
memcpy(expr_item->keywords, (line + column_offset), column_len);
ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len);
if (ret < 0) {
goto error;
}
expr_type = atoi(line + column_offset);
expr_item->expr_type = int_to_expr_type(expr_type);
ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len);
if (ret < 0) {
goto error;
}
match_method_type = atoi(line + column_offset);
expr_item->match_method = int_to_match_method_type(match_method_type);
ret = get_column_pos(line, expr_schema->is_hexbin_column, &column_offset, &column_len);
if (ret < 0) {
goto error;
}
db_hexbin = atoi(line + column_offset);
switch (db_hexbin) {
case 0:
expr_item->is_hexbin = FALSE;
expr_item->is_case_sensitive = FALSE;
break;
case 1:
expr_item->is_hexbin = TRUE;
expr_item->is_case_sensitive = TRUE;
break;
case 2:
expr_item->is_hexbin = FALSE;
expr_item->is_case_sensitive = TRUE;
break;
default:
log_error(logger, MODULE_EXPR,
"update error: expr table[%s]:item_id[%d] invalid hexbin value:%d",
table_name, expr_item->item_id, db_hexbin);
goto error;
}
return expr_item;
error:
FREE(expr_item);
return NULL;
}
void expr_item_free(struct expr_item *expr_item)
{
FREE(expr_item);
}
void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
{
int read_cnt = 0;
struct expr_schema *expr_schema = ALLOC(struct expr_schema, 1);
struct maat_kv_store *scan_mode_map = maat_kv_store_new();
maat_kv_register(scan_mode_map, "block", HS_SCAN_MODE_BLOCK);
maat_kv_register(scan_mode_map, "stream", HS_SCAN_MODE_STREAM);
int ret = -1;
cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_EXPR, "table %s has no custom column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "scan_mode");
if (custom_item != NULL && custom_item->type == cJSON_String) {
ret = maat_kv_read(scan_mode_map, custom_item->valuestring, (int*)&(expr_schema->scan_mode));
if (ret < 0) {
log_error(logger, MODULE_EXPR, "scan_mode %s illegal", custom_item->valuestring);
goto error;
}
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->item_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->group_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "keywords");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->keywords_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "district");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->district_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "expr_type");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->expr_type_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "match_method");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->match_method_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "is_hexbin");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->is_hexbin_column = custom_item->valueint;
read_cnt++;
}
if (read_cnt < 8) {
goto error;
}
maat_kv_store_free(scan_mode_map);
return expr_schema;
error:
maat_kv_store_free(scan_mode_map);
FREE(expr_schema);
return NULL;
}
void expr_schema_free(void *expr_schema)
{
FREE(expr_schema);
}
void *expr_runtime_new(void *expr_schema, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == expr_schema) {
return NULL;
}
struct expr_schema *schema = (struct expr_schema *)expr_schema;
struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1);
expr_rt->htable = rcu_hash_new(expr_ex_data_free);
expr_rt->scan_mode = schema->scan_mode;
expr_rt->item_user_data_free = maat_item_inner_free;
expr_rt->ref_garbage_bin = garbage_bin;
expr_rt->logger = logger;
// expr_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
// expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
// expr_rt->not_grp_hit_cnt = alignment_int64_array_alloc(max_thread_num);
// expr_rt->stream_num = alignment_int64_array_alloc(max_thread_num);
return expr_rt;
}
void expr_runtime_free(void *expr_runtime)
{
if (NULL == expr_runtime) {
return;
}
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
if (expr_rt->hs != NULL) {
adapter_hs_destroy(expr_rt->hs);
expr_rt->hs = NULL;
}
if (expr_rt->hs_stream != NULL) {
adapter_hs_stream_close(expr_rt->hs_stream);
expr_rt->hs_stream = NULL;
}
if (expr_rt->htable != NULL) {
rcu_hash_free(expr_rt->htable);
expr_rt->htable = NULL;
}
struct maat_item *item = NULL, *tmp = NULL;
HASH_ITER(hh, expr_rt->item_hash, item, tmp) {
HASH_DELETE(hh, expr_rt->item_hash, item);
maat_item_free(item, expr_rt->item_user_data_free);
}
FREE(expr_rt);
}
int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len,
and_expr_t *expr_rule, int is_valid, struct log_handle *logger)
{
void *data = NULL;
if (0 == is_valid) {
//delete
data = rcu_hash_find(expr_rt->htable, key, key_len);
if (NULL == data) {
log_error(logger, MODULE_EXPR,
"the key of expr rule not exist, can't be deleted, expr_id:%d", expr_rule->expr_id);
return -1;
}
rcu_hash_del(expr_rt->htable, key, key_len);
} else {
//add
data = rcu_hash_find(expr_rt->htable, key, key_len);
if (data != NULL) {
log_error(logger, MODULE_EXPR,
"the key of expr rule already exist, can't be added, expr_id:%d", expr_rule->expr_id);
return -1;
}
rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule);
}
return 0;
}
and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handle *logger)
{
size_t i = 0;
size_t sub_expr_cnt = 0;
char *pos = NULL;
char *saveptr = NULL;
char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM];
and_expr_t *expr_rule = ALLOC(and_expr_t, 1);
switch (expr_item->expr_type) {
case EXPR_TYPE_AND:
case EXPR_TYPE_REGEX:
for (i = 0, pos = expr_item->keywords; ; i++, pos = NULL) {
char *tmp = strtok_r_esc(pos, '&', &saveptr);
if (NULL == tmp) {
break;
}
if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
log_error(logger, MODULE_TABLE_RUNTIME, "expr item_id:%d too many patterns",
expr_item->item_id);
return NULL;
}
sub_key_array[i] = tmp;
if (expr_item->expr_type == EXPR_TYPE_REGEX) {
sub_key_array[i] = str_unescape_and(sub_key_array[i]);
} else {
sub_key_array[i] = str_unescape(sub_key_array[i]);
}
}
sub_expr_cnt = i;
break;
case EXPR_TYPE_STRING:
sub_expr_cnt = 1;
sub_key_array[0] = expr_item->keywords;
sub_key_array[0] = str_unescape(sub_key_array[0]);
break;
default:
break;
}
for (i = 0; i < sub_expr_cnt; i++) {
expr_rule->expr_id = expr_item->item_id;
expr_rule->patterns[i].pat = ALLOC(char, strlen(sub_key_array[i]));
memcpy(expr_rule->patterns[i].pat, sub_key_array[i], strlen(sub_key_array[i]));
expr_rule->patterns[i].pat_len = strlen(sub_key_array[i]);
expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type);
}
expr_rule->n_patterns = sub_expr_cnt;
return expr_rule;
}
int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column)
{
if (NULL == expr_runtime || NULL == expr_schema) {
return -1;
}
int ret = -1;
struct maat_item_inner *u_para = NULL;
struct maat_item *item = NULL;
and_expr_t *expr_rule = NULL;
struct expr_schema *schema = (struct expr_schema *)expr_schema;
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
int item_id = get_column_value(line, schema->item_id_column);
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
//delete
HASH_FIND_INT(expr_rt->item_hash, &item_id, item);
if (NULL == item) {
return -1;
}
u_para = (struct maat_item_inner *)item->user_data;
item->user_data = NULL;
if (NULL == u_para) {
return -1;
}
HASH_DELETE(hh, expr_rt->item_hash, item);
maat_garbage_bagging(expr_rt->ref_garbage_bin, u_para, (void (*)(void *))maat_item_inner_free);
} else {
//add
HASH_FIND_INT(expr_rt->item_hash, &item_id, item);
if (item) {
log_error(expr_rt->logger, MODULE_EXPR,
"expr runtime add item %d to item_hash failed, already exist", item_id);
return -1;
}
struct expr_item *expr_item = expr_item_new(line, schema, expr_rt->logger);
if (NULL == expr_item) {
log_error(expr_rt->logger, MODULE_EXPR, "expr line %s to item failed", line);
return -1;
}
// TODO: by luis
//int district_id = get_district_id(maat_rt, expr_item->district);
int district_id = -1;
u_para = maat_item_inner_new(expr_item->group_id, item_id, district_id);
item = maat_item_new(item_id, group_id, u_para);
HASH_ADD_INT(expr_rt->item_hash, item_id, item);
expr_rule = expr_item_to_expr_rule(expr_item, expr_rt->logger);
expr_item_free(expr_item);
if (NULL == expr_rule) {
log_error(expr_rt->logger, MODULE_EXPR, "transform expr table:%s item to expr_rule failed, item_id:%d",
table_name, item_id);
return -1;
}
}
char *key = (char *)&item_id;
ret = expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule, is_valid, expr_rt->logger);
if (ret < 0) {
if (expr_rule != NULL) {
expr_rule_free(expr_rule);
}
return -1;
} else {
if (0 == is_valid) {
expr_rt->rule_num--;
} else {
expr_rt->rule_num++;
}
}
return 0;
}
int expr_runtime_commit(void *expr_runtime)
{
if (NULL == expr_runtime) {
return -1;
}
int ret = 0;
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list_updating_data(expr_rt->htable, &ex_data_array);
if (0 == rule_cnt) {
FREE(ex_data_array);
return 0;
}
and_expr_t *rules = ALLOC(and_expr_t, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(and_expr_t *)ex_data_array[i];
}
struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL;
log_info(expr_rt->logger, MODULE_EXPR,
"committing %zu expr rules for rebuilding adapter_hs engine", rule_cnt);
new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, nr_worker_thread, rules, rule_cnt, expr_rt->logger);
if (NULL == new_adapter_hs) {
log_error(expr_rt->logger, MODULE_EXPR,
"rebuild adapter_hs engine failed when update %zu expr rules", rule_cnt);
ret = -1;
}
old_adapter_hs = expr_rt->hs;
expr_rt->hs = new_adapter_hs;
maat_garbage_bagging(table_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy);
rcu_hash_commit(expr_rt->htable);
expr_rt->rule_num = rcu_hash_count(expr_rt->htable);
rule_cnt = rcu_hash_updating_count(expr_rt->htable);
assert(rule_cnt == 0);
FREE(rules);
FREE(ex_data_array);
return ret;
}
int expr_runtime_updating_flag(struct expr_runtime *expr_rt)
{
return rcu_hash_updating_flag(expr_rt->htable);
}
void expr_rule_free(and_expr_t *expr_rule)
{
if (NULL == expr_rule) {
return;
}
for (size_t i = 0; i < expr_rule->n_patterns; i++) {
FREE(expr_rule->patterns[i].pat);
}
FREE(expr_rule);
}
void expr_ex_data_free(void *user_ctx, void *data)
{
and_expr_t *expr_rule = (and_expr_t *)data;
expr_rule_free(expr_rule);
}
int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const char *data, size_t data_len,
int group_id_array[], size_t n_group_id_array, int virtual_table_id, struct maat_state *state)
{
if (NULL == table_rt) {
return -1;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
size_t n_hit_item = 0;
int ret = adapter_hs_scan(table_rt->expr_rt.hs, thread_id, data, data_len, hit_item_ids, &n_hit_item);
if (ret < 0) {
return -1;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
struct maat_compile_state *compile_state = state->compile_mid;
//tranform item_id to group_id
struct maat_item *item = NULL;
size_t n_group_id = 0;
size_t i = 0;
for (i = 0; i < n_hit_item; i++) {
HASH_FIND_INT(table_rt->item_hash, &(hit_item_ids[i]), item);
assert(item != NULL);
if (!item) {
// should not come here
continue;
}
if (n_group_id >= n_group_id_array) {
n_group_id = n_group_id_array;
//Prevent group_id_array out of bounds
} else {
group_id_array[n_group_id++] = item->group_id;
}
maat_compile_state_update_hit_path(compile_state, hit_item_ids[i], item->group_id, virtual_table_id, state->scan_cnt, i);
}
// literal_id{group_id,vt_id} to clause_id
// STEP 1: get compile table runtime
int compile_table_id = -1;
if (state->compile_table_id == -1) {
compile_table_id = state->maat_instance->default_compile_table_id;
} else {
compile_table_id = state->compile_table_id;
}
struct maat_runtime *maat_rt = state->maat_instance->maat_rt;
struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, compile_table_id);
assert(compile_table_rt->table_type == TABLE_TYPE_COMPILE);
// STEP 2: get the specified compile table's hit clause_id array by literal_id
for (i = 0; i < n_group_id; i++) {
maat_compile_state_update_hit_clause(compile_state, &(compile_table_rt->compile_rt.compile_hash), group_id_array[i], virtual_table_id);
}
return n_group_id;
}
void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
{
if (NULL == table_rt) {
return;
}
struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(table_rt->expr_rt.hs, thread_id);
table_rt->expr_rt.hs_stream = hs_stream;
}
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len,
int result[], size_t *n_result)
{
if (NULL == table_rt) {
return -1;
}
return adapter_hs_scan_stream(table_rt->expr_rt.hs_stream, data, data_len, result, n_result);
}
void expr_runtime_stream_close(struct expr_runtime *expr_rt)
{
if (table_rt != NULL) {
adapter_hs_stream_close(table_rt->expr_rt.hs_stream);
table_rt->expr_rt.hs_stream = NULL;
}
}

207
src/maat_group.cpp
View File

@@ -20,6 +20,27 @@
#define MODULE_GROUP module_name_str("maat.group")
struct group2group_item {
int group_id;
int superior_group_id;
};
struct group2group_schema {
int group_id_column;
int superior_group_id_column;
int table_id;//ugly
};
struct group2group_runtime {
struct maat_group_topology *group_topo;
uint32_t rule_num;
uint32_t updating_rule_num;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
};
struct maat_group_topology {
struct maat_group *hash_group_by_id; //key: group_id, value: struct maat_group *.
struct maat_group *hash_group_by_vertex; //key: vetex_id, value: struct maat_group *. Multimap (Items with multiple keys).
@@ -32,6 +53,116 @@ struct maat_group_topology {
struct log_handle *logger;
};
void *group2group_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
{
int read_cnt = 0;
struct group2group_schema *g2g_schema = ALLOC(struct group2group_schema, 1);
cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) {
goto error;
}
g2g_schema->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_GROUP, "table %s has no custom column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
g2g_schema->group_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "superior_group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
g2g_schema->superior_group_id_column = custom_item->valueint;
read_cnt++;
}
if (read_cnt < 2) {
goto error;
}
return g2g_schema;
error:
FREE(g2c_schema);
return NULL;
}
void group2group_schema_free(void *g2g_schema)
{
FREE(g2g_schema);
}
struct group2group_runtime *
group2group_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
struct group2group_runtime *g2g_rt = ALLOC(struct group2group_runtime, 1);
g2g_rt->group_topo = maat_group_topology_new(logger);
g2g_rt->ref_garbage_bin = garbage_bin;
g2g_rt->logger = logger;
return g2g_rt;
}
void group2group_runtime_free(void *g2g_runtime)
{
if (NULL == g2g_runtime) {
return;
}
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
if (g2g_rt->group_topo != NULL) {
maat_group_topology_free(g2g_rt->group_topo);
}
FREE(g2g_rt);
}
struct group2group_item *
group2group_item_new(const char *line, struct group2group_schema *g2g_schema,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
struct group2group_item *g2g_item = ALLOC(struct group2group_item, 1);
int ret = get_column_pos(line, g2g_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_GROUP,
"group2group table(table_id:%d) line:%s has no group_id",
g2g_schema->table_id, line);
goto error;
}
g2g_item->group_id = atoi(line + column_offset);
ret = get_column_pos(line, g2g_schema->superior_group_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_GROUP,
"group2group table(table_id:%d) line:%s has no superior_group_id",
g2g_schema->table_id, line);
goto error;
}
g2g_item->superior_group_id = atoi(line + column_offset);
return g2g_item;
error:
FREE(g2g_item);
return NULL;
}
void group2group_item_free(struct group2group_item *g2g_item)
{
FREE(g2g_item);
}
void group_vertex_free(struct maat_group *group)
{
free(group->top_group_ids);
@@ -260,8 +391,7 @@ int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo)
//Orphan, Not reference by any one, free it.
if (0 == group->ref_by_compile_cnt
&& 0 == group->ref_by_superior_group_cnt
&& 0 == group->ref_by_subordinate_group_cnt
&& 0 == group->ref_by_item_cnt) {
&& 0 == group->ref_by_subordinate_group_cnt) {
FREE(group->top_group_ids);
maat_group_topology_remove_group(group_topo, group);
@@ -269,8 +399,7 @@ int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo)
}
//A group is need to build top groups when it has items and referenced by superior groups or compiles.
if (group->ref_by_item_cnt > 0 &&
(group->ref_by_compile_cnt > 0 || group->ref_by_superior_group_cnt > 0)) {
if (group->ref_by_compile_cnt > 0 || group->ref_by_superior_group_cnt > 0) {
if (0 == group->ref_by_superior_group_cnt) {
//fast path, group is only referenced by compile rules.
top_group_cnt = 1;
@@ -311,3 +440,73 @@ int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo)
return 0;
}
int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line,
int valid_column)
{
if (NULL == g2g_runtime || NULL == g2g_schema || NULL == line) {
return -1;
}
int ret = -1;
struct group2group_item *g2g_item = NULL;
struct group2group_schema *schema = (struct group2group_schema *)g2g_schema;
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
int item_id = get_column_value(line, schema->table_id);
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
//delete
ret = maat_group_topology_remove_group_from_group(g2g_rt->group_topo, g2g_item->group_id, g2g_item->superior_group_id);
} else {
//add
g2g_item = group2group_item_new(line, schema, logger);
}
if (1 == is_valid) {
//add
ret = maat_group_topology_add_group_to_group(g2g_rt->group_topo, g2g_item->group_id, g2g_item->superior_group_id);
} else {
//delete
}
return ret;
}
int group2group_runtime_commit(void *g2g_runtime)
{
if (NULL == g2g_runtime) {
return -1;
}
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
int ret = maat_group_topology_build_top_groups(g2g_rt->group_topo);
return ret;
}
int group2group_runtime_get_top_groups(struct group2group_runtime *g2g_rt, int *group_ids, size_t n_group_ids,
int *top_group_ids)
{
if (NULL == table_rt || NULL == group_ids || 0 == n_group_ids ||
table_rt->table_type != TABLE_TYPE_GROUP2GROUP) {
return -1;
}
size_t top_group_index = 0;
struct group2group_runtime *g2g_rt = &(table_rt->g2g_rt);
for (size_t i = 0; i < n_group_ids; i++) {
struct maat_group *group = maat_group_topology_find_group(g2g_rt->group_topo, group_ids[i]);
if (!group) {
continue;
}
for (size_t j = 0; j < group->top_group_cnt; j++) {
top_group_ids[top_group_index++] = group->top_group_ids[j];
}
}
return top_group_index;
}

1149
src/maat_hierarchy.cpp
View File

File diff suppressed because it is too large Load Diff

708
src/maat_ip.cpp Normal file
View File

@@ -0,0 +1,708 @@
/*
**********************************************************************************************
* File: maat_ip.cpp
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include <stdint.h>
#include "utils.h"
#include "log/log.h"
#include "cJSON/cJSON.h"
#include "utils.h"
#include "maat_utils.h"
#include "maat_ex_data.h"
#include "IPMatcher.h"
#include "maat_ip.h"
#include "maat_rule.h"
#include "maat_garbage_collection.h"
#define MODULE_IP module_name_str("maat.ip")
struct port_range {
uint16_t min_port;
uint16_t max_port;
};
struct ip_plus_schema {
int item_id_column;
int group_id_column;
int addr_type_column;
int saddr_format_column;
int sip1_column;
int sip2_column;
int sport_format_column;
int sport1_column;
int sport2_column;
int daddr_format_column;
int dip1_column;
int dip2_column;
int dport_format_column;
int dport1_column;
int dport2_column;
int proto_column;
int direction_column;
int table_id; //ugly
};
struct ipv4_item_rule {
uint32_t min_sip; /* 源地址下界；0表示忽略本字段 */
uint32_t max_sip; /* 源地址上界；0表示固定IP=min_saddr */
uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */
uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */
uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP；0表示忽略本字段 */
uint16_t direction; /* 方向，0表示双向，1表示单向 */
};
struct ipv6_item_rule {
uint32_t min_sip[4]; /* 源地址下界；全0表示忽略本字段 */
uint32_t max_sip[4]; /* 源地址上界；全0表示固定IP=min_saddr */
uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */
uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */
uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP，无限制默认为0 */
uint16_t direction; /* 方向，0表示双向，1表示单向 */
};
struct ip_plus_item {
int item_id;
int group_id;
int addr_type;
union {
struct ipv4_item_rule ipv4;
struct ipv6_item_rule ipv6;
};
};
struct ip_plus_runtime {
struct ip_matcher* ip_matcher;
struct ex_data_runtime* ex_data_rt;
uint32_t rule_num;
uint32_t updating_rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
// long long *scan_cnt;
// long long *hit_cnt;
// long long *not_grp_hit_cnt;
// long long *stream_num;
};
void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
{
size_t read_cnt = 0;
struct ip_plus_schema *ip_plus_schema = ALLOC(struct ip_plus_schema, 1);
cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) {
goto error;
}
ip_plus_schema->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_IP, "ip_plus table %s has no custom column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->item_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->group_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "addr_type");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->addr_type_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "saddr_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->saddr_format_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(json, "sip1");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sip1_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(json, "sip2");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sip2_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(json, "sport_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sport_format_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(json, "sport1");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sport1_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(json, "sport2");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sport2_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(json, "daddr_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->daddr_format_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(json, "dip1");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dip1_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(json, "dip2");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dip2_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(json, "dport_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dport_format_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(json, "dport1");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dport1_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(json, "dport2");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dport2_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(json, "proto");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->proto_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(json, "direction");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->direction_column = custom_item->valueint;
read_cnt++;
}
if (read_cnt < 17) {
goto error;
}
return ip_plus_schema;
error:
FREE(ip_plus_schema);
return NULL;
}
void ip_plus_schema_free(void *ip_plus_schema)
{
FREE(ip_plus_schema);
}
void *ip_plus_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == ip_plus_schema) {
return NULL;
}
struct ip_plus_schema *schema = (struct ip_plus_schema *)ip_plus_schema;
struct ip_plus_runtime *ip_plus_rt = ALLOC(struct ip_plus_runtime, 1);
ip_plus_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free);
ip_plus_rt->item_user_data_free = maat_item_inner_free;
ip_plus_rt->ref_garbage_bin = garbage_bin;
ip_plus_rt->logger = logger;
return ip_plus_rt;
}
void ip_plus_runtime_free(void *ip_plus_runtime)
{
if (NULL == ip_plus_runtime) {
return;
}
struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime;
if (ip_plus_rt->ip_matcher != NULL) {
ip_matcher_free(ip_plus_rt->ip_matcher);
}
if (ip_plus_rt->ex_data_rt != NULL) {
ex_data_runtime_free(ip_plus_rt->ex_data_rt);
}
struct maat_item *item = NULL, *tmp_item = NULL;
HASH_ITER(hh, ip_plus_rt->item_hash, item, tmp_item) {
HASH_DELETE(hh, ip_plus_rt->item_hash, item);
maat_item_free(item, ip_plus_rt->item_user_data_free);
}
FREE(ip_plus_rt);
}
struct ip_plus_item *ip_plus_item_new(const char *line, struct ip_plus_schema *ip_plus_schema,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
char saddr_format[16] = {0};
char sport_format[16] = {0};
char sip1_str[40] = {0};
char sip2_str[40] = {0};
uint16_t sport1 = 0;
uint16_t sport2 = 0;
uint16_t protocol = 0;
uint16_t direction = 0;
struct ip_plus_item *ip_plus_item = ALLOC(struct ip_plus_item, 1);
int ret = get_column_pos(line, ip_plus_schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no item_id",
ip_plus_schema->table_id, line);
goto error;
}
ip_plus_item->item_id = atoi(line + column_offset);
ret = get_column_pos(line, ip_plus_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no group_id",
ip_plus_schema->table_id, line);
goto error;
}
ip_plus_item->group_id = atoi(line + column_offset);
ret = get_column_pos(line, ip_plus_schema->addr_type_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no addr_type",
ip_plus_schema->table_id, line);
goto error;
}
ip_plus_item->addr_type = atoi(line + column_offset);
if (ip_plus_item->addr_type != 4 && ip_plus_item->addr_type != 6) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has invalid addr type:%d",
ip_plus_schema->table_id, line, ip_plus_item->addr_type);
goto error;
}
ret = get_column_pos(line, ip_plus_schema->saddr_format_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
memcpy(saddr_format, (line + column_offset), column_len);
if (IP_FORMAT_UNKNOWN == ip_format_str2int(saddr_format)) {
log_error(logger, MODULE_IP,
"ip_plus table(table_id:%d) line:%s has invalid saddr_format, should be range/mask/CIDR",
ip_plus_schema->table_id, line);
goto error;
}
ret = get_column_pos(line, ip_plus_schema->sip1_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sip1",
ip_plus_schema->table_id, line);
goto error;
}
memcpy(sip1_str, (line + column_offset), column_len);
ret = get_column_pos(line, ip_plus_schema->sip2_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sip2",
ip_plus_schema->table_id, line);
goto error;
}
memcpy(sip2_str, (line + column_offset), column_len);
ret = get_column_pos(line, ip_plus_schema->sport_format_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport_format",
ip_plus_schema->table_id, line);
goto error;
}
memcpy(sport_format, (line + column_offset), column_len);
if (IP_FORMAT_UNKNOWN == ip_format_str2int(sport_format)) {
log_error(logger, MODULE_IP,
"ip_plus table(table_id:%d) line:%s has invalid sport_format, should be range/mask/CIDR",
ip_plus_schema->table_id, line);
goto error;
}
ret = get_column_pos(line, ip_plus_schema->sport1_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport1",
ip_plus_schema->table_id, line);
goto error;
}
sport1 = atoi(line + column_offset);
ret = get_column_pos(line, ip_plus_schema->sport2_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport2",
ip_plus_schema->table_id, line);
goto error;
}
sport2 = atoi(line + column_offset);
if (4 == ip_plus_item->addr_type) {
ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str,
&ip_plus_item->ipv4.min_sip, &ip_plus_item->ipv4.max_sip);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s ip_format2range(ip4) failed",
ip_plus_schema->table_id, line);
goto error;
}
if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) {
ip_plus_item->ipv4.min_sport = sport1 & sport2;
ip_plus_item->ipv4.max_sport = sport1 | ~sport2;
} else {
ip_plus_item->ipv4.min_sport = sport1;
ip_plus_item->ipv4.max_sport = sport2;
}
ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no proto",
ip_plus_schema->table_id, line);
goto error;
}
ip_plus_item->ipv4.proto = atoi(line + column_offset);
protocol = ip_plus_item->ipv4.proto;
ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no direction",
ip_plus_schema->table_id, line);
goto error;
}
ip_plus_item->ipv4.direction = atoi(line + column_offset);
direction = ip_plus_item->ipv4.direction;
} else {
//ipv6
ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str,
ip_plus_item->ipv6.min_sip, ip_plus_item->ipv6.max_sip);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s ip_format2range(ip6) failed",
ip_plus_schema->table_id, line);
goto error;
}
if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) {
ip_plus_item->ipv6.min_sport = sport1 & sport2;
ip_plus_item->ipv6.max_sport = sport1 | ~sport2;
} else {
ip_plus_item->ipv6.min_sport = sport1;
ip_plus_item->ipv6.max_sport = sport2;
}
ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no proto",
ip_plus_schema->table_id, line);
goto error;
}
ip_plus_item->ipv6.proto = atoi(line + column_offset);
protocol = ip_plus_item->ipv6.proto;
ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no direction",
ip_plus_schema->table_id, line);
goto error;
}
ip_plus_item->ipv6.direction = atoi(line + column_offset);
direction = ip_plus_item->ipv6.direction;
}
if (protocol > 65535 || protocol < 0) {
log_error(logger, MODULE_IP,
"ip_plus table(table_id:%d) line:%s has invalid proto:%d",
ip_plus_schema->table_id, line, protocol);
goto error;
}
if (direction != 0 && direction != 1) {
log_error(logger, MODULE_IP,
"ip_plus table(table_id:%d) line:%s has invalid direction:%d",
ip_plus_schema->table_id, line, direction);
goto error;
}
return ip_plus_item;
error:
FREE(ip_plus_item);
return NULL;
}
void ip_plus_item_free(struct ip_plus_item *ip_plus_item)
{
FREE(ip_plus_item);
}
void ip_plus_item_to_ip_rule(struct ip_plus_item *item, struct ip_rule *rule)
{
struct port_range *sport_range = ALLOC(struct port_range, 1);
if (4 == item->addr_type) {
rule->type = IPv4;
sport_range->min_port = item->ipv4.min_sport;
sport_range->max_port = item->ipv4.max_sport;
rule->ipv4_rule.start_ip = item->ipv4.min_sip;
rule->ipv4_rule.end_ip = item->ipv4.max_sip;
} else {
rule->type = IPv6;
sport_range->min_port = item->ipv6.min_sport;
sport_range->max_port = item->ipv6.max_sport;
memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip, sizeof(item->ipv6.min_sip));
memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip, sizeof(item->ipv6.max_sip));
}
rule->rule_id = item->item_id;
rule->user_tag = sport_range;
}
struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt)
{
return ip_plus_rt->ex_data_rt;
}
int ip_plus_runtime_update_row(struct ip_plus_runtime *rt, char *key, size_t key_len,
struct ip_plus_item *item, int is_valid)
{
int ret = -1;
struct ex_data_runtime *ex_data_rt = rt->ex_data_rt;
if (0 == is_valid) {
// delete
ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len);
if (ret < 0) {
return -1;
}
} else {
// add
struct ex_data_container *ex_container = ex_data_container_new(NULL, (void *)item);
ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container);
if (ret < 0) {
return -1;
}
}
return 0;
}
int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line,
int valid_column)
{
if (NULL == ip_plus_runtime || NULL == ip_plus_schema || NULL == line) {
return -1;
}
struct maat_item *item = NULL;
struct ip_plus_item *ip_plus_item = NULL;
struct maat_item_inner *u_para = NULL;
struct ip_plus_schema *schema = (struct ip_plus_schema *)ip_plus_schema;
struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime;
int item_id = get_column_value(line, schema->item_id_column);
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
//delete
HASH_FIND_INT(ip_plus_rt->item_hash, &item_id, item);
if (NULL == item) {
return -1;
}
u_para = (struct maat_item_inner *)item->user_data;
item->user_data = NULL;
if (NULL == u_para) {
return -1;
}
HASH_DELETE(hh, ip_plus_rt->item_hash, item);
maat_garbage_bagging(ip_plus_rt->ref_garbage_bin, u_para, (void (*)(void *))maat_item_inner_free);
} else {
//add
HASH_FIND_INT(ip_plus_rt->item_hash, &item_id, item);
if (item) {
log_error(ip_plus_rt->logger, MODULE_IP,
"ip_plus runtime add item %d to item_hash failed, already exist", item_id);
return -1;
}
ip_plus_item = ip_plus_item_new(line, schema, ip_plus_rt->logger);
if (NULL == ip_plus_item) {
log_error(ip_plus_rt->logger, MODULE_IP, "ip_plus line:%s to item failed", line);
return -1;
}
u_para = maat_item_inner_new(ip_plus_item->group_id, item_id, 0);
item = maat_item_new(item_id, group_id, u_para);
HASH_ADD_INT(ip_plus_rt->item_hash, item_id, item);
}
char *key = (char *)&item_id;
int ret = ip_plus_runtime_update_row(ip_plus_rt, key, sizeof(int), ip_plus_item, is_valid);
if (ret < 0) {
if (ip_plus_item != NULL) {
ip_plus_item_free(ip_plus_item);
ip_plus_item = NULL;
}
return -1;
} else {
if (0 == is_valid) {
ip_plus_rt->rule_num--;
} else {
ip_plus_rt->rule_num++;
}
}
return 0;
}
int ip_plus_runtime_commit(void *ip_plus_runtime)
{
if (NULL == ip_plus_runtime) {
return -1;
}
int ret = 0;
struct ex_data_container **ex_container = NULL;
struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime;
struct ex_data_runtime *ex_data_rt = ip_plus_rt->ex_data_rt;
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
if (0 == rule_cnt) {
FREE(ex_container);
return 0;
}
struct ip_rule *rules = ALLOC(struct ip_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
struct ip_plus_item *item = (struct ip_plus_item *)ex_container[i]->custom_data;
ip_plus_item_to_ip_rule(item, &rules[i]);
}
struct ip_matcher *new_ip_matcher = NULL;
struct ip_matcher *old_ip_matcher = NULL;
size_t mem_used = 0;
if (rule_cnt > 0) {
log_info(logger, MODULE_IP,
"committing %zu ip_plus rules for rebuilding ip_matcher engine", rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
log_error(logger, MODULE_IP,
"rebuild ip_matcher engine failed when update %zu ip_plus rules", rule_cnt);
ret = -1;
}
}
old_ip_matcher = ip_plus_rt->ip_matcher;
ip_plus_rt->ip_matcher = new_ip_matcher;
maat_garbage_bagging(garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
ex_data_runtime_commit(ex_data_rt);
ip_plus_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt);
FREE(rules);
FREE(ex_container);
return ret;
}
int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr *data,
int *group_id_array, size_t n_group_id_array, int virtual_table_id,
struct maat_state *state)
{
if (NULL == table_rt) {
return -1;
}
int n_hit_item = 0;
struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
struct ip_data ip;
ip.type = ip_type_transform(scan_data->ip_type);
if (ip.type == IPv4) {
ip.ipv4 = scan_data->ipv4;
} else {
memcpy(ip.ipv6, scan_data->ipv6, sizeof(scan_data->ipv6));
}
n_hit_item = ip_matcher_match(table_rt->ip_plus_rt.ip_matcher, &ip, scan_results, MAX_SCANNER_HIT_ITEM_NUM);
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
struct maat_compile_state *compile_state = state->compile_mid;
//tranform item_id to group_id
struct maat_item *item = NULL;
size_t n_group_id = 0;
int i = 0;
for (i = 0; i < n_hit_item; i++) {
HASH_FIND_INT(table_rt->item_hash, &(scan_results[i].rule_id), item);
assert(item != NULL);
if (!item) {
// should not come here
continue;
}
if (n_group_id >= n_group_id_array) {
n_group_id = n_group_id_array;
//Prevent group_id_array out of bounds
} else {
group_id_array[n_group_id++] = item->group_id;
}
// update hit path
maat_compile_state_update_hit_path(compile_state, scan_results[i].rule_id, item->group_id, virtual_table_id, state->scan_cnt, i);
}
// update hit clause: literal_id{group_id,vt_id} to clause_id
int compile_table_id = -1;
if (state->compile_table_id == -1) {
compile_table_id = state->maat_instance->default_compile_table_id;
} else {
compile_table_id = state->compile_table_id;
}
struct maat_runtime *maat_rt = state->maat_instance->maat_rt;
struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, compile_table_id);
assert(compile_table_rt->table_type == TABLE_TYPE_COMPILE);
for (size_t idx = 0; idx < n_group_id; idx++) {
maat_compile_state_update_hit_clause(compile_state, &(compile_table_rt->compile_rt.compile_hash), group_id_array[idx], virtual_table_id);
}
return n_group_id;
}

448
src/maat_ip_plugin.cpp Normal file
View File

@@ -0,0 +1,448 @@
/*
**********************************************************************************************
* File: maat_ip_plugin.cpp
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include "maat_ip_plugin.h"
#include "cJSON/cJSON.h"
#include "log/log.h"
#include "utils.h"
#include "maat_utils.h"
#include "maat_ex_data.h"
#include "IPMatcher.h"
#include "maat_rule.h"
#define MODULE_IP_PLUGIN module_name_str("maat.ip_plugin")
#define MAX_IP_STR 128
struct ip_plugin_item {
int item_id;
int ip_type;
char start_ip[MAX_IP_STR];
char end_ip[MAX_IP_STR];
int rule_tag;
};
struct ip_plugin_schema {
int item_id_column;
int ip_type_column;
int start_ip_column;
int end_ip_column;
int rule_tag_column;
struct ex_data_schema *ex_schema;
int table_id; //ugly
};
struct ip_plugin_runtime {
struct ip_matcher *ip_matcher;
struct ex_data_runtime *ex_data_rt;
uint32_t rule_num;
uint32_t updating_rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
// long long *scan_cnt;
// long long *hit_cnt;
// long long *not_grp_hit_cnt;
// long long *stream_num;
};
void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
{
size_t read_cnt = 0;
struct ip_plugin_schema *ip_plugin_schema = ALLOC(struct ip_plugin_schema, 1);
cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) {
goto error;
}
ip_plugin_schema->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_IP_PLUGIN, "table %s has no custom column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plugin_schema->item_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "ip_type");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plugin_schema->ip_type_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "start_ip");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plugin_schema->start_ip_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "end_ip");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plugin_schema->end_ip_column = custom_item->valueint;
read_cnt++;
}
if (read_cnt < 4) {
goto error;
}
return ip_plugin_schema;
error:
FREE(ip_plugin_schema);
return NULL;
}
void ip_plugin_schema_free(void *ip_plugin_schema)
{
if (NULL == ip_plugin_schema) {
return;
}
struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
if (schema->ex_schema != NULL) {
ex_data_schema_free(schema->ex_schema);
schema->ex_schema = NULL;
}
FREE(schema);
}
struct ex_data_schema *ip_plugin_table_get_ex_data_schema(void *ip_plugin_schema)
{
if (NULL == ip_plugin_schema) {
return NULL;
}
struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
return schema->ex_schema;
}
struct ip_plugin_item *ip_plugin_item_new(const char *line, struct ip_plugin_schema *ip_plugin_schema,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
struct ip_plugin_item *ip_plugin_item = ALLOC(struct ip_plugin_item, 1);
int ret = get_column_pos(line, ip_plugin_schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, "ip plugin table(table_id:%d) line:%s has no item_id",
ip_plugin_schema->table_id, line);
goto error;
}
ip_plugin_item->item_id = atoi(line + column_offset);
ret = get_column_pos(line, ip_plugin_schema->ip_type_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, "ip plugin table(table_id:%d) line:%s has no ip_type",
ip_plugin_schema->table_id, line);
goto error;
}
ip_plugin_item->ip_type = atoi(line + column_offset);
if (ip_plugin_item->ip_type != 4 && ip_plugin_item->ip_type != 6) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s ip_type[%d] invalid",
ip_plugin_schema->table_id, line, ip_plugin_item->ip_type);
goto error;
}
ret = get_column_pos(line, ip_plugin_schema->start_ip_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s has no start_ip",
ip_plugin_schema->table_id, line);
goto error;
}
strncpy(ip_plugin_item->start_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->start_ip)));
ret = get_column_pos(line, ip_plugin_schema->end_ip_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s has no end_ip",
ip_plugin_schema->table_id, line);
goto error;
}
strncpy(ip_plugin_item->end_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->end_ip)));
return ip_plugin_item;
error:
FREE(ip_plugin_item);
return NULL;
}
void ip_plugin_item_free(struct ip_plugin_item *item)
{
FREE(item);
}
int ip_plugin_table_ex_data_schema_flag(struct ip_plugin_schema *ip_plugin_schema)
{
}
int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp)
{
struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
struct ex_data_schema *ex_schema = schema->ex_schema;
if (ex_schema->set_flag) {
assert(0);
log_error(logger, MODULE_TABLE, "Error: %s, EX data schema already registed",
__FUNCTION__);
return -1;
}
ex_schema->new_func = new_func;
ex_schema->free_func = free_func;
ex_schema->dup_func = dup_func;
ex_schema->argl = argl;
ex_schema->argp = argp;
//ex_schema->set_flag = 1;
return 0;
}
int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_schema *schema,
const char *row, char *key, size_t key_len, struct ip_plugin_item *item,
int is_valid)
{
int ret = -1;
struct ex_data_runtime *ex_data_rt = rt->ex_data_rt;
int set_flag = ip_plugin_table_schema_ex_data_schema_flag(schema);
if (1 == set_flag) {
if (0 == is_valid) {
//delete
ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len);
if (ret < 0) {
return -1;
}
} else {
//add
void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, (void *)item);
ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container);
if (ret < 0) {
return -1;
}
}
} else {
ex_data_runtime_cache_row_put(ex_data_rt, row);
}
return 0;
}
void *ip_plugin_runtime_new(void *ip_plugin_schema, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == ip_plugin_schema) {
return NULL;
}
struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
struct ip_plugin_runtime *ip_plugin_rt = ALLOC(struct ip_plugin_runtime, 1);
ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free);
ip_plugin_rt->item_user_data_free = maat_item_inner_free;
ip_plugin_rt->ref_garbage_bin = garbage_bin;
ip_plugin_rt->logger = logger;
return ip_plugin_rt;
}
void ip_plugin_runtime_free(void *ip_plugin_runtime)
{
if (NULL == ip_plugin_runtime) {
return;
}
struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
if (ip_plugin_rt->ip_matcher != NULL) {
ip_matcher_free(ip_plugin_rt->ip_matcher);
}
if (ip_plugin_rt->ex_data_rt != NULL) {
ex_data_runtime_free(ip_plugin_rt->ex_data_rt);
}
struct maat_item *item = NULL, *tmp_item = NULL;
HASH_ITER(hh, ip_plugin_rt->item_hash, item, tmp_item) {
HASH_DELETE(hh, ip_plugin_rt->item_hash, item);
maat_item_free(item, ip_plugin_rt->item_user_data_free);
}
FREE(ip_plugin_rt);
}
int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line,
int valid_column)
{
if (NULL == ip_plugin_runtime || NULL == ip_plugin_schema || NULL == line) {
return -1;
}
struct maat_item *item = NULL;
struct ip_plugin_item *ip_plugin_item = NULL;
struct maat_item_inner *u_para = NULL;
struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
int item_id = get_column_value(line, schema->item_id_column);
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
//delete
HASH_FIND_INT(ip_plugin_rt->item_hash, &item_id, item);
if (NULL == item) {
return -1;
}
u_para = (struct maat_item_inner *)item->user_data;
item->user_data = NULL;
if (NULL == u_para) {
return -1;
}
HASH_DELETE(hh, ip_plugin_rt->item_hash, item);
maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, u_para, (void (*)(void *))maat_item_inner_free);
} else {
//add
HASH_FIND_INT(ip_plugin_rt->item_hash, &item_id, item);
if (item) {
log_error(ip_plus_rt->logger, MODULE_IP_PLUGIN,
"ip_plugin runtime add item %d to item_hash failed, already exist", item_id);
return -1;
}
ip_plugin_item = ip_plugin_item_new(line, schema, ip_plugin_rt->logger);
if (NULL == ip_plugin_item) {
log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN, "ip_plugin line:%s to item failed", line);
return -1;
}
u_para = maat_item_inner_new(ip_plugin_item->group_id, item_id, 0);
item = maat_item_new(item_id, group_id, u_para);
HASH_ADD_INT(ip_plugin_rt->item_hash, item_id, item);
}
char *key = (char *)&item_id;
int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, row, key, sizeof(int), ip_plugin_item, is_valid);
if (ret < 0) {
if (ip_plugin_item != NULL) {
ip_plugin_item_free(ip_plugin_item);
ip_plugin_item = NULL;
}
return -1;
} else {
if (0 == is_valid) {
ip_plugin_rt->rule_num--;
} else {
ip_plugin_rt->rule_num++;
}
}
return 0;
}
void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule)
{
if (4 == item->ip_type) {
rule->type = IPv4;
ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip, &(rule->ipv4_rule.start_ip), &(rule->ipv4_rule.end_ip));
} else {
rule->type = IPv6;
ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip, &(rule->ipv6_rule.start_ip), &(rule->ipv6_rule.end_ip));
}
rule->rule_id = item->item_id;
rule->user_tag = NULL;
}
int ip_plugin_runtime_commit(void *ip_plugin_runtime)
{
if (NULL == ip_plugin_runtime) {
return -1;
}
int ret = 0;
struct ex_data_container **ex_container = NULL;
struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
struct ex_data_runtime *ex_data_rt = ip_plugin_rt->ex_data_rt;
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
if (0 == rule_cnt) {
FREE(ex_container);
return 0;
}
struct ip_rule *rules = ALLOC(struct ip_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
struct ip_plugin_item *item = (struct ip_plugin_item *)ex_container[i]->custom_data;
ip_plugin_item_to_ip_rule(item, &rules[i]);
}
struct ip_matcher *new_ip_matcher = NULL;
struct ip_matcher *old_ip_matcher = NULL;
size_t mem_used = 0;
if (rule_cnt > 0) {
log_info(ip_plugin_rt->logger, MODULE_TABLE_RUNTIME,
"committing %zu ip_plugin rules for rebuilding ip_matcher engine", rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
log_error(ip_plugin_rt->logger, MODULE_TABLE_RUNTIME,
"rebuild ip_matcher engine failed when update %zu ip_plugin rules", rule_cnt);
ret = -1;
}
}
old_ip_matcher = ip_plugin_rt->ip_matcher;
ip_plugin_rt->ip_matcher = new_ip_matcher;
maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
ex_data_runtime_commit(ex_data_rt);
ip_plugin_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt);
FREE(rules);
FREE(ex_container);
return ret;
}
int ip_plugin_runtime_updating_flag(struct ip_plugin_runtime *ip_plugin_rt)
{
return ex_data_runtime_updating_flag(ip_plugin_rt->ex_data_rt);
}
struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime)
{
if (NULL == ip_plugin_runtime) {
return NULL;
}
struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
return ip_plugin_rt->ex_data_rt;
}

13
src/maat_kv.cpp
View File

@@ -131,3 +131,16 @@ int maat_kv_read(struct maat_kv_store * store, const char * key, int * value)
{
return maat_kv_read_unNull(store, key, strlen(key), value);
}
struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *origin_map)
{
struct maat_kv_store *target = maat_kv_store_new();
struct maat_kv_pair *kv = NULL, *tmp_kv = NULL, *copy_kv = NULL;
HASH_ITER (hh, origin_map->hash, kv, tmp_kv) {
copy_kv = maat_kv_pair_new(kv->key, kv->keylen, kv->val);
HASH_ADD_KEYPTR(hh, target->hash, copy_kv->key, copy_kv->keylen, copy_kv);
}
return target;
}

375
src/maat_plugin.cpp Normal file
View File

@@ -0,0 +1,375 @@
/*
**********************************************************************************************
* File: maat_plugin.cpp
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include <assert.h>
#include "log/log.h"
#include "cJSON/cJSON.h"
#include "utils.h"
#include "maat_utils.h"
#include "maat.h"
#include "maat_plugin.h"
#include "maat_ex_data.h"
#include "maat_limits.h"
#define MODULE_PLUGIN module_name_str("maat.plugin")
struct plugin_callback_schema {
maat_start_callback_t *start;
maat_update_callback_t *update;
maat_finish_callback_t *finish;
void *u_para;
};
struct plugin_item {
char key[MAX_KEYWORDS_STR];
size_t key_len;
};
struct plugin_runtime {
uint64_t acc_line_num;
struct ex_data_runtime *ex_data_rt;
uint32_t rule_num;
uint32_t updating_rule_num;
struct maat_garbage_bin *ref_garbage_bin;
};
#define MAX_PLUGIN_PER_TABLE 32
struct plugin_schema {
int item_id_column;
int key_column;
int rule_tag_column;
int n_foreign;
int foreign_columns[MAX_FOREIGN_CLMN_NUM];
size_t cb_cnt;
struct plugin_callback_schema cb[MAX_PLUGIN_PER_TABLE];
struct ex_data_schema *ex_schema;
int table_id; //ugly
};
static int read_integer_array(char *string, int *array, int size)
{
int i = 0;
char *token = NULL, *sub_token = NULL, *saveptr;
for (i = 0, token = string; i < size; token= NULL, i++) {
sub_token = strtok_r(token, ",", &saveptr);
if (sub_token == NULL) {
break;
}
sscanf(sub_token, "%d", array + i);
}
return i;
}
void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
{
size_t read_cnt = 0;
struct plugin_schema *plugin_schema = ALLOC(struct plugin_schema, 1);
cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) {
goto error;
}
plugin_schema->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_PLUGIN, "table %s has no custom column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item == NULL || custom_item->type != cJSON_Object) {
plugin_schema->item_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "key");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
plugin_schema->key_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "tag");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
plugin_schema->rule_tag_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "foreign");
if (custom_item != NULL) {
read_cnt++;
if (custom_item->type == cJSON_String) {
plugin_schema->n_foreign = read_integer_array(custom_item->valuestring,
plugin_schema->foreign_columns,
MAX_FOREIGN_CLMN_NUM);
} else if (custom_item->type == cJSON_Array) {
plugin_schema->n_foreign = cJSON_GetArraySize(custom_item);
for (int i = 0; i < plugin_schema->n_foreign; i++) {
cJSON *foreign_item = cJSON_GetArrayItem(custom_item, i);
assert(foreign_item->type == cJSON_Number);
plugin_schema->foreign_columns[i] = foreign_item->valueint;
}
}
}
if (read_cnt < 4) {
goto error;
}
return plugin_schema;
error:
FREE(plugin_schema);
return NULL;
}
void plugin_schema_free(void *plugin_schema)
{
if (NULL == plugin_schema) {
return;
}
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
if (schema->ex_schema != NULL) {
ex_data_schema_free(schema->ex_schema);
schema->ex_schema = NULL;
}
FREE(schema);
}
int plugin_table_add_callback(void *plugin_schema, int table_id,
maat_start_callback_t *start,
maat_update_callback_t *update,
maat_finish_callback_t *finish,
void *u_para, struct log_handle *logger)
{
if (NULL == plugin_schema) {
return -1;
}
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
size_t idx = schema->cb_cnt;
if (idx == MAX_PLUGIN_PER_TABLE) {
log_error(logger, MODULE_PLUGIN, "the plugin number of table_id: %d exceed maxium:%d",
table_id, MAX_PLUGIN_PER_TABLE);
return -1;
}
schema->cb_cnt++;
schema->cb[idx].start = start;
schema->cb[idx].update = update;
schema->cb[idx].finish = finish;
schema->cb[idx].u_para = u_para;
return 0;
}
void plugin_table_all_callback_start(struct plugin_schema *plugin_schema, int update_type)
{
for (size_t i = 0; i < plugin_schema->cb_cnt; i++) {
if (plugin_schema->cb[i].start != NULL) {
plugin_schema->cb[i].start(update_type, plugin_schema->cb[i].u_para);
}
}
}
void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema)
{
for (size_t i = 0; i < plugin_schema->cb_cnt; i++) {
if (plugin_schema->cb[i].finish != NULL) {
plugin_schema->cb[i].finish(plugin_schema->cb[i].u_para);
}
}
}
struct plugin_item *plugin_item_new(const char *line, struct plugin_schema *plugin_schema,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
struct plugin_item *plugin_item = ALLOC(struct plugin_item, 1);
int ret = get_column_pos(line, plugin_schema->key_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_PLUGIN,
"plugin table(table_id:%d) line:%s has no key",
plugin_schema->table_id, line);
goto error;
}
if (column_len > MAX_KEYWORDS_STR) {
log_error(logger, MODULE_PLUGIN,
"plugin table(table_id:%d): key:%s length:%zu too long, exceed %d",
plugin_schema->table_id, (line + column_offset), column_len, MAX_KEYWORDS_STR);
goto error;
}
memcpy(plugin_item->key, (line + column_offset), column_len);
plugin_item->key_len = column_len;
return plugin_item;
error:
FREE(plugin_item);
return NULL;
}
void plugin_item_free(struct plugin_item *plugin_item)
{
FREE(plugin_item);
}
int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns)
{
if (NULL == plugin_schema) {
return -1;
}
int n_foreign = plugin_schema->n_foreign;
for (int i = 0; i < n_foreign; i++) {
foreign_columns[i] = plugin_schema->foreign_columns[i];
}
return n_foreign;
}
int plugin_table_set_ex_data_schema(void *plugin_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger)
{
if (NULL == plugin_schema) {
return -1;
}
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
if (schema->ex_schema != NULL) {
assert(0);
log_error(logger, MODULE_PLUGIN,
"Error: %s, EX data schema already registed", __FUNCTION__);
return -1;
}
schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp);
return 0;
}
struct ex_data_schema *plugin_table_get_ex_data_schema(void *plugin_schema)
{
if (NULL == plugin_schema) {
return NULL;
}
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
return schema->ex_schema;
}
int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_schema *plugin_schema,
const char *row, char *key, size_t key_len, int is_valid)
{
int ret = -1;
void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL);
struct ex_data_schema *ex_schema = plugin_schema->ex_schema;
/* already set plugin_table_schema's ex_data_schema */
if (ex_schema != NULL) {
if (is_valid == 0) {
// delete
ret = ex_data_runtime_del_ex_container(plugin_rt->ex_data_rt, key, key_len);
if (ret < 0) {
return -1;
}
} else {
// add
ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container);
if (ret < 0) {
return -1;
}
}
}
/* plugin table schema has callback */
size_t cb_count = plugin_schema->cb_cnt;
if (cb_count > 0) {
for (size_t i = 0; i < cb_count; i++) {
plugin_schema->cb[i].update(plugin_schema->table_id, row, plugin_schema->cb[i].u_para);
}
}
if ((NULL == ex_schema) && (0 == cb_count)) {
ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row);
}
plugin_rt->acc_line_num++;
return 0;
}
int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, const char *line,
int valid_column)
{
if (NULL == plugin_runtime || NULL == plugin_runtime) {
return -1;
}
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
int item_id = get_column_value(line, schema->item_id_column);
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
return -1;
}
char *key = (char *)&item_id;
int ret = plugin_runtime_update_row(plugin_rt, schema, line, key, sizeof(int), is_valid);
if (ret < 0) {
} else {
}
return 0;
}
int plugin_runtime_commit(void *plugin_runtime)
{
if (NULL == plugin_runtime) {
return -1;
}
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
ex_data_runtime_commit(plugin_rt->ex_data_rt);
//table_rt->rule_num = ex_data_runtime_ex_container_count(table_rt->plugin_rt.ex_data_rt);
return 0;
}
int plugin_runtime_updating_flag(struct plugin_runtime *plugin_rt)
{
return ex_data_runtime_updating_flag(plugin_rt->ex_data_rt);
}
struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime)
{
if (NULL == plugin_runtime) {
return NULL;
}
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
return plugin_rt->ex_data_rt;
}

30
src/maat_redis_monitor.cpp
View File

@@ -17,7 +17,7 @@
#include "maat_command.h"
#include "maat_config_monitor.h"
#include "maat_redis_monitor.h"
#include "maat_table_schema.h"
#include "maat_plugin.h"
#define MODULE_REDIS_MONITOR module_name_str("maat.redis_monitor")
@@ -97,22 +97,22 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, int rule_num, struct maat *maat_instance, const char *dir)
{
int rule_with_foreign_key = 0;
struct table_schema *table_schema = NULL;
void *schema = NULL;
for (int i = 0; i < rule_num; i++) {
if (NULL == rule_list[i].table_line) {
continue;
}
int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, rule_list[i].table_name);
table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
enum table_type table_type = table_schema_get_table_type(table_schema);
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, rule_list[i].table_name);
schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (!table_schema || table_type != TABLE_TYPE_PLUGIN) {
continue;
}
int foreign_columns[8];
int n_foreign_column = table_schema_get_foreign_column(table_schema, foreign_columns);
int n_foreign_column = plugin_table_get_foreign_column((struct plugin_schema *)schema, foreign_columns);
if (0 == n_foreign_column) {
continue;
}
@@ -467,7 +467,7 @@ int recovery_history_version(const struct serial_rule *current, int current_num,
}
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version,
long long *new_version, struct table_schema_manager* table_schema_mgr,
long long *new_version, struct table_manager *tbl_mgr,
struct serial_rule **list, int *update_type, int cumulative_off,
struct log_handle *logger)
{
@@ -608,7 +608,7 @@ FULL_UPDATE:
}
if (table_schema_mgr) {
int table_id = table_schema_manager_get_table_id(table_schema_mgr, s_rule_array[full_idx].table_name);
int table_id = table_manager_get_table_id(tbl_mgr, s_rule_array[full_idx].table_name);
//Unrecognized table.
if (table_id < 0) {
continue;
@@ -1277,7 +1277,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
int valid_column = -1;
enum table_type table_type;
enum scan_type scan_type;
struct table_schema *table_schema = NULL;
void *table_schema = NULL;
struct maat *maat_instance = (struct maat *)u_param;
//authorized to write
@@ -1369,19 +1369,19 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
continue;
}
table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, rule_list[i].table_name);
table_id = table_manager_get_table_id(maat_instance->tbl_mgr, rule_list[i].table_name);
//Unrecognized table.
if (table_id < 0) {
no_table_num++;
continue;
}
table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
if (rule_list[i].op == MAAT_OP_DEL) {
scan_type = table_schema_get_scan_type(table_schema);
table_type = table_schema_get_table_type(table_schema);
table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, scan_type, NULL);
valid_column = table_schema_get_valid_flag_column(table_schema);
//TODO: by luis
//scan_type = table_schema_get_scan_type(table_schema);
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
//table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, scan_type, NULL);
valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id);
ret = invalidate_line(rule_list[i].table_line, table_type, valid_column);
if (ret < 0) {
log_error(maat_instance->logger, MODULE_REDIS_MONITOR,

281
src/maat_rule.cpp
View File

@@ -23,28 +23,223 @@
#include "maat_rule.h"
#include "maat_config_monitor.h"
#include "maat_redis_monitor.h"
#include "maat_table_runtime.h"
#include "maat_table_schema.h"
#include "maat_hierarchy.h"
#include "maat_table.h"
#include "maat_compile.h"
#include "maat_plugin.h"
#include "alignment.h"
#define MODULE_MAAT_RULE module_name_str("maat.rule")
struct maat_item *maat_item_new(int item_id, int group_id, void *user_data)
{
struct maat_item *item = NULL;
item = ALLOC(struct maat_item, 1);
item->group_id = group_id;
item->item_id = item_id;
item->user_data = user_data;
return item;
}
void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *))
{
if (item_user_data_free && item->user_data) {
item_user_data_free(item->user_data);
item->user_data = NULL;
}
free(item);
}
static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, int n_accept_tag)
{
if (NULL == tag_obj || NULL == accept_tags) {
return -1;
}
cJSON *tab_name_obj = cJSON_GetObjectItem(tag_obj, "tag");
if (NULL == tab_name_obj || tab_name_obj->type != cJSON_String) {
return -1;
}
const char *tag_name = tab_name_obj->valuestring;
cJSON *tag_vals_array = cJSON_GetObjectItem(tag_obj, "value");
if (NULL == tag_vals_array || tag_vals_array->type != cJSON_Array) {
return -1;
}
int name_matched = 0;
int n_val = cJSON_GetArraySize(tag_vals_array);
for (int i = 0; i < n_accept_tag; i++) {
if (0 != strcmp(accept_tags[i].tag_name, tag_name)) {
continue;
}
name_matched++;
for (int j = 0; j < n_val; j++) {
cJSON *tag_val_obj = cJSON_GetArrayItem(tag_vals_array, j);
if (NULL == tag_val_obj || tag_val_obj->type != cJSON_String) {
return -1;
}
const char *tag_val = tag_val_obj->valuestring;
// compare a/b/c with a/b/c/d is a miss.
if (strlen(accept_tags[i].tag_val) < strlen(tag_val)) {
continue;
}
// compare a1a2/b1/c1 with a1a2/b/ is a miss.
//make sure the overlap is ended with a '/'
if (0 == strncmp(accept_tags[i].tag_val, tag_val, strlen(tag_val)) &&
(strlen(accept_tags[i].tag_val) == strlen(tag_val) || accept_tags[i].tag_val[strlen(tag_val)] == '/')) {
return 1;
}
}
}
//no matched name is considered as a
if (name_matched > 0) {
return 0;
} else {
return 1;
}
}
//@param tag_set likes [{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}]
static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_tags, int n_accept_tag)
{
int matched = 0;
int n_tag = cJSON_GetArraySize(tag_set);
for (int i = 0; i < n_tag; i++) {
cJSON *tag_obj = cJSON_GetArrayItem(tag_set, i);
if (NULL == tag_obj || tag_obj->type != cJSON_Object) {
goto error;
}
int ret = compare_each_tag(tag_obj, accept_tags, n_accept_tag);
if (ret < 0) {
return -1;
}
if(1 == ret) {
matched++;
}
}
if (matched == n_tag) {
return 1;
} else {
return 0;
}
error:
return -1;
}
//@param value is a JSON, like {"tags":[{"tag":"location","value":"北京/朝阳/华严北里/甲22号},{"tag":"isp","value":"电信"}]}
int parse_accept_tag(const char *value, struct rule_tag **result, void *logger)
{
cJSON *json = JSON_Parse(value);
if (!json) {
log_error(logger, MODULE_MAAT_RULE,
"parse accept tag Error before: %-200.200s", cJSON_GetErrorPtr());
return 0;
}
cJSON *tag = NULL, *tmp = NULL;
cJSON *array = cJSON_GetObjectItem(json, "tags");
int n_tag = cJSON_GetArraySize(array);
struct rule_tag *p = ALLOC(struct rule_tag, n_tag);
for (int i = 0; i < n_tag; i++) {
tag = cJSON_GetArrayItem(array, i);
tmp = cJSON_GetObjectItem(tag, "tag");
p[i].tag_name = maat_strdup(tmp->valuestring);
tmp = cJSON_GetObjectItem(tag, "value");
p[i].tag_val = maat_strdup(tmp->valuestring);
}
cJSON_Delete(json);
*result = p;
return n_tag;
}
//@param value {"tag_sets":[[{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}],[{"tag":"location","value":["北京"]},{"tag":"isp","value":["联通"]}]]}
//@return 1 on match, 0 on not match, -1 on error.
int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, int n_tag)
{
int ret = -1;
int n_set = 0;
cJSON *tag_set = NULL;
cJSON *tag_set_array = NULL;
cJSON *root = cJSON_Parse(value);
if (NULL == root) {
goto error;
}
tag_set_array = cJSON_GetObjectItem(root, "tag_sets");
if (NULL == tag_set_array || tag_set_array->type != cJSON_Array) {
goto error;
}
n_set = cJSON_GetArraySize(tag_set_array);
for (int i = 0; i < n_set; i++) {
tag_set = cJSON_GetArrayItem(tag_set_array, i);
if (NULL == tag_set || tag_set->type != cJSON_Array) {
goto error;
}
ret = compare_each_tag_set(tag_set, accept_tags, n_tag);
//match or error occurs.
if (ret != 0) {
break;
}
}
error:
cJSON_Delete(root);
return ret;
}
struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int district_id)
{
struct maat_item_inner *item = ALLOC(struct maat_item_inner, 1);
item->magic_num = ITEM_RULE_MAGIC;
item->item_id = item_id;
item->group_id = group_id;
item->district_id = district_id;
return item;
}
void maat_item_inner_free(struct maat_item_inner *item)
{
assert(item->magic_num == ITEM_RULE_MAGIC);
assert(item->expr_id_cnt == 0 || item->expr_id_cnt == item->expr_id_ub - item->expr_id_lb + 1);
item->magic_num = 0;
free(item);
}
struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance)
{
struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1);
maat_rt->version = version;
maat_rt->table_rt_mgr = table_runtime_manager_create(maat_instance->table_schema_mgr,
maat_instance->nr_worker_thread,
maat_instance->garbage_bin,
maat_instance->logger);
maat_rt->max_table_num = table_schema_manager_get_size(maat_instance->table_schema_mgr);
int ret = table_manager_init(maat_instance->tbl_mgr, maat_instance->garbage_bin);
if (ret < 0) {
FREE(maat_rt);
return NULL;
}
maat_rt->ref_tbl_mgr = maat_instance->tbl_mgr;
maat_rt->max_table_num = table_manager_table_count(maat_instance->table_mgr);
maat_rt->max_thread_num = maat_instance->nr_worker_thread;
maat_rt->logger = maat_instance->logger;
maat_rt->ref_garbage_bin = maat_instance->garbage_bin;
maat_rt->item_result_buff = ALLOC(struct scan_result, MAX_SCANNER_HIT_NUM * maat_instance->nr_worker_thread);
maat_rt->ref_cnt = alignment_int64_array_alloc(maat_instance->nr_worker_thread);
return maat_rt;
@@ -53,12 +248,12 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in
void maat_runtime_commit(struct maat_runtime *maat_rt, struct log_handle *logger)
{
for (size_t i = 0; i < maat_rt->max_table_num; i++) {
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, i);
if (NULL == table_rt) {
void *runtime = table_manager_get_runtime(maat_rt->ref_tbl_mgr, i);
if (NULL == runtime) {
continue;
}
table_runtime_commit(table_rt, maat_rt->version, maat_rt->max_thread_num, logger);
table_manager_commit_runtime(runtime, maat_rt->version, maat_rt->max_thread_num, logger);
}
maat_rt->last_update_time = time(NULL);
@@ -71,8 +266,8 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt)
}
if (maat_rt->table_rt_mgr != NULL) {
table_runtime_manager_destroy(maat_rt->table_rt_mgr);
maat_rt->table_rt_mgr = NULL;
table_manager_runtime_destroy(maat_rt->ref_tbl_mgr);
maat_rt->ref_table_mgr = NULL;
}
FREE(maat_rt);
@@ -81,7 +276,7 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt)
int maat_runtime_updating_flag(struct maat_runtime *maat_rt)
{
for (size_t i = 0; i < maat_rt->max_table_num; i++) {
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, i);
struct table_runtime *table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, i);
if (NULL == table_rt) {
continue;
}
@@ -105,42 +300,58 @@ void maat_start_cb(long long new_version, int update_type, void *u_param)
maat_instance->maat_version = new_version;
}
table_schema_manager_all_plugin_cb_start(maat_instance->table_schema_mgr, update_type);
int table_id = -1;
enum table_type table_type = TABLE_TYPE_MAX;
size_t table_cnt = table_manager_table_count(maat_instance->tbl_mgr);
for (size_t i = 0; i < table_cnt; i++) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, i);
if (table_type != TABLE_TYPE_PLUGIN) {
continue;
}
void *schema = table_manager_get_schema(maat_instance->tbl_mgr, i);
plugin_table_all_callback_start((struct plugin_schema *)schema, update_type);
}
}
int maat_update_cb(const char *table_name, const char *line, void *u_param)
{
if (NULL == table_name || NULL == line || NULL == u_param) {
return 0;
}
struct maat *maat_instance =(struct maat *)u_param;
struct maat_runtime* maat_rt = NULL;
int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, table_name);
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, table_name);
if (table_id < 0) {
log_error(maat_instance->logger, MODULE_MAAT_RULE, "update warning, unknown table name %s", table_name);
log_error(maat_instance->logger, MODULE_MAAT_RULE,
"update warning, unknown table name %s", table_name);
return -1;
}
struct table_schema* table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
if (NULL == table_schema) {
void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
if (NULL == schema) {
log_error(maat_instance->logger, MODULE_MAAT_RULE,
"update warning, table name %s doesn't have table schema", table_name);
return -1;
}
table_schema_set_updating_name(table_schema, table_name);
if (maat_instance->creating_maat_rt != NULL) {
maat_rt = maat_instance->creating_maat_rt;
} else {
maat_rt = maat_instance->maat_rt;
}
struct table_item *table_item = table_schema_line_to_item(line, table_schema, maat_instance->accept_tags,
maat_instance->n_accept_tag, maat_instance->logger);
if (table_item != NULL) {
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id);
table_runtime_update(table_rt, table_schema, line, table_item, maat_instance->logger);
table_item_free(table_item);
void *runtime = table_manager_get_runtime(maat_rt->tbl_mgr, table_id);
int ret = table_manager_update_runtime(runtime, schema, line);
if (ret < 0) {
log_error(maat_instance->logger, MODULE_MAAT_RULE,
"table manager update runtime error, table_name:%s", table_name);
return -1;
}
//TODO: by luis
//int is_valid = table_manager_get_valid
return 0;
}
@@ -150,7 +361,7 @@ uint32_t maat_runtime_rule_num(struct maat_runtime *maat_rt)
struct table_runtime *table_rt = NULL;
for (size_t i = 0; i < maat_rt->max_table_num; i++) {
table_rt = table_runtime_get(maat_rt->table_rt_mgr, i);
table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, i);
if (table_rt != NULL) {
total += table_runtime_rule_count(table_rt);
}
@@ -163,8 +374,8 @@ void maat_finish_cb(void *u_param)
{
struct maat *maat_instance = (struct maat *)u_param;
table_schema_manager_all_plugin_cb_finish(maat_instance->table_schema_mgr);
//table_manager_all_plugin_cb_finish(maat_instance->table_schema_mgr);
plugin_table_all_callback_finish(maat_)
if (maat_instance->creating_maat_rt != NULL) {
maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt);
maat_runtime_commit(maat_instance->creating_maat_rt, maat_instance->logger);
@@ -290,7 +501,7 @@ void *rule_monitor_loop(void *arg)
maat_runtime_destroy(maat_instance->maat_rt);
maat_garbage_bin_free(maat_instance->garbage_bin);
table_schema_manager_destroy(maat_instance->table_schema_mgr);
table_manager_destroy(maat_instance->tbl_mgr);
if (maat_instance->input_mode == DATA_SOURCE_REDIS) {
if (maat_instance->mr_ctx.read_ctx != NULL) {
@@ -304,6 +515,12 @@ void *rule_monitor_loop(void *arg)
}
}
for (int i = 0; i < maat_instance->n_accept_tag; i++) {
FREE(maat_instance->accept_tags[i].tag_name);
FREE(maat_instance->accept_tags[i].tag_val);
}
FREE(maat_instance->accept_tags);
FREE(maat_instance);
return NULL;

759
src/maat_table.cpp Normal file
View File

@@ -0,0 +1,759 @@
/*
**********************************************************************************************
* File: maat_table.cpp
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include <limits.h>
#include <assert.h>
#include "log/log.h"
#include "maat_utils.h"
#include "maat_table.h"
#include "maat_rule.h"
#include "maat_garbage_collection.h"
#include "maat_kv.h"
#include "maat_expr.h"
#include "maat_ip.h"
#include "maat_compile.h"
#include "maat_group.h"
#include "maat_plugin.h"
#include "maat_ip_plugin.h"
#define MODULE_TABLE module_name_str("maat.table")
struct table_item {
enum table_type table_type;
void *custom_item;
};
struct maat_table {
int table_id;
char table_name[NAME_MAX];
enum table_type table_type;
int valid_column;
void *schema;
void *runtime;
};
struct table_manager {
struct maat_table *tbl[MAX_TABLE_NUM];
size_t n_table;
struct rule_tag *accept_tags;
int n_accept_tag;
struct maat_kv_store *tablename2id_map;
struct log_handle *logger;
};
struct table_operations {
enum table_type type;
void *(*new_schema)(cJSON *json, const char *table_name, struct log_handle *logger);
void (*free_schema)(void *schema);
void *(*new_runtime)(void *schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
void (*free_runtime)(void *runtime);
int (*update_runtime)(void *runtime, void *schema, const char *line, int valid_column);
int (*commit_runtime)(void *runtime);
};
struct table_operations table_ops[TABLE_TYPE_MAX] = {
{
.type = TABLE_TYPE_EXPR,
.new_schema = expr_schema_new,
.free_schema = expr_schema_free,
.new_runtime = expr_runtime_new,
.free_runtime = expr_runtime_free,
.update_runtime = expr_runtime_update,
.commit_runtime = expr_runtime_commit
},
{
.type = TABLE_TYPE_EXPR_PLUS,
.new_schema = expr_schema_new,
.free_schema = expr_schema_free,
.new_runtime = expr_runtime_new,
.free_runtime = expr_runtime_free,
.update_runtime = expr_runtime_update,
.commit_runtime = expr_runtime_commit
},
{
.type = TABLE_TYPE_IP_PLUS,
.new_schema = ip_plus_schema_new,
.free_schema = ip_plus_schema_free,
.new_runtime = ip_plus_runtime_new,
.free_runtime = ip_plus_runtime_free,
.update_runtime = ip_plus_runtime_update,
.commit_runtime = ip_plus_runtime_commit
},
{
.type = TABLE_TYPE_INTERVAL,
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_INTERVAL_PLUS,
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_DIGEST,
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_SIMILARITY,
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_CONJUNCTION,
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_PLUGIN,
.new_schema = plugin_schema_new,
},
{
.type = TABLE_TYPE_IP_PLUGIN,
.new_schema = ip_plugin_schema_new,
.free_schema = ip_plugin_schema_free,
.new_runtime = ip_plugin_runtime_new,
.free_runtime = ip_plugin_runtime_free,
.update_runtime = ip_plugin_runtime_update,
.commit_runtime = ip_plugin_runtime_commit
},
{
.type = TABLE_TYPE_FQDN_PLUGIN,
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_BOOL_PLUGIN,
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_VIRTUAL,
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_COMPILE,
.new_schema = compile_schema_new,
.free_schema = compile_schema_free,
.new_runtime = compile_runtime_new,
.free_runtime = compile_runtime_free,
.update_runtime = compile_runtime_update,
.commit_runtime = compile_runtime_commit
},
{
.type = TABLE_TYPE_GROUP2COMPILE,
.new_schema = group2compile_schema_new,
.free_schema = group2compile_schema_free,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = group2compile_runtime_update,
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_GROUP2GROUP,
.new_schema = group2group_schema_new,
.free_schema = group2group_schema_free,
.new_runtime = group2group_runtime_new,
.free_runtime = group2group_runtime_free,
.update_runtime = group2group_runtime_update,
.commit_runtime = group2group_runtime_commit
}
};
void *maat_table_schema_new(cJSON *json, const char *table_name, enum table_type table_type)
{
void *schema = NULL;
if (table_ops[table_type].new_schema != NULL) {
schema = table_ops[table_type].new_schema(json, table_name, logger);
}
return schema;
}
void maat_table_schema_free(void *schema, enum table_type table_type)
{
if (NULL == schema) {
return;
}
if (table_ops[table_type].free_schema != NULL) {
table_ops[table_type].free_schema(schema);
}
}
static void register_reserved_word(struct maat_kv_store *reserved_word_map)
{
maat_kv_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE);
maat_kv_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE);
maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP);
maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR);
maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS);
maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS);
maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN);
maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN);
maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL);
}
static void register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id_map,
struct log_handle *logger)
{
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) {
return;
}
int table_id = item->valueint;
item = cJSON_GetObjectItem(json, "table_name");
if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"table(table_id:%d) has no table name", table_id);
return;
}
if (strlen(item->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE_SCHEMA,
"table(table_id:%d) name %s length too long", table_id, item->valuestring);
return;
}
maat_kv_register(tablename2id_map, item->valuestring, table_id);
}
struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_word_map,
struct log_handle *logger)
{
struct maat_table *ptable = ALLOC(struct maat_table, 1);
int ret = -1;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) {
goto error;
}
if (item->valueint >= MAX_TABLE_NUM) {
log_error(logger, MODULE_TABLE,
"table(table_id:%d) exceed maxium %d", MAX_TABLE_NUM);
goto error;
}
ptable->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "table_name");
if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"table(table_id:%d) has no table name", ptable->table_id);
goto error;
}
if (strlen(item->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE,
"table(table_id:%d) name %s length too long",
ptable->table_id, item->valuestring);
goto error;
}
memcpy(ptable->table_name, item->valuestring, strlen(item->valuestring));
item = cJSON_GetObjectItem(json, "table_type");
if (NULL == item || item->type != cJSON_String) {
goto error;
}
ret = maat_kv_read(reserved_word_map, item->valuestring, (int*)&(ptable->table_type));
if (ret < 0) {
log_error(logger, MODULE_TABLE, "table_type %s is illegal", item->valuestring);
goto error;
}
item = cJSON_GetObjectItem(json, "valid_column");
if (NULL == item && item->type != cJSON_Number) {
goto error;
}
ptable->valid_column = item->valueint;
return ptable;
error:
FREE(ptable);
return NULL;
}
void maat_table_free(struct maat_table *maat_tbl)
{
if (NULL == maat_tbl) {
return;
}
if (maat_tbl->schema != NULL) {
maat_table_schema_free(maat_tbl->schema);
maat_tbl->schema = NULL;
}
if (maat_tbl->runtime != NULL) {
maat_table_runtime_free(maat_tbl->runtime);
maat_tbl->runtime = NULL;
}
FREE(maat_tbl);
}
struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags,
struct log_handle *logger)
{
if (NULL == table_info_path) {
return NULL;
}
unsigned char *json_buff = NULL;
size_t json_buff_sz = 0;
int ret = load_file_to_memory(table_info_path, &json_buff, &json_buff_sz);
if (ret < 0) {
log_error(logger, MODULE_TABLE, "Maat read table info %s error.", table_info_path);
return NULL;
}
cJSON *root = NULL;
cJSON *json = NULL;
root = cJSON_Parse((const char *)json_buff);
if (!root) {
log_error(logger, MODULE_TABLE, "Error before: %-200.200s", cJSON_GetErrorPtr());
FREE(json_buff);
return NULL;
}
int json_array_size = cJSON_GetArraySize(root);
if (json_array_size <= 0) {
log_error(logger, MODULE_TABLE, "invalid json content in %s", table_info_path);
free(json_buff);
return NULL;
}
struct maat_kv_store *reserved_word_map = maat_kv_store_new();
register_reserved_word(reserved_word_map);
struct table_manager *tbl_mgr = ALLOC(struct table_manager, 1);
tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger);
tbl_mgr->logger = logger;
tbl_mgr->tablename2id_map = maat_kv_store_new();
int default_compile_table_id = MAX_TABLE_NUM;
for (int i = 0; i < json_array_size; i++) {
json = cJSON_GetArrayItem(root, i);
if (json != NULL && json->type == cJSON_Object) {
register_tablename2id(json, tbl_mgr->tablename2id_map, logger);
}
}
for (int i = 0; i < json_array_size; i++) {
json = cJSON_GetArrayItem(root, i);
if (json != NULL && json->type == cJSON_Object) {
struct maat_table *maat_tbl = maat_table_new(json, reserved_word_map, logger);
if (NULL == maat_tbl) {
log_error(logger, MODULE_TABLE, "Maat table new failed.");
continue;
}
maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->table_type);
if (NULL == maat_tbl->schema) {
log_error(logger, MODULE_TABLE, "Maat table schema new failed, table_name:%d",
maat_tbl->table_name);
maat_table_free(maat_tbl);
continue;
}
if (maat_tbl->table_type == TABLE_TYPE_COMPILE) {
if (maat_tbl->table_id < default_compile_table_id) {
default_compile_table_id = maat_tbl->table_id;
}
}
tbl_mgr->tbl[maat_tbl->table_id] = maat_tbl;
tbl_mgr->n_table++;
}
}
assert(default_compile_table_id != MAX_TABLE_NUM);
tbl_mgr->default_compile_table_id = default_compile_table_id;
log_info(logger, MODULE_TABLE, "default compile table id: %d", default_compile_table_id);
maat_kv_store_free(reserved_word_map);
cJSON_Delete(root);
FREE(json_buff);
return tbl_mgr;
}
void *maat_table_runtime_new(void *schema, enum table_type table_type,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
void *runtime = NULL;
if (table_ops[table_type].new_runtime != NULL) {
runtime = table_ops[table_type].new_runtime(schema, garbage_bin, logger);
}
return runtime;
#if 0
switch (table_rt->table_type) {
case TABLE_TYPE_COMPILE:
break;
case TABLE_TYPE_GROUP2COMPILE:
break;
case TABLE_TYPE_GROUP2GROUP:
table_rt->custom_rt = group2group_runtime_new(logger);
table_rt->g2g_rt.group_topo = maat_group_topology_new(logger);
break;
case TABLE_TYPE_EXPR:
table_rt->expr_rt.htable = rcu_hash_new(expr_ex_data_free);
table_rt->expr_rt.scan_mode = expr_table_schema_get_scan_mode(table_schema);
break;
case TABLE_TYPE_IP_PLUS:
table_rt->ip_plus_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free);
ex_container_ctx = ALLOC(struct ex_container_ctx, 1);
ex_container_ctx->custom_data_free = free;
//ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx);
break;
case TABLE_TYPE_PLUGIN:
table_rt->plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free);
break;
case TABLE_TYPE_IP_PLUGIN:
table_rt->ip_plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free);
ex_container_ctx = ALLOC(struct ex_container_ctx, 1);
ex_container_ctx->custom_data_free = free;
//ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx);
break;
default:
break;
}
#endif
}
void maat_table_runtime_free(void *runtime, enum table_type table_type)
{
if (NULL == runtime) {
return;
}
table_ops[table_type].free_runtime(runtime);
#if 0
switch (table_rt->table_type) {
case TABLE_TYPE_COMPILE:
bool_matcher_free(table_rt->compile_rt.bm);
maat_compile_hash_free(&(table_rt->compile_rt.compile_hash));
break;
case TABLE_TYPE_GROUP2COMPILE:
break;
case TABLE_TYPE_GROUP2GROUP:
maat_group_topology_free(table_rt->g2g_rt.group_topo);
break;
case TABLE_TYPE_EXPR:
adapter_hs_destroy(table_rt->expr_rt.hs);
rcu_hash_free(table_rt->expr_rt.htable);
break;
case TABLE_TYPE_PLUGIN:
ex_data_runtime_free(table_rt->plugin_rt.ex_data_rt);
break;
case TABLE_TYPE_IP_PLUGIN:
ip_matcher_free(table_rt->ip_plugin_rt.ip_matcher);
ex_data_runtime_free(table_rt->ip_plugin_rt.ex_data_rt);
break;
default:
break;
}
#endif
}
int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *garbage_bin)
{
if (NULL == tbl_mgr) {
return -1;
}
assert(tbl_mgr->n_table != 0);
size_t i = 0;
int g2g_group_id = MAX_TABLE_NUM;
enum table_type table_type = TABLE_TYPE_MAX;
for (i = 0; i < MAX_TABLE_NUM; i++) {
void *schema = table_manager_get_schema(tbl_mgr, i);
if (NULL == schema) {
continue;
}
table_type = table_manager_get_table_type(tbl_mgr, i);
if (table_type == TABLE_TYPE_GROUP2GROUP) {
g2g_group_id = i;
}
tbl_mgr->tbl[i]->runtime = maat_table_runtime_new(schema, table_type, garbage_bin, logger);
}
assert(g2g_group_id != MAX_TABLE_NUM);
/* group2compile runtime depends on associated compile runtime,
must make sure associated compile runtime already exist */
for (i = 0; i < MAX_TABLE_NUM; i++) {
void *runtime = tbl_mgr->tbl[i]->runtime;
if (NULL == runtime) {
continue;
}
table_type = table_manager_get_table_type(tbl_mgr, i);
if (table_type != TABLE_TYPE_GROUP2COMPILE) {
continue;
}
void *schema = table_manager_get_schema(tbl_mgr, i);
//int associated_compile_table_id = table_schema_get_associated_table_id(table_schema);
//TODO: by luis
int associated_compile_table_id = -1;
void *compile_rt = table_manager_get_runtime(tbl_mgr, associated_compile_table_id);
void *g2g_rt = table_manager_get_runtime(tbl_mgr, g2g_group_id);
table_rt->g2c_rt.ref_compile_rt = &(compile_table_rt->compile_rt);
table_rt->g2c_rt.ref_g2g_rt = &(g2g_table_rt->g2g_rt);
assert(table_rt->g2c_rt.ref_compile_rt != NULL);
assert(table_rt->g2c_rt.ref_g2g_rt != NULL);
}
return 0;
}
void table_manager_deinit(struct table_manager *tbl_mgr)
{
if (NULL == tbl_mgr) {
return;
}
for(size_t i = 0; i < MAX_TABLE_NUM; i++) {
maat_table_runtime_free(tbl_mgr->tbl[i]->runtime, tbl_mgr->tbl[i]->table_type);
tbl_mgr->tbl[i]->runtime = NULL;
}
}
void table_manager_destroy(struct table_manager *tbl_mgr)
{
if (NULL == tbl_mgr) {
return;
}
for (size_t i = 0; i < MAX_TABLE_NUM; i++) {
assert(NULL == tbl_mgr->tbl[i]->runtime);
if (NULL == tbl_mgr->tbl[i]->schema) {
continue;
}
table_schema_free(tbl_mgr->tbl[i]->schema);
tbl_mgr->tbl[i]->schema = NULL;
}
maat_kv_store_free(tbl_mgr->tablename2id_map);
FREE(tbl_mgr);
}
size_t table_manager_table_count(struct table_manager *tbl_mgr)
{
return MAX_TABLE_NUM;
}
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name)
{
}
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id)
{
}
int table_manager_get_logger(struct table_manager *tbl_mgr)
{
}
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id)
{
if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) {
return NULL;
}
if (NULL == tbl_mgr->tbl[table_id]) {
return NULL;
}
return tbl_mgr->tbl[table_id]->schema;
}
struct ex_data_schema *table_manager_get_table_ex_data_schema(struct table_manager *tbl_mgr, int table_id)
{
}
int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id)
{
if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) {
return -1;
}
if (NULL == tbl_mgr->tbl[table_id]) {
return -1;
}
return tbl_mgr->tbl[table_id]->valid_column;
}
int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags)
{
}
void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id)
{
if (NULL == tbl_mgr || (table_id < 0) || (table_id >= MAX_TABLE_NUM)) {
return NULL;
}
assert(table_id < (int)tbl_mgr->n_table);
if (NULL == tbl_mgr->tbl[table_id]) {
return NULL;
}
return tbl_mgr->tbl[table_id]->runtime;
}
int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line)
{
void *schema = table_manager_get_schema(tbl_mgr, table_id);
if (NULL == schema) {
return -1;
}
void *runtime = table_manager_get_runtime(tbl_mgr, table_id);
if (NULL == runtime) {
return -1;
}
int valid_column = table_manager_get_valid_column(tbl_mgr, table_id);
if (valid_column < 0) {
return -1;
}
enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
if (table_type == TABLE_TYPE_MAX) {
return -1;
}
if (NULL == table_ops[table_type].update_runtime) {
return -1;
}
return table_ops[table_type].update_runtime(runtime, schema, line, valid_column);
#if 0
switch (table_rt->table_type) {
case TABLE_TYPE_COMPILE:
ret = compile_runtime_update(table_rt->custom_rt, table_item->custom_item, table_schema, table_name, table_rt->ref_garbage_bin, logger);
break;
case TABLE_TYPE_GROUP2COMPILE:
ret = group2compile_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, table_rt->ref_garbage_bin, logger);
break;
case TABLE_TYPE_GROUP2GROUP:
ret = group2group_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, logger);
break;
case TABLE_TYPE_EXPR:
ret = expr_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, table_rt->ref_garbage_bin, logger);
break;
case TABLE_TYPE_IP_PLUS:
ret = ip_plus_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, table_rt->ref_garbage_bin, logger);
break;
case TABLE_TYPE_PLUGIN:
ret = plugin_runtime_update(table_rt->custom_rt, table_item->custom_item, table_schema, table_name, row, logger);
break;
case TABLE_TYPE_IP_PLUGIN:
ret = ip_plugin_runtime_update(table_rt->custom_rt, table_item->custom_item, table_schema, table_name, row, logger);
break;
default:
break;
}
if (ret < 0) {
return;
}
#endif
#if 0
if (is_valid == 0) {
table_rt->rule_num--;
} else {
table_rt->rule_num++;
}
#endif
}
void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id)
{
enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
if (table_type == TABLE_TYPE_MAX) {
return;
}
void *runtime = table_manager_get_runtime(tbl_mgr, table_id);
if (NULL == runtime) {
return;
}
table_ops[table_type].commit_runtime(runtime);
}

1290
src/maat_table_runtime.cpp
View File

File diff suppressed because it is too large Load Diff

2481
src/maat_table_schema.cpp
View File

File diff suppressed because it is too large Load Diff

13
src/maat_utils.cpp
View File

@@ -45,6 +45,19 @@ char *maat_strdup(const char *s)
return d;
}
int get_column_value(const char *line, int column_seq)
{
size_t column_offset = 0;
size_t column_len = 0;
int ret = get_column_pos(line, column_seq, &column_offset, &column_len);
if (ret < 0) {
return -1;
}
return atoi(line + column_offset);
}
int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len)
{
int i = 0;

59
src/maat_virtual.cpp Normal file
View File

@@ -0,0 +1,59 @@
/*
**********************************************************************************************
* File: maat_virtual.cpp
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include "cJSON/cJSON.h"
#include "maat_kv.h"
#include "utils.h"
#include "maat_utils.h"
#include "log/log.h"
#include "maat_virtual.h"
#define MODULE_VIRTUAL module_name_str("maat.virtual")
struct virtual_schema {
int physical_table_id[SCAN_TYPE_MAX];
};
void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
{
size_t read_cnt = 0;
cJSON *item = cJSON_GetObjectItem(json, "physical_table");
if (NULL == item || item->type != cJSON_Array) {
log_error(logger, MODULE_VIRTUAL, "virtual table %s has no physical_table column", table_name);
return NULL;
}
read_cnt++;
#if 0
struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);
int cnt = cJSON_GetArraySize(item);
for (int i = 0; i < cnt; i++) {
cJSON *tmp_item = cJSON_GetArrayItem(item, i);
if (tmp_item != NULL && tmp_item->type == cJSON_String) {
int table_id = -1;
/* physical table should already exist */
int ret = maat_kv_read(tablename2id_map, tmp_item->valuestring, &table_id);
if (ret < 0) {
return -1;
}
enum scan_type table_scan_type = table_schema_get_scan_type(table_array[table_id]);
vt_schema->physical_table_id[table_scan_type]= table_id;
}
}
if (read_cnt < 3) {
FREE(vt_schema);
return NULL;
}
return vt_schema;
#endif
}

13
src/rcu_hash.cpp
View File

@@ -267,6 +267,19 @@ size_t rcu_hash_count(struct rcu_hash_table *htable)
}
}
size_t rcu_hash_updating_count(struct rcu_hash_table *htable)
{
if (NULL == htable) {
return 0;
}
if (htable->effective_hash == 'a') {
return HASH_CNT(hh_b, htable->hashmap_b);
} else {
return HASH_CNT(hh_a, htable->hashmap_a);
}
}
void rcu_hash_commit(struct rcu_hash_table *htable)
{
if (NULL == htable) {

223
test/maat_framework_gtest.cpp
View File

@@ -4,8 +4,6 @@
#include "maat/maat.h"
#include "maat_rule.h"
#include "maat_utils.h"
#include "maat_table_schema.h"
#include "maat_table_runtime.h"
#include "maat_command.h"
#include "IPMatcher.h"
#include "json2iris.h"
@@ -17,250 +15,238 @@ const char *json_path="./maat_json.json";
const char *json_filename = "maat_json.json";
TEST(maat_scan_string, hit_one_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL");
char data[128] = "i.ytimg.com";
char scan_data[128] = "hello";
int results[5] = {0};
size_t n_result = 0;
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(results[0], 30);
int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191);
struct maat_hit_path hit_path[128] = {0};
int n_read = 0;
n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, sizeof(hit_path));
maat_state_free(&state);
}
TEST(maat_scan_string, hit_two_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL");
char data[128] = "should hit aaa bbb";
int results[5] = {0};
size_t n_result = 0;
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state);
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 2);
EXPECT_EQ(n_hit_result, 2);
EXPECT_EQ(results[0], 28);
EXPECT_EQ(results[1], 27);
maat_state_free(&state);
}
TEST(maat_scan_string, hit_three_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL");
char data[128] = "should hit aaa bbb C#中国";
int results[5] = {0};
size_t n_result = 0;
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state);
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 3);
EXPECT_EQ(n_hit_result, 3);
EXPECT_EQ(results[0], 28);
EXPECT_EQ(results[1], 27);
EXPECT_EQ(results[2], 18);
maat_state_free(&state);
}
TEST(maat_scan_ipv4, hit_ip_and_port) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG");
int table_id = maat_table_get_id(g_maat_instance, "IP_PLUS_CONFIG");
char ip_str[32] = "192.168.58.19";
uint16_t port = 20000;
struct addr_4tuple addr;
addr.type = IP_TYPE_V4;
int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip);
uint32_t sip;
int ret = inet_pton(AF_INET, ip_str, &sip);
EXPECT_EQ(ret, 1);
addr.ipv4.sport = htons(port);
int results[3] = {-1};
size_t n_result = 0;
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 7);
maat_state_free(&state);
port = 20001;
addr.ipv4.sport = htons(port);
state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
}
TEST(maat_scan_ipv4, hit_ip_and_port_range) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG");
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG");
char ip_str[32] = "192.168.50.24";
uint16_t port = 1;
struct addr_4tuple addr;
struct addr_2tuple addr;
addr.type = IP_TYPE_V4;
int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip);
EXPECT_EQ(ret, 1);
addr.ipv4.sport = htons(port);
int results[3] = {-1};
size_t n_result = 0;
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 4);
maat_state_free(&state);
port = 40000;
addr.ipv4.sport = htons(port);
memset(results, 0, sizeof(results));
n_result = 0;
n_hit_result = 0;
state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 4);
maat_state_free(&state);
port = 40001;
addr.ipv4.sport = htons(port);
memset(results, 0, sizeof(results));
n_result = 0;
n_hit_result = 0;
state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
EXPECT_EQ(n_hit_result, 0);
}
TEST(maat_scan_ipv4, hit_ip_range_and_port_range) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG");
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG");
char ip_str1[32] = "10.0.1.20";
char ip_str2[32] = "10.0.1.25";
char ip_str3[32] = "10.0.1.26";
uint16_t port1 = 1;
uint16_t port2 = 443;
struct addr_4tuple addr;
struct addr_2tuple addr;
addr.type = IP_TYPE_V4;
int ret = inet_pton(AF_INET, ip_str1, &addr.ipv4.sip);
EXPECT_EQ(ret, 1);
addr.ipv4.sport = htons(port1);
int results[3] = {-1};
size_t n_result = 0;
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 8);
maat_state_free(&state);
ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip);
EXPECT_EQ(ret, 1);
addr.ipv4.sport = htons(port2);
state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 8);
maat_state_free(&state);
ret = inet_pton(AF_INET, ip_str3, &addr.ipv4.sip);
EXPECT_EQ(ret, 1);
addr.ipv4.sport = htons(port2);
state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
}
TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG");
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG");
char ip_str1[32] = "192.168.0.1";
char ip_str2[32] = "192.168.0.0";
uint16_t port = 5210;
struct addr_4tuple addr;
struct addr_2tuple addr;
addr.type = IP_TYPE_V4;
int ret = inet_pton(AF_INET, ip_str1, &addr.ipv4.sip);
EXPECT_EQ(ret, 1);
addr.ipv4.sport = htons(port);
int results[3] = {-1};
size_t n_result = 0;
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 50);
maat_state_free(&state);
ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip);
EXPECT_EQ(ret, 1);
state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
}
TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG");
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG");
char ip_str[32] = "192.168.40.10";
uint16_t port = 443;
struct addr_4tuple addr;
struct addr_2tuple addr;
addr.type = IP_TYPE_V4;
int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip);
EXPECT_EQ(ret, 1);
addr.ipv4.sport = htons(port);
int results[3] = {-1};
size_t n_result = 0;
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 2);
EXPECT_EQ(n_hit_result, 2);
EXPECT_EQ(results[0], 63);
EXPECT_EQ(results[1], 67);
maat_state_free(&state);
port = 442;
addr.ipv4.sport = htons(port);
state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
}
TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG");
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG");
char ip_str[32] = "1001:da8:205:1::101";
uint16_t port = 5210;
struct addr_4tuple addr;
struct addr_2tuple addr;
addr.type = IP_TYPE_V6;
int ret = inet_pton(AF_INET6, ip_str, &addr.ipv6.sip);
EXPECT_EQ(ret, 1);
addr.ipv6.sport = htons(port);
int results[3] = {-1};
size_t n_result = 0;
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 47);
maat_state_free(&state);
port = 442;
addr.ipv6.sport = htons(port);
state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
}
TEST(maat_scan_string, dynamic_config) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "HTTP_URL");
char data[128] = "hello world";
int results[5] = {0};
size_t n_result = 0;
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state);
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
const char *table_name = "HTTP_URL";
const char *table_line = "9999\t8888\thello world\t0\t0\t0\t1\t";
@@ -273,30 +259,29 @@ TEST(maat_scan_string, dynamic_config) {
sleep(2);
state = NULL;
ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state);
ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 9999);
maat_state_free(&state);
}
TEST(maat_scan_ip, dynamic_config) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG");
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG");
char ip_str[32] = "10.0.6.201";
uint16_t port = 443;
struct addr_4tuple addr;
struct addr_2tuple addr;
addr.type = IP_TYPE_V4;
int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip);
EXPECT_EQ(ret, 1);
addr.ipv4.sport = htons(port);
int results[3] = {-1};
size_t n_result = 0;
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
const char *table_name = "IP_PLUS_CONFIG";
const char *table_line = "9998\t8887\t4\trange\t10.0.6.201\t255.255.0.0\trange\t0\t65535\t6\t0\t1";
@@ -309,10 +294,11 @@ TEST(maat_scan_ip, dynamic_config) {
sleep(2);
state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state);
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 9998);
maat_state_free(&state);
}
int count_line_num_cb(const char *table_name, const char *line, void *u_para)
@@ -424,6 +410,7 @@ int main(int argc, char ** argv)
maat_options_set_logger(opts, logger);
g_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
ret=RUN_ALL_TESTS();

55
test/maat_input_mode_gtest.cpp
View File

@@ -2,7 +2,6 @@
#include "maat/maat.h"
#include "maat_utils.h"
#include "maat_rule.h"
#include "maat_table_schema.h"
#include "json2iris.h"
#include "maat_config_monitor.h"
@@ -31,17 +30,19 @@ TEST(json_mode, maat_scan_string) {
struct maat *maat_instance = maat_new(opts, table_info_path);
EXPECT_NE(maat_instance, nullptr);
struct table_schema_manager *table_schema_mgr = maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL");
char data[128] = "i.ytimg.com";
int result_array[5] = {0};
size_t n_result_array = 0;
int ret = maat_scan_string(maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
char scan_data[128] = "hello";
int results[5] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 1);
EXPECT_EQ(result_array[0], 30);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191);
maat_options_free(opts);
maat_state_free(&state);
maat_free(maat_instance);
}
@@ -72,17 +73,19 @@ TEST(iris_mode, maat_scan_string) {
struct maat *maat_instance = maat_new(opts, table_info_path);
EXPECT_NE(maat_instance, nullptr);
struct table_schema_manager *table_schema_mgr = maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL");
char data[128] = "i.ytimg.com";
int result_array[5] = {0};
size_t n_result_array = 0;
int ret = maat_scan_string(maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
char scan_data[128] = "hello";
int results[5] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 1);
EXPECT_EQ(result_array[0], 30);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191);
maat_options_free(opts);
maat_state_free(&state);
maat_free(maat_instance);
}
@@ -190,17 +193,19 @@ TEST(redis_mode, maat_scan_string) {
maat_options_set_logger(opts, logger);
struct maat *maat_instance = maat_new(opts, table_info_path);
struct table_schema_manager *table_schema_mgr = maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr "HTTP_URL");
char data[128] = "i.ytimg.com";
int result_array[5] = {0};
size_t n_result_array = 0;
int ret = maat_scan_string(maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
char scan_data[128] = "hello";
int results[5] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result_array, 1);
EXPECT_EQ(result_array[0], 30);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191);
maat_options_free(opts);
maat_state_free(&state);
log_handle_destroy(maat_instance->logger);
maat_free(maat_instance);
}

26
test/maat_json.json
View File

@@ -2283,6 +2283,32 @@
]
}
]
},
{
"compile_id": 191,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "hello",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
],
"plugin_table": [

228
test/table_info.conf
View File

@@ -1,50 +1,123 @@
[
{
"table_id":1,
"table_name":"HTTP_URL",
"table_type":"expr",
"scan_mode":"block",
"item_id":1,
"group_id":2,
"table_name":"COMPILE",
"table_type":"compile",
"valid_column":8,
"custom": {
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6,
"is_valid":7
"compile_id":1,
"service_id":2,
"action":3,
"do_blacklist":4,
"do_log":5,
"tags":6,
"user_region":7,
"clause_num":9,
"evaluation_order":10
}
},
{
"table_id":2,
"table_name":"IP_PLUGIN_TABLE",
"table_type":"ip_plugin",
"item_id":1,
"table_name":"GROUP2COMPILE",
"table_type":"group2compile",
"valid_column":3,
"custom": {
"ip_type":2,
"start_ip":3,
"end_ip":4,
"is_valid":5
"associated_compile_table_id":1,
"group_id":1,
"compile_id":2,
"not_flag":4,
"virtual_table_name":5,
"clause_index":6
}
},
{
"table_id":3,
"table_name":"PLUGIN_TABLE",
"table_type":"plugin",
"item_id":1,
"table_name":"COMPILE_2",
"table_type":"compile",
"valid_column":8,
"custom": {
"key":2,
"tag":3,
"is_valid":4,
"foreign":[6,8,10]
"compile_id":1,
"service_id":2,
"action":3,
"do_blacklist":4,
"do_log":5,
"tags":6,
"user_region":7,
"clause_num":9,
"evaluation_order":10
}
},
{
"table_id":4,
"table_name":"IP_PLUS_CONFIG",
"table_type":"ip_plus",
"table_name":"GROUP2COMPILE_2",
"table_type":"group2compile",
"associated_compile_table_id":3,
"valid_column":3,
"custom": {
"group_id":1,
"compile_id":2,
"not_flag":4,
"virtual_table_name":5,
"clause_index":6
}
},
{
"table_id":5,
"table_name":"GROUP2GROUP",
"table_type":"group2group",
"valid_column":3,
"custom": {
"group_id":1,
"superior_group_id":2
}
},
{
"table_id":6,
"table_name":"HTTP_URL",
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":7,
"table_name":"IP_PLUGIN_TABLE",
"table_type":"ip_plugin",
"valid_column":5,
"custom": {
"item_id":1,
"ip_type":2,
"start_ip":3,
"end_ip":4
}
},
{
"table_id":8,
"table_name":"PLUGIN_TABLE",
"table_type":"plugin",
"valid_column":4,
"custom": {
"item_id":1,
"key":2,
"tag":3,
"foreign":[6,8,10]
}
},
{
"table_id":9,
"table_name":"IP_PLUS_CONFIG",
"table_type":"ip_plus",
"valid_column":18,
"custom": {
"item_id":1,
"group_id":2,
"addr_type":3,
"saddr_format":4,
"sip1":5,
@@ -59,108 +132,13 @@
"dport1":14,
"dport2":15,
"proto":16,
"direction":17,
"is_valid":18
}
},
{
"table_id":5,
"table_name":"COMPOSITION_IP_SOURCE",
"table_type":"virtual",
"physical_table":["IP_PLUS_CONFIG"]
},
{
"table_id":6,
"table_name":"COMPOSITION_IP_DEST",
"table_type":"virtual",
"physical_table":["IP_PLUS_CONFIG"]
},
{
"table_id":7,
"table_name":"COMPOSITION_IP_SESSION",
"table_type":"virtual",
"physical_table":["IP_PLUS_CONFIG"]
},
{
"table_id":8,
"table_name":"COMPOSITION_IP",
"table_type":"composition",
"composition_table": {
"source": "COMPOSITION_IP_SOURCE",
"dest": "COMPOSITION_IP_DEST",
"session": "COMPOSITION_IP_SESSION"
}
},
{
"table_id":9,
"table_name":"COMPILE_1",
"table_type":"compile",
"custom": {
"compile_id":1,
"service_id":2,
"action":3,
"do_blacklist":4,
"do_log":5,
"tags":6,
"user_region":7,
"is_valid":8,
"clause_num":9,
"evaluation_order":10
"direction":17
}
},
{
"table_id":10,
"table_name":"GROUP2COMPILE_1",
"table_type":"group2compile",
"associated_compile_table_id":9,
"custom": {
"group_id":1,
"compile_id":2,
"is_valid":3,
"not_flag":4,
"virtual_table_name":5,
"clause_index":6
}
},
{
"table_id":11,
"table_name":"COMPILE_2",
"table_type":"compile",
"custom": {
"compile_id":1,
"service_id":2,
"action":3,
"do_blacklist":4,
"do_log":5,
"tags":6,
"user_region":7,
"is_valid":8,
"clause_num":9,
"evaluation_order":10
}
},
{
"table_id":12,
"table_name":"GROUP2COMPILE_2",
"table_type":"group2compile",
"associated_compile_table_id":11,
"custom": {
"group_id":1,
"compile_id":2,
"is_valid":3,
"not_flag":4,
"virtual_table_name":5,
"clause_index":6
}
},
{
"table_id":13,
"table_name":"GROUP2GROUP",
"table_type":"group2group",
"custom": {
"group_id":1,
"superior_group_id":2,
"is_valid":3
}
"table_name":"VIRTUAL_IP_PLUS_TABLE",
"table_type":"virtual",
"physical_table": ["IP_PLUS_CONFIG"]
}
]