From 00b2d2815d855a09d84179cb1b9d337adf4d01de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=96=87=E5=9D=9B?= Date: Wed, 27 Sep 2023 07:15:29 +0000 Subject: [PATCH] [FEATURE]support Ipport plugin table => TSG-17217 --- include/maat.h | 12 +- src/CMakeLists.txt | 2 +- src/inc_internal/maat_ipport_plugin.h | 66 +++ src/inc_internal/maat_table.h | 1 + src/inc_internal/maat_virtual.h | 14 - src/maat_api.c | 94 +++- src/maat_ex_data.c | 2 +- src/maat_ipport_plugin.c | 611 ++++++++++++++++++++++++++ src/maat_rule.c | 5 + src/maat_stat.c | 3 +- src/maat_table.c | 13 + src/version.map | 1 + test/maat_framework_gtest.cpp | 139 ++++++ test/maat_json.json | 8 + test/table_info.conf | 14 + 15 files changed, 946 insertions(+), 39 deletions(-) create mode 100644 src/inc_internal/maat_ipport_plugin.h create mode 100644 src/maat_ipport_plugin.c diff --git a/include/maat.h b/include/maat.h index 689af33..7ac6d24 100644 --- a/include/maat.h +++ b/include/maat.h @@ -205,16 +205,20 @@ void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id, const char *key, size_t key_len); int maat_ip_plugin_table_get_ex_data(struct maat *instance, int table_id, - const struct ip_addr *ip, void **ex_data_array, - size_t n_ex_data); + const struct ip_addr *ip_addr, + void **ex_data_array, size_t array_size); + +int maat_ipport_plugin_table_get_ex_data(struct maat *instance, int table_id, + const struct ip_addr *ip_addr, uint16_t port, + void **ex_data_array, size_t array_size); int maat_fqdn_plugin_table_get_ex_data(struct maat *instance, int table_id, const char *fqdn, void **ex_data_array, - size_t n_ex_data); + size_t array_size); int maat_bool_plugin_table_get_ex_data(struct maat *instance, int table_id, unsigned long long *item_ids, size_t n_item, - void **ex_data_array, size_t n_ex_data); + void **ex_data_array, size_t array_size); /* maat scan API */ struct maat_state; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4e21418..8cc1dd0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,7 +14,7 @@ add_definitions(-fPIC) set(MAAT_SRC alignment.c json2iris.c maat_api.c rcu_hash.c maat_garbage_collection.c maat_config_monitor.c maat_rule.c maat_kv.c maat_ex_data.c maat_utils.c maat_command.c maat_redis_monitor.c maat_table.c maat_compile.c maat_group.c maat_ip.c maat_flag.c maat_interval.c maat_expr.c maat_plugin.c - maat_ip_plugin.c maat_bool_plugin.c maat_fqdn_plugin.c maat_virtual.c maat_stat.c) + maat_ip_plugin.c maat_ipport_plugin.c maat_bool_plugin.c maat_fqdn_plugin.c maat_virtual.c maat_stat.c) set(LIB_SOURCE_FILES ${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c ${PROJECT_SOURCE_DIR}/deps/log/log.c) diff --git a/src/inc_internal/maat_ipport_plugin.h b/src/inc_internal/maat_ipport_plugin.h new file mode 100644 index 0000000..c38f9fc --- /dev/null +++ b/src/inc_internal/maat_ipport_plugin.h @@ -0,0 +1,66 @@ +/* +********************************************************************************************** +* File: maat_ipport_plugin.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_IPPORT_PLUGIN_H_ +#define _MAAT_IPPORT_PLUGIN_H_ + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "maat.h" +#include "cJSON/cJSON.h" +#include "maat_table.h" +#include "maat_ex_data.h" + +struct ipport_plugin_runtime; + +/* ipport plugin schema API */ +void *ipport_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); +void ipport_plugin_schema_free(void *ipport_plugin_schema); + +/* ipport plugin table ex data API */ +int ipport_plugin_table_set_ex_container_schema(void *ipport_plugin_schema, int table_id, + maat_ex_new_func_t *new_func, + maat_ex_free_func_t *free_func, + maat_ex_dup_func_t *dup_func, + void (*custom_data_free)(void *), + long argl, void *argp); +struct ex_container_schema * +ipport_plugin_table_get_ex_container_schema(void *ipport_plugin_schema); + +/* ipport plugin runtime API */ +void *ipport_plugin_runtime_new(void *ipport_plugin_schema, size_t max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void ipport_plugin_runtime_free(void *ipport_plugin_runtime); + +int ipport_plugin_runtime_update(void *ipport_plugin_runtime, void *ipport_plugin_schema, + const char *table_name, const char *line, int valid_column); + +int ipport_plugin_runtime_commit(void *ipport_plugin_runtime, const char *table_name, + long long maat_rt_version); + +long long ipport_plugin_runtime_rule_count(void *ipport_plugin_runtime); + +struct ex_data_runtime *ipport_plugin_runtime_get_ex_data_rt(void *ipport_plugin_runtime); + +int ipport_plugin_runtime_get_ex_data(void *ipport_plugin_runtime, const struct ip_addr *ip_addr, + uint16_t port, void **ex_data_array, size_t n_ex_data_array); + +long long ipport_plugin_runtime_update_err_count(void *ipport_plugin_runtime); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_table.h b/src/inc_internal/maat_table.h index 977ab98..768df6f 100644 --- a/src/inc_internal/maat_table.h +++ b/src/inc_internal/maat_table.h @@ -34,6 +34,7 @@ enum table_type { TABLE_TYPE_INTERVAL_PLUS, TABLE_TYPE_PLUGIN, TABLE_TYPE_IP_PLUGIN, + TABLE_TYPE_IPPORT_PLUGIN, TABLE_TYPE_FQDN_PLUGIN, TABLE_TYPE_BOOL_PLUGIN, //above are physical table diff --git a/src/inc_internal/maat_virtual.h b/src/inc_internal/maat_virtual.h index 191cc9d..e4a0104 100644 --- a/src/inc_internal/maat_virtual.h +++ b/src/inc_internal/maat_virtual.h @@ -19,20 +19,6 @@ extern "C" #include "cJSON/cJSON.h" #include "maat_table.h" -enum scan_type { - SCAN_TYPE_INVALID = -1, - SCAN_TYPE_NONE = 0, - SCAN_TYPE_PLUGIN, - SCAN_TYPE_IP_PLUGIN, - SCAN_TYPE_FQDN_PLUGIN, - SCAN_TYPE_BOOL_PLUGIN, - SCAN_TYPE_IP, - SCAN_TYPE_INTERVAL, - SCAN_TYPE_STRING, - SCAN_TYPE_FLAG, - SCAN_TYPE_MAX -}; - void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger); diff --git a/src/maat_api.c b/src/maat_api.c index d4962d2..21f2317 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -36,6 +36,7 @@ #include "maat_ip.h" #include "maat_plugin.h" #include "maat_ip_plugin.h" +#include "maat_ipport_plugin.h" #include "maat_fqdn_plugin.h" #include "maat_bool_plugin.h" #include "maat_virtual.h" @@ -598,6 +599,11 @@ static int generic_plugin_table_set_ex_schema(struct table_manager *tbl_mgr, int new_func, free_func, dup_func, free, argl, argp); break; + case TABLE_TYPE_IPPORT_PLUGIN: + ret = ipport_plugin_table_set_ex_container_schema(schema, table_id, + new_func, free_func, dup_func, + free, argl, argp); + break; case TABLE_TYPE_FQDN_PLUGIN: ret = fqdn_plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func, dup_func, @@ -661,6 +667,27 @@ static void ip_plugin_runtime_commit_ex_schema(void *runtime, void *schema, ip_plugin_runtime_commit(runtime, table_name, 0); } +static void ipport_plugin_runtime_commit_ex_schema(void *runtime, void *schema, + const char *table_name, + int valid_column) +{ + struct ex_container_schema *container_schema = NULL; + struct ex_data_runtime *ex_data_rt = NULL; + + container_schema = ipport_plugin_table_get_ex_container_schema(schema); + ex_data_rt = ipport_plugin_runtime_get_ex_data_rt(runtime); + ex_data_runtime_set_ex_container_schema(ex_data_rt, container_schema); + + size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt); + for (size_t i = 0; i < n_cached_row; i++) { + const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i); + ipport_plugin_runtime_update(runtime, schema, table_name, row, valid_column); + } + + ex_data_runtime_clear_row_cache(ex_data_rt); + ipport_plugin_runtime_commit(runtime, table_name, 0); +} + static void fqdn_plugin_runtime_commit_ex_schema(void *runtime, void *schema, const char *table_name, int valid_column) @@ -723,6 +750,9 @@ static int generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, case TABLE_TYPE_IP_PLUGIN: ip_plugin_runtime_commit_ex_schema(runtime, schema, table_name, valid_column); break; + case TABLE_TYPE_IPPORT_PLUGIN: + ipport_plugin_runtime_commit_ex_schema(runtime, schema, table_name, valid_column); + break; case TABLE_TYPE_FQDN_PLUGIN: fqdn_plugin_runtime_commit_ex_schema(runtime, schema, table_name, valid_column); break; @@ -854,10 +884,10 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, int maat_ip_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, const struct ip_addr *ip_addr, - void **ex_data_array, size_t n_ex_data) + void **ex_data_array, size_t array_size) { if (NULL == maat_inst || table_id < 0 || table_id >= MAX_TABLE_NUM - || NULL == ip_addr || NULL == ex_data_array || 0 == n_ex_data) { + || NULL == ip_addr || NULL == ex_data_array || 0 == array_size) { return -1; } @@ -871,21 +901,49 @@ int maat_ip_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, return -1; } - int n_hit_ex_data = ip_plugin_runtime_get_ex_data(ip_plugin_rt, ip_addr, - ex_data_array, n_ex_data); - if (n_hit_ex_data < 0) { + int n_ex_data = ip_plugin_runtime_get_ex_data(ip_plugin_rt, ip_addr, + ex_data_array, array_size); + if (n_ex_data < 0) { return -1; } - return n_hit_ex_data; + return n_ex_data; +} + +int maat_ipport_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, + const struct ip_addr *ip_addr, uint16_t port, + void **ex_data_array, size_t array_size) +{ + if (NULL == maat_inst || table_id < 0 || table_id >= MAX_TABLE_NUM + || NULL == ip_addr || NULL == ex_data_array || 0 == array_size) { + return -1; + } + + struct maat_runtime *maat_rt = maat_inst->maat_rt; + if (NULL == maat_rt) { + return -1; + } + + void *ipport_plugin_rt = table_manager_get_runtime(maat_rt->ref_tbl_mgr, table_id); + if (NULL == ipport_plugin_rt) { + return -1; + } + + int n_ex_data = ipport_plugin_runtime_get_ex_data(ipport_plugin_rt, ip_addr, port, + ex_data_array, array_size); + if (n_ex_data < 0) { + return -1; + } + + return n_ex_data; } int maat_fqdn_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, const char *fqdn, void **ex_data_array, - size_t n_ex_data) + size_t array_size) { if (NULL == maat_inst || table_id < 0 || table_id >= MAX_TABLE_NUM - || NULL == fqdn || NULL == ex_data_array || 0 == n_ex_data) { + || NULL == fqdn || NULL == ex_data_array || 0 == array_size) { return -1; } @@ -899,21 +957,21 @@ int maat_fqdn_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, return -1; } - int n_hit_ex_data = fqdn_plugin_runtime_get_ex_data(fqdn_plugin_rt, fqdn, - ex_data_array, n_ex_data); - if (n_hit_ex_data < 0) { + int n_ex_data = fqdn_plugin_runtime_get_ex_data(fqdn_plugin_rt, fqdn, + ex_data_array, array_size); + if (n_ex_data < 0) { return -1; } - return n_hit_ex_data; + return n_ex_data; } int maat_bool_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, unsigned long long *item_ids, size_t n_item, - void **ex_data_array, size_t n_ex_data) + void **ex_data_array, size_t array_size) { if (NULL == maat_inst || table_id < 0 || table_id >= MAX_TABLE_NUM - || NULL == item_ids || NULL == ex_data_array || 0 == n_ex_data) { + || NULL == item_ids || NULL == ex_data_array || 0 == array_size) { return -1; } @@ -927,13 +985,13 @@ int maat_bool_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, return -1; } - int n_hit_ex_data = bool_plugin_runtime_get_ex_data(bool_plugin_rt, item_ids, n_item, - ex_data_array, n_ex_data); - if (n_hit_ex_data < 0) { + int n_ex_data = bool_plugin_runtime_get_ex_data(bool_plugin_rt, item_ids, n_item, + ex_data_array, array_size); + if (n_ex_data < 0) { return -1; } - return n_hit_ex_data; + return n_ex_data; } static inline int scan_status_should_compile_NOT(struct maat_state *state) diff --git a/src/maat_ex_data.c b/src/maat_ex_data.c index d7cbfcc..da9be96 100644 --- a/src/maat_ex_data.c +++ b/src/maat_ex_data.c @@ -264,7 +264,7 @@ void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_r void *dup_ex_data = NULL; container_schema->ex_schema.dup_func(ex_data_rt->table_id, &dup_ex_data, - &(ex_container->ex_data), + &(ex_container->ex_data), container_schema->ex_schema.argl, container_schema->ex_schema.argp); return dup_ex_data; diff --git a/src/maat_ipport_plugin.c b/src/maat_ipport_plugin.c new file mode 100644 index 0000000..89575c8 --- /dev/null +++ b/src/maat_ipport_plugin.c @@ -0,0 +1,611 @@ +/* +********************************************************************************************** +* File: maat_ipport_plugin.c +* Description: +* Authors: Liu wentan +* Date: 2022-10-31 +* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved. +*********************************************************************************************** +*/ + +#include + +#include "alignment.h" +#include "log/log.h" +#include "maat_utils.h" +#include "maat_ipport_plugin.h" +#include "ip_matcher.h" +#include "interval_matcher.h" +#include "maat_rule.h" +#include "maat_garbage_collection.h" + +#define MODULE_IPPORT_PLUGIN module_name_str("maat.ipport_plugin") + +struct ipport_plugin_schema { + int item_id_column; + int ip_type_column; + int ip_addr_column; + int port1_column; + int port2_column; + int gc_timeout_s; + int table_id; + struct ex_container_schema container_schema; + struct table_manager *ref_tbl_mgr; + struct log_handle *logger; +}; + +struct ipv4_item { + uint32_t min_ip; + uint32_t max_ip; +}; + +struct ipv6_item { + uint32_t min_ip[4]; + uint32_t max_ip[4]; +}; + +struct ipport_item { + long long item_id; + int ip_type; //IPV4 or IPV6 + union { + struct ipv4_item ipv4; + struct ipv6_item ipv6; + }; + uint16_t min_port; + uint16_t max_port; +}; + +struct ipport_plugin_runtime { + struct ip_matcher *ip_matcher; + struct interval_matcher *intval_matcher; + struct rcu_hash_table *item_hash; // + + long long rule_num; + long long update_err_cnt; + struct ex_data_runtime *ex_data_rt; + size_t n_worker_thread; + struct maat_garbage_bin *ref_garbage_bin; + struct log_handle *logger; +}; + +void *ipport_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) +{ + struct ipport_plugin_schema *schema = ALLOC(struct ipport_plugin_schema, 1); + schema->logger = logger; + + cJSON *custom_item = NULL; + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (item != NULL && item->type == cJSON_Number) { + schema->table_id = item->valueint; + } else { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport_plugin table:<%s> schema has no table_id column", + __FUNCTION__, __LINE__, table_name); + goto error; + } + + item = cJSON_GetObjectItem(json, "custom"); + if (NULL == item || item->type != cJSON_Object) { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport_plugin table:<%s> schema has no custom column", + __FUNCTION__, __LINE__, table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "item_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->item_id_column = custom_item->valueint; + } else { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport_plugin table:<%s> schema has no item_id column", + __FUNCTION__, __LINE__, table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "ip_type"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->ip_type_column = custom_item->valueint; + } else { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport_plugin table:<%s> schema has no ip_type column", + __FUNCTION__, __LINE__, table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "ip_addr"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->ip_addr_column = custom_item->valueint; + } else { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport_plugin table:<%s> schema has no ip_addr column", + __FUNCTION__, __LINE__, table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "port1"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->port1_column = custom_item->valueint; + } else { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport_plugin table:<%s> schema has no port1 column", + __FUNCTION__, __LINE__, table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "port2"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->port2_column = custom_item->valueint; + } else { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport_plugin table:<%s> schema has no port2 column", + __FUNCTION__, __LINE__, table_name); + goto error; + } + + //gc_timeout_s is optional + custom_item = cJSON_GetObjectItem(item, "gc_timeout_s"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->gc_timeout_s = custom_item->valueint; + } + + schema->ref_tbl_mgr = tbl_mgr; + + return schema; +error: + FREE(schema); + return NULL; +} + +void ipport_plugin_schema_free(void *ipport_plugin_schema) +{ + if (NULL == ipport_plugin_schema) { + return; + } + + FREE(ipport_plugin_schema); +} + +int ipport_plugin_table_set_ex_container_schema(void *ipport_plugin_schema, int table_id, + maat_ex_new_func_t *new_func, + maat_ex_free_func_t *free_func, + maat_ex_dup_func_t *dup_func, + void (*custom_data_free)(void *), + long argl, void *argp) +{ + struct ipport_plugin_schema *schema = (struct ipport_plugin_schema *)ipport_plugin_schema; + + if (1 == schema->container_schema.set_flag) { + log_error(schema->logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport_plugin table(table_id:%d) ex_container_schema has been set" + ", can't set again", __FUNCTION__, __LINE__, table_id); + return -1; + } + + schema->container_schema.table_id = table_id; + schema->container_schema.custom_data_free = custom_data_free; + schema->container_schema.ex_schema.new_func = new_func; + schema->container_schema.ex_schema.free_func = free_func; + schema->container_schema.ex_schema.dup_func = dup_func; + schema->container_schema.ex_schema.argl = argl; + schema->container_schema.ex_schema.argp = argp; + schema->container_schema.set_flag = 1; + + return 0; +} + +struct ex_container_schema * +ipport_plugin_table_get_ex_container_schema(void *ipport_plugin_schema) +{ + struct ipport_plugin_schema *schema = (struct ipport_plugin_schema *)ipport_plugin_schema; + + return &(schema->container_schema); +} + +void *ipport_plugin_runtime_new(void *ipport_plugin_schema, size_t max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + if (NULL == ipport_plugin_schema) { + return NULL; + } + + struct ipport_plugin_schema *schema = (struct ipport_plugin_schema *)ipport_plugin_schema; + struct ipport_plugin_runtime *ipport_plugin_rt = ALLOC(struct ipport_plugin_runtime, 1); + + ipport_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, schema->gc_timeout_s, + logger); + if (1 == schema->container_schema.set_flag) { + ex_data_runtime_set_ex_container_schema(ipport_plugin_rt->ex_data_rt, + &(schema->container_schema)); + } + + ipport_plugin_rt->n_worker_thread = max_thread_num; + ipport_plugin_rt->ref_garbage_bin = garbage_bin; + ipport_plugin_rt->logger = logger; + + return ipport_plugin_rt; +} + +void ipport_plugin_runtime_free(void *ipport_plugin_runtime) +{ + if (NULL == ipport_plugin_runtime) { + return; + } + + struct ipport_plugin_runtime *ipport_plugin_rt = (struct ipport_plugin_runtime *)ipport_plugin_runtime; + if (ipport_plugin_rt->ip_matcher != NULL) { + ip_matcher_free(ipport_plugin_rt->ip_matcher); + ipport_plugin_rt->ip_matcher = NULL; + } + + if (ipport_plugin_rt->intval_matcher != NULL) { + interval_matcher_free(ipport_plugin_rt->intval_matcher); + ipport_plugin_rt->intval_matcher = NULL; + } + + if (ipport_plugin_rt->ex_data_rt != NULL) { + ex_data_runtime_free(ipport_plugin_rt->ex_data_rt); + ipport_plugin_rt->ex_data_rt = NULL; + } + + FREE(ipport_plugin_rt); +} + +static struct ipport_item * +ipport_item_new(struct ipport_plugin_schema *schema, const char *table_name, + const char *line, struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + char ip_str[40] = {0}; + struct ipport_item *ipport_item = ALLOC(struct ipport_item, 1); + + int ret = get_column_pos(line, schema->item_id_column, &column_offset, + &column_len); + if (ret < 0) { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport table:<%s> has no item_id in line:%s", + __FUNCTION__, __LINE__, table_name, line); + goto error; + } + ipport_item->item_id = atoll(line + column_offset); + + ret = get_column_pos(line, schema->ip_type_column, &column_offset, + &column_len); + if (ret < 0) { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport table:<%s> has no ip_type in line:%s", + __FUNCTION__, __LINE__, table_name, line); + goto error; + } + ipport_item->ip_type = atoi(line + column_offset); + + if (ipport_item->ip_type != IPv4 && ipport_item->ip_type != IPv6) { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport table:<%s> has invalid ip type:%d in line:%s", + __FUNCTION__, __LINE__, table_name, ipport_item->ip_type, line); + goto error; + } + + ret = get_column_pos(line, schema->ip_addr_column, &column_offset, + &column_len); + if (ret < 0) { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport table:<%s> has no ip_addr in line:%s", + __FUNCTION__, __LINE__, table_name, line); + goto error; + } + memcpy(ip_str, (line + column_offset), column_len); + + if (IPv4 == ipport_item->ip_type) { + ret = ip_format2range(ipport_item->ip_type, IP_FORMAT_RANGE, + ip_str, ip_str, &ipport_item->ipv4.min_ip, + &ipport_item->ipv4.max_ip); + if (ret < 0) { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport table:<%s> ip_format2range(ip4) failed in line:%s", + __FUNCTION__, __LINE__, table_name, line); + goto error; + } + } else { + //ipv6 + ret = ip_format2range(ipport_item->ip_type, IP_FORMAT_RANGE, + ip_str, ip_str, ipport_item->ipv6.min_ip, + ipport_item->ipv6.max_ip); + if (ret < 0) { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport table:<%s> ip_format2range(ip6) failed in line:%s", + __FUNCTION__, __LINE__, table_name, line); + goto error; + } + } + + ret = get_column_pos(line, schema->port1_column, &column_offset, + &column_len); + if (ret < 0) { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport table:<%s>) has no port1 in line:%s", + __FUNCTION__, __LINE__, table_name, line); + goto error; + } + ipport_item->min_port = atoi(line + column_offset); + + ret = get_column_pos(line, schema->port2_column, &column_offset, + &column_len); + if (ret < 0) { + log_error(logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport table:<%s> has no port2 in line:%s", + __FUNCTION__, __LINE__, table_name, line); + goto error; + } + ipport_item->max_port = atoi(line + column_offset); + + return ipport_item; +error: + FREE(ipport_item); + return NULL; +} + +static void ipport_item_free(struct ipport_item *item) +{ + if (NULL == item) { + return; + } + + FREE(item); +} + +static int ipport_plugin_runtime_update_row(struct ipport_plugin_runtime *ipport_plugin_rt, + const char *table_name, const char *row, + const char *key, size_t key_len, + struct ipport_item *ipport_item, int is_valid) +{ + int ret = -1; + struct ex_data_runtime *ex_data_rt = ipport_plugin_rt->ex_data_rt; + if (NULL == ex_data_rt) { + return -1; + } + + if (0 == is_valid) { + // delete + ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len); + if (ret < 0) { + return -1; + } + } else { + // add + void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, table_name, row, key, key_len); + struct ex_container *ex_container = ex_container_new(ex_data, (void *)ipport_item); + ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container); + if (ret < 0) { + return -1; + } + } + + return 0; +} + +int ipport_plugin_runtime_update(void *ipport_plugin_runtime, void *ipport_plugin_schema, + const char *table_name, const char *line, int valid_column) +{ + if (NULL == ipport_plugin_runtime || NULL == ipport_plugin_schema || + NULL == line) { + return -1; + } + + struct ipport_item *ipport_item = NULL; + struct ipport_plugin_schema *schema = (struct ipport_plugin_schema *)ipport_plugin_schema; + struct ipport_plugin_runtime *ipport_plugin_rt = (struct ipport_plugin_runtime *)ipport_plugin_runtime; + size_t item_id_offset = 0, item_id_len = 0; + + int is_valid = get_column_value(line, valid_column); + if (is_valid < 0) { + ipport_plugin_rt->update_err_cnt++; + return -1; + } + + int ret = get_column_pos(line, schema->item_id_column, &item_id_offset, + &item_id_len); + if (ret < 0) { + ipport_plugin_rt->update_err_cnt++; + return -1; + } + + if (1 == schema->container_schema.set_flag) { + if (1 == is_valid) { + // add + ipport_item = ipport_item_new(schema, table_name, line, + ipport_plugin_rt->logger); + if (NULL == ipport_item) { + ipport_plugin_rt->update_err_cnt++; + return -1; + } + } + + const char *key = line + item_id_offset; + size_t key_len = item_id_len; + ret = ipport_plugin_runtime_update_row(ipport_plugin_rt, table_name, line, + key, key_len, ipport_item, is_valid); + if (ret < 0) { + if (ipport_item != NULL) { + ipport_item_free(ipport_item); + } + ipport_plugin_rt->update_err_cnt++; + return -1; + } + } else { + //ex_schema not set + ex_data_runtime_cache_row_put(ipport_plugin_rt->ex_data_rt, line); + ipport_plugin_rt->rule_num = ex_data_runtime_cached_row_count(ipport_plugin_rt->ex_data_rt); + } + + return 0; +} + +static void ipport_item_to_ip_rule(struct ipport_item *item, struct ip_rule *rule) +{ + if (IPv4 == item->ip_type) { + rule->type = IPv4; + rule->ipv4_rule.start_ip = item->ipv4.min_ip; + rule->ipv4_rule.end_ip = item->ipv4.max_ip; + } else { + rule->type = IPv6; + memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_ip, + sizeof(item->ipv6.min_ip)); + memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_ip, + sizeof(item->ipv6.max_ip)); + } + rule->rule_id = item->item_id; +} + +int ipport_plugin_runtime_commit(void *ipport_plugin_runtime, const char *table_name, + long long maat_rt_version) +{ + if (NULL == ipport_plugin_runtime) { + return -1; + } + + struct ipport_plugin_runtime *ipport_plugin_rt = (struct ipport_plugin_runtime *)ipport_plugin_runtime; + struct ex_data_runtime *ex_data_rt = ipport_plugin_rt->ex_data_rt; + if (NULL == ex_data_rt) { + return -1; + } + + int updating_flag = ex_data_runtime_is_updating(ex_data_rt); + if (0 == updating_flag) { + return 0; + } + + ex_data_runtime_commit(ex_data_rt); + + struct ip_rule *rules = NULL; + struct ex_container **ex_container = NULL; + size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container); + if (rule_cnt > 0) { + rules = ALLOC(struct ip_rule, rule_cnt); + for (size_t i = 0; i < rule_cnt; i++) { + struct ipport_item *item = (struct ipport_item *)ex_container[i]->custom_data; + ipport_item_to_ip_rule(item, &rules[i]); + rules[i].user_tag = ex_container[i]; + } + } + + int ret = 0; + size_t mem_used = 0; + struct ip_matcher *new_ip_matcher = NULL; + struct ip_matcher *old_ip_matcher = NULL; + + if (rule_cnt > 0) { + new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); + if (NULL == new_ip_matcher) { + log_error(ipport_plugin_rt->logger, MODULE_IPPORT_PLUGIN, + "[%s:%d] ipport_plugin table[%s] rebuild ip_matcher failed when " + "update %zu rules", __FUNCTION__, __LINE__, table_name, rule_cnt); + ret = -1; + } else { + log_info(ipport_plugin_rt->logger, MODULE_IPPORT_PLUGIN, + "table[%s] commit %zu ipport_plugin rules and rebuild ip_matcher " + "completed, version:%lld", table_name, rule_cnt, maat_rt_version); + } + } + + old_ip_matcher = ipport_plugin_rt->ip_matcher; + ipport_plugin_rt->ip_matcher = new_ip_matcher; + if (old_ip_matcher != NULL) { + maat_garbage_bagging(ipport_plugin_rt->ref_garbage_bin, old_ip_matcher, NULL, + garbage_ip_matcher_free); + } + + ipport_plugin_rt->rule_num = rule_cnt; + + if (rules != NULL) { + FREE(rules); + } + + if (ex_container != NULL) { + FREE(ex_container); + } + + return ret; +} + +long long ipport_plugin_runtime_rule_count(void *ipport_plugin_runtime) +{ + return 0; +} + +struct ex_data_runtime *ipport_plugin_runtime_get_ex_data_rt(void *ipport_plugin_runtime) +{ + if (NULL == ipport_plugin_runtime) { + return NULL; + } + + struct ipport_plugin_runtime *ipport_plugin_rt = (struct ipport_plugin_runtime *)ipport_plugin_runtime; + + return ipport_plugin_rt->ex_data_rt; +} + +static int validate_port(struct ipport_item *item, uint16_t port) +{ + uint16_t host_port = ntohs(port); + + if (item->min_port > host_port || item->max_port < host_port) { + return -1; + } + + return 0; +} + +int ipport_plugin_runtime_get_ex_data(void *ipport_plugin_runtime, const struct ip_addr *ip_addr, + uint16_t port, void **ex_data_array, size_t array_size) +{ + if (NULL == ipport_plugin_runtime || NULL == ip_addr || + NULL == ex_data_array || 0 == array_size) { + return -1; + } + + struct ipport_plugin_runtime *ipport_plugin_rt = (struct ipport_plugin_runtime *)ipport_plugin_runtime; + if (0 == ipport_plugin_rt->rule_num) { + return 0; + } + + if (NULL == ipport_plugin_rt->ip_matcher) { + return 0; + } + + struct ip_data ip_data = *(const struct ip_data *)ip_addr; + if (ip_data.type == IPv4) { + ip_data.ipv4 = ntohl(ip_data.ipv4); + } else { + ipv6_ntoh(ip_data.ipv6); + } + + struct scan_result ip_results[MAX_SCANNER_HIT_ITEM_NUM]; + int n_hit_ip_item = ip_matcher_match(ipport_plugin_rt->ip_matcher, &ip_data, ip_results, + MAX_SCANNER_HIT_ITEM_NUM); + if (n_hit_ip_item <= 0) { + return n_hit_ip_item; + } + + size_t hit_result_cnt = 0; + for (size_t i = 0; i < n_hit_ip_item; i++) { + struct ex_container *ex_container = ip_results[i].tag; + struct ipport_item *item = (struct ipport_item *)ex_container->custom_data; + + int ret = validate_port(item, port); + if (ret < 0) { + continue; + } + + ex_data_array[hit_result_cnt++] = ex_data_runtime_get_ex_data_by_container(ipport_plugin_rt->ex_data_rt, + ex_container); + } + + return hit_result_cnt; +} + +long long ipport_plugin_runtime_update_err_count(void *ipport_plugin_runtime) +{ + return 0; +} \ No newline at end of file diff --git a/src/maat_rule.c b/src/maat_rule.c index 6223952..67f04ca 100644 --- a/src/maat_rule.c +++ b/src/maat_rule.c @@ -26,6 +26,7 @@ #include "maat_compile.h" #include "maat_plugin.h" #include "maat_ip_plugin.h" +#include "maat_ipport_plugin.h" #include "maat_fqdn_plugin.h" #include "maat_bool_plugin.h" #include "maat_stat.h" @@ -213,6 +214,10 @@ static void maat_plugin_table_garbage_collect_routine(struct table_manager *tbl_ runtime = table_manager_get_runtime(tbl_mgr, i); ex_data_rt = ip_plugin_runtime_get_ex_data_rt(runtime); break; + case TABLE_TYPE_IPPORT_PLUGIN: + runtime = table_manager_get_runtime(tbl_mgr, i); + ex_data_rt = ipport_plugin_runtime_get_ex_data_rt(runtime); + break; case TABLE_TYPE_FQDN_PLUGIN: runtime = table_manager_get_runtime(tbl_mgr, i); ex_data_rt = fqdn_plugin_runtime_get_ex_data_rt(runtime); diff --git a/src/maat_stat.c b/src/maat_stat.c index 8c9e65a..19b3b9f 100644 --- a/src/maat_stat.c +++ b/src/maat_stat.c @@ -380,7 +380,8 @@ static void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on) total_rule_num += rule_num; if (table_type == TABLE_TYPE_PLUGIN || table_type == TABLE_TYPE_IP_PLUGIN || - table_type == TABLE_TYPE_BOOL_PLUGIN || table_type == TABLE_TYPE_FQDN_PLUGIN) { + table_type == TABLE_TYPE_IPPORT_PLUGIN || table_type == TABLE_TYPE_BOOL_PLUGIN || + table_type == TABLE_TYPE_FQDN_PLUGIN) { continue; } diff --git a/src/maat_table.c b/src/maat_table.c index 4955a02..e08ee43 100644 --- a/src/maat_table.c +++ b/src/maat_table.c @@ -23,6 +23,7 @@ #include "maat_flag.h" #include "maat_plugin.h" #include "maat_ip_plugin.h" +#include "maat_ipport_plugin.h" #include "maat_bool_plugin.h" #include "maat_fqdn_plugin.h" #include "maat_interval.h" @@ -207,6 +208,17 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .rule_count = ip_plugin_runtime_rule_count, .update_err_count = ip_plugin_runtime_update_err_count }, + { + .type = TABLE_TYPE_IPPORT_PLUGIN, + .new_schema = ipport_plugin_schema_new, + .free_schema = ipport_plugin_schema_free, + .new_runtime = ipport_plugin_runtime_new, + .free_runtime = ipport_plugin_runtime_free, + .update_runtime = ipport_plugin_runtime_update, + .commit_runtime = ipport_plugin_runtime_commit, + .rule_count = ipport_plugin_runtime_rule_count, + .update_err_count = ipport_plugin_runtime_update_err_count + }, { .type = TABLE_TYPE_FQDN_PLUGIN, .new_schema = fqdn_plugin_schema_new, @@ -472,6 +484,7 @@ static void register_reserved_word(struct maat_kv_store *reserved_word_map) maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); + maat_kv_register(reserved_word_map, "ipport_plugin", TABLE_TYPE_IPPORT_PLUGIN); maat_kv_register(reserved_word_map, "bool_plugin", TABLE_TYPE_BOOL_PLUGIN); maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN); maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); diff --git a/src/version.map b/src/version.map index 06f0034..984f5b1 100644 --- a/src/version.map +++ b/src/version.map @@ -10,6 +10,7 @@ global: maat_compile_table*; maat_plugin_table*; maat_ip_plugin_table*; + maat_ipport_plugin_table*; maat_fqdn_plugin_table*; maat_bool_plugin_table*; maat_scan*; diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 4eb779d..affc274 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -3739,6 +3739,145 @@ TEST_F(IPPluginTable, EX_DATA) { EXPECT_EQ(ret, 0); } +class IPPortPluginTable : public testing::Test +{ +protected: + static void SetUpTestCase() { + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," + "{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + logger = log_handle_create("./maat_framework_gtest.log", 0); + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger); + if (ret < 0) { + log_error(logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_stat_file(opts, "./stat.log"); + maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_inst = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_inst) { + log_error(logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in IPPortPluginTable failed.", + __FUNCTION__, __LINE__); + } + } + + static void TearDownTestCase() { + maat_free(_shared_maat_inst); + log_handle_destroy(logger); + } + + static struct log_handle *logger; + static struct maat *_shared_maat_inst; +}; + +struct maat *IPPortPluginTable::_shared_maat_inst; +struct log_handle *IPPortPluginTable::logger; + +struct ipport_plugin_ud { + long long rule_id; + char *buffer; + size_t buf_len; +}; +void ipport_plugin_ex_new_cb(const char *table_name, int table_id, const char *key, + const char *table_line, void **ad, long argl, void *argp) +{ + int *counter = (int *)argp; + size_t column_offset=0, column_len=0; + struct ipport_plugin_ud *ud = ALLOC(struct ipport_plugin_ud, 1); + + int ret = get_column_pos(table_line, 1, &column_offset, &column_len); + EXPECT_EQ(ret, 0); + + ud->rule_id = atoll(table_line + column_offset); + + ret = get_column_pos(table_line, 5, &column_offset, &column_len); + EXPECT_EQ(ret, 0); + + ud->buffer = ALLOC(char, column_len + 1); + strncpy(ud->buffer, table_line + column_offset, column_len); + + ud->buf_len = column_len + 1; + *ad = ud; + (*counter)++; +} + +void ipport_plugin_ex_free_cb(int table_id, void **ad, long argl, void *argp) +{ + struct ipport_plugin_ud *ud = (struct ipport_plugin_ud *)(*ad); + + ud->rule_id = 0; + memset(ud->buffer, 0, ud->buf_len); + ud->buf_len = 0; + + free(ud->buffer); + free(ud); + *ad = NULL; +} + +void ipport_plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void *argp) +{ + struct ipport_plugin_ud *ud = (struct ipport_plugin_ud *)(*from); + + *to = ud; +} + +TEST_F(IPPortPluginTable, EX_DATA) { + int ex_data_counter = 0; + const char *table_name = "TEST_IPPORT_PLUGIN_WITH_EXDATA"; + struct maat *maat_inst = IPPortPluginTable::_shared_maat_inst; + + int table_id = maat_get_table_id(maat_inst, table_name); + ASSERT_GT(table_id, 0); + + int ret = maat_plugin_table_ex_schema_register(maat_inst, table_name, + ip_plugin_ex_new_cb, + ip_plugin_ex_free_cb, + ip_plugin_ex_dup_cb, + 0, &ex_data_counter); + EXPECT_EQ(ret, 0); + EXPECT_EQ(ex_data_counter, 3); + + struct ip_addr ipv4; + ipv4.ip_type = IPv4; + ret = inet_pton(AF_INET, "192.168.100.1", &ipv4.ipv4); + EXPECT_EQ(ret, 1); + + uint16_t port = htons(150); + + struct ipport_plugin_ud *results[ARRAY_SIZE]; + ret = maat_ipport_plugin_table_get_ex_data(maat_inst, table_id, &ipv4, port, + (void **)results, ARRAY_SIZE); + EXPECT_EQ(ret, 1); + EXPECT_EQ(results[0]->rule_id, 101); + + // struct ip_addr ipv6; + // ipv6.ip_type = IPv6; + // inet_pton(AF_INET6, "2001:db8:1234::5210", &(ipv6.ipv6)); + // memset(results, 0, sizeof(results)); + + // ret = maat_ip_plugin_table_get_ex_data(maat_inst, table_id, &ipv6, + // (void**)results, ARRAY_SIZE); + // EXPECT_EQ(ret, 2); + // EXPECT_EQ(results[0]->rule_id, 104); + // EXPECT_EQ(results[1]->rule_id, 103); + + // //Reproduce BugReport-Liumengyan-20210515 + // inet_pton(AF_INET6, "240e:97c:4010:104::17", &(ipv6.ipv6)); + // ret = maat_ip_plugin_table_get_ex_data(maat_inst, table_id, &ipv6, + // (void**)results, ARRAY_SIZE); + // EXPECT_EQ(ret, 0); +} + class FQDNPluginTable : public testing::Test { protected: diff --git a/test/maat_json.json b/test/maat_json.json index d4c65ba..06bd618 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -3029,6 +3029,14 @@ "105\t6\t2620:100:3000::\t2620:0100:30ff:ffff:ffff:ffff:ffff:ffff\tBugreport-liumengyan-20210517\t1\trange" ] }, + { + "table_name": "TEST_IPPORT_PLUGIN_WITH_EXDATA", + "table_content": [ + "101\t4\t192.168.100.1\t100\t200\t1", + "102\t4\t192.168.100.2\t100\t200\t1", + "103\t4\t192.168.100.1\t201\t300\t1" + ] + }, { "table_name": "TEST_FQDN_PLUGIN_WITH_EXDATA", "table_content": [ diff --git a/test/table_info.conf b/test/table_info.conf index 1dcffe7..2265e23 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -565,5 +565,19 @@ "flag":3, "flag_mask":4 } + }, + { + "table_id":47, + "table_name":"TEST_IPPORT_PLUGIN_WITH_EXDATA", + "table_type":"ipport_plugin", + "valid_column":6, + "custom": { + "gc_timeout_s": 3, + "item_id":1, + "ip_type":2, + "ip_addr":3, + "port1":4, + "port2":5 + } } ] \ No newline at end of file