gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[OPTIMIZE]replace ipport plugin engine(ip_matcher -> ipport_matcher)

Browse Source
This commit is contained in:
刘文坛
2023-10-10 11:23:44 +00:00
parent 1d106cd4c2
commit 461d43c6b7
21 changed files with 393910 additions and 196717 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
include/maat.h
View File

@@ -56,7 +56,7 @@ enum maat_update_type {
};
enum maat_expr_engine {
MAAT_EXPR_ENGINE_HS = 0, //default engine(hyperscan)
MAAT_EXPR_ENGINE_HS = 0, //hyperscan(default engine)
MAAT_EXPR_ENGINE_RS //rulescan
};
@@ -66,10 +66,10 @@ enum maat_list_type {
};
struct ip_addr {
int ip_type; //4: IPv4, 6: IPv6
int ip_type; //4:IPV4, 6:IPV6
union {
unsigned int ipv4; //network order
unsigned int ipv6[4];
unsigned int ipv6[4];//network order
};
};
@@ -208,6 +208,10 @@ int maat_ip_plugin_table_get_ex_data(struct maat *instance, int table_id,
const struct ip_addr *ip_addr,
void **ex_data_array, size_t array_size);
/**
* NOTE: @retval -1(ERROR) 0(no ex_data) 1(only one ex_data)
* this function return only one ex_data if matched
*/
int maat_ipport_plugin_table_get_ex_data(struct maat *instance, int table_id,
const struct ip_addr *ip_addr, uint16_t port,
void **ex_data_array, size_t array_size);

2
scanner/CMakeLists.txt
View File

@@ -11,5 +11,5 @@ add_library(adapter-static bool_matcher/bool_matcher.cpp expr_matcher/expr_match
expr_matcher/adapter_hs/adapter_hs.cpp expr_matcher/adapter_rs/adapter_rs.cpp
fqdn_engine/fqdn_engine.cpp ip_matcher/ip_matcher.cpp ip_matcher/ipv4_match.cpp
ip_matcher/ipv6_match.cpp flag_matcher/flag_matcher.cpp interval_matcher/cgranges.c
interval_matcher/interval_matcher.cpp)
interval_matcher/interval_matcher.cpp ipport_matcher/ipport_matcher.cpp)
target_link_libraries(adapter-static hyperscan_static hyperscan_runtime_static rulescan_static interval_index_static)

6
scanner/ip_matcher/ip_matcher.h
View File

@@ -84,9 +84,9 @@ struct ip_matcher *ip_matcher_new(struct ip_rule *rules, size_t rule_num,
* @brief scan ip_data to find out if has matched rules in ip_matcher
*
* @param matcher[intput]: ip_matcher which created by ip_matcher_new
* @param data[intput]: ip_data to be scanned
* @param result[input]: result array to store the rule_id and user_tag if there are matching rules
* @param size[input]: result array size
* @param data [intput]: ip_data to be scanned
* @param result [output]: result array to store the rule_id and user_tag of the matching rules
* @param size [input]: result array size
*/
int ip_matcher_match(struct ip_matcher *matcher, struct ip_data *data,
struct scan_result *result, size_t size);

176
scanner/ipport_matcher/ipport_matcher.cpp Normal file
View File

@@ -0,0 +1,176 @@
/*
**********************************************************************************************
* File: ipport_matcher.cpp
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2023-10-09
* Copyright: (c) Since 2023 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#include "uthash/utarray.h"
#include "uthash/uthash.h"
#include "maat_utils.h"
#include "maat_limits.h"
#include "ipport_matcher.h"
struct port_range_entity {
long long rule_id;
void *tag;
struct port_range range;
};
struct ipport_node {
char *key; //key must be ipv4/ipv6 string
size_t key_len;
UT_array *port_range_entities; //array to store <struct port_range_entity>
UT_hash_handle hh;
};
struct ipport_matcher {
struct ipport_node *ipport_hash;
};
UT_icd ut_port_range_entity_icd = {sizeof(struct port_range_entity), NULL, NULL, NULL};
static inline int compare_port_range_entity_for_sort(const void *a, const void *b)
{
struct port_range_entity entity_a = *(const struct port_range_entity *)a;
struct port_range_entity entity_b = *(const struct port_range_entity *)b;
int ret = entity_a.range.min_port - entity_b.range.min_port;
if (0 == ret) {
ret = entity_a.range.max_port - entity_b.range.max_port;
}
return ret;
}
static inline int compare_port_range_entity_for_find(const void *a, const void *b)
{
struct port_range_entity entity_a = *(const struct port_range_entity *)a;
struct port_range_entity entity_b = *(const struct port_range_entity *)b;
int ret = -1;
if (entity_a.range.min_port >= entity_b.range.min_port &&
entity_a.range.min_port <= entity_b.range.max_port) {
ret = 0;
} else if (entity_a.range.max_port < entity_b.range.min_port) {
ret = -1;
} else {
ret = 1;
}
return ret;
}
struct ipport_matcher *ipport_matcher_new(struct ipport_rule *rules, size_t rule_num)
{
if (NULL == rules || 0 == rule_num) {
return NULL;
}
struct ipport_matcher *matcher = ALLOC(struct ipport_matcher, 1);
struct ipport_node *node = NULL;
char *key = NULL;
size_t key_len = 0;
for (size_t i = 0; i < rule_num; i++) {
if (rules[i].ip.ip_type == IPV4) {
key = (char *)&rules[i].ip.ipv4;
key_len = 4;
} else {
key = (char *)&rules[i].ip.ipv6;
key_len = 16;
}
HASH_FIND(hh, matcher->ipport_hash, key, key_len, node);
if (NULL == node) {
node = ALLOC(struct ipport_node, 1);
node->key = ALLOC(char, key_len);
memcpy(node->key, key, key_len);
node->key_len = key_len;
utarray_new(node->port_range_entities, &ut_port_range_entity_icd);
HASH_ADD_KEYPTR(hh, matcher->ipport_hash, node->key, node->key_len, node);
}
struct port_range_entity entity;
entity.range = rules[i].port_range;
entity.rule_id = rules[i].rule_id;
entity.tag = rules[i].user_tag;
utarray_push_back(node->port_range_entities, &entity);
}
struct ipport_node *tmp_node = NULL;
HASH_ITER(hh, matcher->ipport_hash, node, tmp_node) {
utarray_sort(node->port_range_entities, compare_port_range_entity_for_sort);
}
return matcher;
}
int ipport_matcher_match(struct ipport_matcher *matcher, const struct ip_addr *ip_addr,
uint16_t port, struct ipport_result *result_array, size_t array_size)
{
if (NULL == matcher || NULL == ip_addr || NULL == result_array || 0 == array_size) {
return -1;
}
char *key = NULL;
size_t key_len = 0;
if (ip_addr->ip_type == IPV4) {
key = (char *)&ip_addr->ipv4;
key_len = 4;
} else {
key = (char *)ip_addr->ipv6;
key_len = 16;
}
struct ipport_node *node = NULL;
HASH_FIND(hh, matcher->ipport_hash, key, key_len, node);
if (NULL == node) {
return 0;
}
uint16_t host_port = ntohs(port);
struct port_range_entity entity;
entity.range.min_port = host_port;
entity.range.max_port = host_port;
struct port_range_entity *tmp_entity = NULL;
tmp_entity = (struct port_range_entity *)utarray_find(node->port_range_entities,
&entity, compare_port_range_entity_for_find);
if (tmp_entity != NULL) {
result_array[0].rule_id = tmp_entity->rule_id;
result_array[0].tag = tmp_entity->tag;
return 1;
}
return 0;
}
void ipport_matcher_free(struct ipport_matcher *matcher)
{
if (NULL == matcher) {
return;
}
struct ipport_node *node = NULL, *tmp_node = NULL;
HASH_ITER(hh, matcher->ipport_hash, node, tmp_node) {
if (node->key != NULL) {
FREE(node->key);
}
if (node->port_range_entities != NULL) {
utarray_free(node->port_range_entities);
node->port_range_entities = NULL;
}
HASH_DEL(matcher->ipport_hash, node);
FREE(node);
}
FREE(matcher);
}

70
scanner/ipport_matcher/ipport_matcher.h Normal file
View File

@@ -0,0 +1,70 @@
/*
**********************************************************************************************
* File: ipport_matcher.h
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2023-10-09
* Copyright: (c) Since 2023 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#ifndef _IPPORT_MATCHER_H_
#define _IPPORT_MATCHER_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <stddef.h>
#include "maat.h"
struct port_range {
uint16_t min_port; /* host order */
uint16_t max_port; /* host order */
};
struct ipport_rule {
long long rule_id; /* rule id */
void *user_tag; /* point to user-defined data which will return with hit results */
struct ip_addr ip;
struct port_range port_range;
};
struct ipport_result {
long long rule_id; /* matched rule id */
void *tag; /* point to the same address as user_tag in struct ipport_rule which has same rule_id */
};
struct ipport_matcher;
/**
* @brief create an ipport_matcher instance
*
* @param rules [input]: a set of ipport rules
* @param rule_num[input]: the number of ipport rules
*/
struct ipport_matcher *ipport_matcher_new(struct ipport_rule *rules, size_t rule_num);
/**
* @brief scan ip_addr to find out if has matched rules in ipport_matcher
*
* @param matcher [intput]: ipport_matcher which created by ipport_matcher_new
* @param data [intput]: ip_addr to be scanned
* @param result [output]: result array to store the rule_id and user_tag of the matching rules
* @param size [input]: result array size
*/
int ipport_matcher_match(struct ipport_matcher *matcher, const struct ip_addr *ip_addr,
uint16_t port, struct ipport_result *result_array, size_t array_size);
/**
* @brief destroy ipport_matcher instance
*/
void ipport_matcher_free(struct ipport_matcher *matcher);
#ifdef __cplusplus
}
#endif
#endif

1
src/CMakeLists.txt
View File

@@ -26,6 +26,7 @@ include_directories(${PROJECT_SOURCE_DIR}/scanner)
include_directories(${PROJECT_SOURCE_DIR}/scanner/fqdn_engine)
include_directories(${PROJECT_SOURCE_DIR}/scanner/bool_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/ip_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/ipport_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/flag_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/interval_matcher)
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)

1
src/inc_internal/maat_limits.h
View File

@@ -19,6 +19,7 @@ extern "C"
#define MAX_KEYWORDS_STR_LEN 1024
#define MAX_MAAT_STAT_NUM 64
#define MAX_NAME_STR_LEN 64
#define MAX_IP_STR_LEN 64
#define MAX_INSTANCE_NAME_LEN 15
#ifdef __cplusplus

3
src/inc_internal/maat_utils.h
View File

@@ -57,6 +57,9 @@ extern "C"
#define MAX_SCANNER_HIT_GROUP_NUM 4096
#define MAX_SCANNER_HIT_ITEM_NUM 4096
#define IPV4 4
#define IPV6 6
enum ip_format {
IP_FORMAT_SINGLE = 1,
IP_FORMAT_RANGE,

3
src/maat_command.c
View File

@@ -108,8 +108,7 @@ long long maat_read_redis_integer(const redisReply *reply)
return 0;
}
static int redis_flushDB(redisContext *ctx, int db_index,
struct log_handle *logger)
static int redis_flushDB(redisContext *ctx, int db_index, struct log_handle *logger)
{
long long maat_redis_version = 0;

26
src/maat_ip_plugin.c
View File

@@ -205,7 +205,6 @@ ip_plugin_rule_new(struct ip_plugin_schema *schema, const char *table_name,
size_t column_offset = 0;
size_t column_len = 0;
char addr_format[16] = {0};
char start_ip_str[40] = {0};
char end_ip_str[40] = {0};
struct ip_rule *ip_plugin_rule = ALLOC(struct ip_rule, 1);
@@ -234,27 +233,6 @@ ip_plugin_rule_new(struct ip_plugin_schema *schema, const char *table_name,
goto error;
}
//TODO: to be added again,
#if 0
ret = get_column_pos(line, schema->addr_format_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table(table_id:%d) line:%s has no addr_format column",
__FUNCTION__, __LINE__, schema->table_id, line);
goto error;
}
memcpy(addr_format, (line + column_offset), column_len);
if (IP_FORMAT_UNKNOWN == ip_format_str2int(addr_format)) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table(table_id:%d) line:%s has invalid addr_format, should be range/CIDR",
__FUNCTION__, __LINE__, schema->table_id, line);
goto error;
}
#endif
const char *tmp_str = "range";
memcpy(addr_format, tmp_str, strlen(tmp_str));
ret = get_column_pos(line, schema->start_ip_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
@@ -274,7 +252,7 @@ ip_plugin_rule_new(struct ip_plugin_schema *schema, const char *table_name,
strncpy(end_ip_str, line + column_offset, column_len);
if (IPv4 == ip_plugin_rule->type) {
ret = ip_format2range(ip_plugin_rule->type, ip_format_str2int(addr_format),
ret = ip_format2range(ip_plugin_rule->type, IP_FORMAT_RANGE,
start_ip_str, end_ip_str,
&ip_plugin_rule->ipv4_rule.start_ip,
&ip_plugin_rule->ipv4_rule.end_ip);
@@ -286,7 +264,7 @@ ip_plugin_rule_new(struct ip_plugin_schema *schema, const char *table_name,
}
} else {
//ipv6
ret = ip_format2range(ip_plugin_rule->type, ip_format_str2int(addr_format),
ret = ip_format2range(ip_plugin_rule->type, IP_FORMAT_RANGE,
start_ip_str, end_ip_str,
ip_plugin_rule->ipv6_rule.start_ip,
ip_plugin_rule->ipv6_rule.end_ip);

111
src/maat_ipport_plugin.c
View File

@@ -13,8 +13,9 @@
#include "alignment.h"
#include "log/log.h"
#include "maat_utils.h"
#include "uthash/utarray.h"
#include "maat_ipport_plugin.h"
#include "ip_matcher.h"
#include "ipport_matcher.h"
#include "interval_matcher.h"
#include "maat_rule.h"
#include "maat_garbage_collection.h"
@@ -56,7 +57,7 @@ struct ipport_item {
};
struct ipport_plugin_runtime {
struct ip_matcher *ip_matcher;
struct ipport_matcher *matcher;
struct rcu_hash_table *item_hash; //<item_id, struct ipport_item>
struct ex_data_runtime *ex_data_rt;
size_t n_worker_thread;
@@ -233,9 +234,9 @@ void ipport_plugin_runtime_free(void *ipport_plugin_runtime)
}
struct ipport_plugin_runtime *ipport_plugin_rt = (struct ipport_plugin_runtime *)ipport_plugin_runtime;
if (ipport_plugin_rt->ip_matcher != NULL) {
ip_matcher_free(ipport_plugin_rt->ip_matcher);
ipport_plugin_rt->ip_matcher = NULL;
if (ipport_plugin_rt->matcher != NULL) {
ipport_matcher_free(ipport_plugin_rt->matcher);
ipport_plugin_rt->matcher = NULL;
}
if (ipport_plugin_rt->ex_data_rt != NULL) {
@@ -275,7 +276,7 @@ ipport_item_new(struct ipport_plugin_schema *schema, const char *table_name,
}
ipport_item->ip_type = atoi(line + column_offset);
if (ipport_item->ip_type != IPv4 && ipport_item->ip_type != IPv6) {
if (ipport_item->ip_type != IPV4 && ipport_item->ip_type != IPV6) {
log_error(logger, MODULE_IPPORT_PLUGIN,
"[%s:%d] ipport table:<%s> has invalid ip type:%d in line:%s",
__FUNCTION__, __LINE__, table_name, ipport_item->ip_type, line);
@@ -292,27 +293,28 @@ ipport_item_new(struct ipport_plugin_schema *schema, const char *table_name,
}
memcpy(ip_str, (line + column_offset), column_len);
if (IPv4 == ipport_item->ip_type) {
ret = ip_format2range(ipport_item->ip_type, IP_FORMAT_RANGE,
ip_str, ip_str, &ipport_item->ipv4.min_ip,
&ipport_item->ipv4.max_ip);
if (IPV4 == ipport_item->ip_type) {
uint32_t ipv4_addr = 0;
ret = inet_pton(AF_INET, ip_str, &ipv4_addr);
if (ret < 0) {
log_error(logger, MODULE_IPPORT_PLUGIN,
"[%s:%d] ipport table:<%s> ip_format2range(ip4) failed in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
ipport_item->ipv4.min_ip = ipv4_addr;
ipport_item->ipv4.max_ip = ipv4_addr;
} else {
//ipv6
ret = ip_format2range(ipport_item->ip_type, IP_FORMAT_RANGE,
ip_str, ip_str, ipport_item->ipv6.min_ip,
ipport_item->ipv6.max_ip);
uint32_t ipv6_addr[4] = {0};
ret = inet_pton(AF_INET6, ip_str, ipv6_addr);
if (ret < 0) {
log_error(logger, MODULE_IPPORT_PLUGIN,
"[%s:%d] ipport table:<%s> ip_format2range(ip6) failed in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
memcpy(ipport_item->ipv6.min_ip, ipv6_addr, sizeof(ipv6_addr));
}
ret = get_column_pos(line, schema->port1_column, &column_offset,
@@ -437,22 +439,28 @@ int ipport_plugin_runtime_update(void *ipport_plugin_runtime, void *ipport_plugi
return 0;
}
static void ipport_item_to_ip_rule(struct ipport_item *item, struct ip_rule *rule)
static void ipport_item_to_ipport_rule(struct ipport_item *item, struct ipport_rule *rule)
{
if (IPv4 == item->ip_type) {
rule->type = IPv4;
rule->ipv4_rule.start_ip = item->ipv4.min_ip;
rule->ipv4_rule.end_ip = item->ipv4.max_ip;
if (IPV4 == item->ip_type) {
rule->ip.ip_type= IPV4;
rule->ip.ipv4 = item->ipv4.min_ip;
rule->port_range.min_port = item->min_port;
rule->port_range.max_port = item->max_port;
} else {
rule->type = IPv6;
memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_ip,
sizeof(item->ipv6.min_ip));
memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_ip,
sizeof(item->ipv6.max_ip));
rule->ip.ip_type = IPV6;
memcpy(rule->ip.ipv6, item->ipv6.min_ip, sizeof(item->ipv6.min_ip));
rule->port_range.min_port = item->min_port;
rule->port_range.max_port = item->max_port;
}
rule->rule_id = item->item_id;
}
static void garbage_ipport_matcher_free(void *ipport_matcher, void *arg)
{
struct ipport_matcher *matcher = (struct ipport_matcher *)ipport_matcher;
ipport_matcher_free(matcher);
}
int ipport_plugin_runtime_commit(void *ipport_plugin_runtime, const char *table_name,
long long maat_rt_version)
{
@@ -471,44 +479,48 @@ int ipport_plugin_runtime_commit(void *ipport_plugin_runtime, const char *table_
return 0;
}
struct ip_rule *rules = NULL;
struct ipport_rule *rules = NULL;
struct ex_container **ex_container = NULL;
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
if (rule_cnt > 0) {
rules = ALLOC(struct ip_rule, rule_cnt);
rules = ALLOC(struct ipport_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
struct ipport_item *item = (struct ipport_item *)ex_container[i]->custom_data;
ipport_item_to_ip_rule(item, &rules[i]);
ipport_item_to_ipport_rule(item, &rules[i]);
rules[i].user_tag = ex_container[i];
}
}
int ret = 0;
size_t mem_used = 0;
struct ip_matcher *new_ip_matcher = NULL;
struct ip_matcher *old_ip_matcher = NULL;
struct ipport_matcher *new_matcher = NULL;
struct ipport_matcher *old_matcher = NULL;
if (rule_cnt > 0) {
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
new_matcher = ipport_matcher_new(rules, rule_cnt);
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
if (NULL == new_matcher) {
log_error(ipport_plugin_rt->logger, MODULE_IPPORT_PLUGIN,
"[%s:%d] ipport_plugin table[%s] rebuild ip_matcher failed when "
"[%s:%d] ipport_plugin table[%s] rebuild ipport_matcher failed when "
"update %zu rules", __FUNCTION__, __LINE__, table_name, rule_cnt);
ret = -1;
} else {
log_info(ipport_plugin_rt->logger, MODULE_IPPORT_PLUGIN,
"table[%s] commit %zu ipport_plugin rules and rebuild ip_matcher "
"completed, version:%lld", table_name, rule_cnt, maat_rt_version);
"table[%s] commit %zu ipport_plugin rules and rebuild ipport_matcher "
"completed, version:%lld, consume:%lldms", table_name, rule_cnt,
maat_rt_version, time_elapse_ms);
}
}
old_ip_matcher = ipport_plugin_rt->ip_matcher;
ipport_plugin_rt->ip_matcher = new_ip_matcher;
old_matcher = ipport_plugin_rt->matcher;
ipport_plugin_rt->matcher = new_matcher;
ex_data_runtime_commit(ex_data_rt);
if (old_ip_matcher != NULL) {
maat_garbage_bagging(ipport_plugin_rt->ref_garbage_bin, old_ip_matcher, NULL,
garbage_ip_matcher_free);
if (old_matcher != NULL) {
maat_garbage_bagging(ipport_plugin_rt->ref_garbage_bin, old_matcher, NULL,
garbage_ipport_matcher_free);
}
ipport_plugin_rt->rule_num = rule_cnt;
@@ -573,26 +585,19 @@ int ipport_plugin_runtime_get_ex_data(void *ipport_plugin_runtime, const struct
return 0;
}
if (NULL == ipport_plugin_rt->ip_matcher) {
if (NULL == ipport_plugin_rt->matcher) {
return 0;
}
struct ip_data ip_data = *(const struct ip_data *)ip_addr;
if (ip_data.type == IPv4) {
ip_data.ipv4 = ntohl(ip_data.ipv4);
} else {
ipv6_ntoh(ip_data.ipv6);
}
struct scan_result ip_results[n_ex_data];
int n_hit_ip_item = ip_matcher_match(ipport_plugin_rt->ip_matcher, &ip_data, ip_results, n_ex_data);
if (n_hit_ip_item <= 0) {
return n_hit_ip_item;
struct ipport_result results[n_ex_data];
int n_hit_item = ipport_matcher_match(ipport_plugin_rt->matcher, ip_addr, port, results, n_ex_data);
if (n_hit_item <= 0) {
return n_hit_item;
}
size_t hit_result_cnt = 0;
for (size_t i = 0; i < n_hit_ip_item; i++) {
struct ex_container *ex_container = ip_results[i].tag;
for (size_t i = 0; i < n_hit_item; i++) {
struct ex_container *ex_container = results[i].tag;
struct ipport_item *item = (struct ipport_item *)ex_container->custom_data;
int ret = validate_port(item, port);

4
src/maat_utils.c
View File

@@ -481,12 +481,12 @@ int ip_format2range(int ip_type, enum ip_format format, const char *ip1, const c
int cidr = 0;
int ret = 0;
if (ip_type != 4 && ip_type != 6) {
if (ip_type != IPV4 && ip_type != IPV6) {
assert(0);
return -1;
}
if (ip_type == 4) {
if (ip_type == IPV4) {
uint32_t ipv4_addr = 0;
ret = inet_pton(AF_INET, ip1, &ipv4_addr);
if (ret <= 0) {

4
test/CMakeLists.txt
View File

@@ -7,6 +7,7 @@ include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_hs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_rs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/ip_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/ipport_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/bool_matcher)
add_executable(rcu_hash_gtest rcu_hash_gtest.cpp)
@@ -27,6 +28,9 @@ target_link_libraries(expr_matcher_gtest maat_frame_static gtest_static)
add_executable(ip_matcher_gtest ip_matcher_gtest.cpp)
target_link_libraries(ip_matcher_gtest maat_frame_static gtest_static)
add_executable(ipport_matcher_gtest ipport_matcher_gtest.cpp)
target_link_libraries(ipport_matcher_gtest maat_frame_static gtest_static)
add_executable(bool_matcher_gtest bool_matcher_gtest.cpp)
target_link_libraries(bool_matcher_gtest maat_frame_static gtest_static)

235
test/ipport_matcher_gtest.cpp Normal file
View File

@@ -0,0 +1,235 @@
#include <gtest/gtest.h>
#include "log/log.h"
#include "ipport_matcher.h"
#include "maat_utils.h"
#include "cJSON/cJSON.h"
#define MAX_ARRAY_SIZE 6
struct log_handle *g_logger = NULL;
/*
rule port range [100, 150]
match input port = 110, which is in port range
match input port = 100, which is on port range boundary
*/
TEST(ipv4port_matcher, MatchedOneRuleInPortRange) {
const char *ip1_str = "192.168.0.1";
struct ipport_rule rules[MAX_ARRAY_SIZE];
rules[0].rule_id = 100;
rules[0].ip.ip_type = IPV4;
rules[0].port_range.min_port = 100;
rules[0].port_range.max_port = 150;
inet_pton(AF_INET, ip1_str, &rules[0].ip.ipv4);
struct ipport_matcher *matcher = ipport_matcher_new(rules, 1);
ASSERT_TRUE(matcher != NULL);
struct ip_addr ip;
ip.ip_type = IPV4;
inet_pton(AF_INET, ip1_str, &ip.ipv4);
uint16_t port = htons(120);
struct ipport_result results[MAX_ARRAY_SIZE];
int ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 100);
ipport_matcher_free(matcher);
matcher = NULL;
}
TEST(ipv4port_matcher, MatchedOneRuleOnPortRangeBoundary) {
const char *ip1_str = "192.168.0.1";
struct ipport_rule rules[MAX_ARRAY_SIZE];
rules[0].rule_id = 100;
rules[0].ip.ip_type = IPV4;
rules[0].port_range.min_port = 100;
rules[0].port_range.max_port = 150;
inet_pton(AF_INET, ip1_str, &rules[0].ip.ipv4);
struct ipport_matcher *matcher = ipport_matcher_new(rules, 1);
ASSERT_TRUE(matcher != NULL);
struct ip_addr ip;
ip.ip_type = IPV4;
inet_pton(AF_INET, ip1_str, &ip.ipv4);
uint16_t port = htons(100);
struct ipport_result results[MAX_ARRAY_SIZE];
int ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 100);
memset(results, 0, sizeof(results));
port = htons(150);
ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 100);
ipport_matcher_free(matcher);
matcher = NULL;
}
TEST(ipv4port_matcher, MatchedMultiRuleInPortRange) {
const char *ip1_str = "192.168.0.1";
struct ipport_rule rules[MAX_ARRAY_SIZE];
memset(rules, 0, sizeof(rules));
rules[0].rule_id = 100;
rules[0].ip.ip_type = IPV4;
rules[0].port_range.min_port = 100;
rules[0].port_range.max_port = 150;
inet_pton(AF_INET, ip1_str, &rules[0].ip.ipv4);
rules[1].rule_id = 200;
rules[1].ip.ip_type = IPV4;
rules[1].port_range.min_port = 110;
rules[1].port_range.max_port = 160;
inet_pton(AF_INET, ip1_str, &rules[1].ip.ipv4);
rules[2].rule_id = 300;
rules[2].ip.ip_type = IPV4;
rules[2].port_range.min_port = 120;
rules[2].port_range.max_port = 170;
inet_pton(AF_INET, ip1_str, &rules[2].ip.ipv4);
rules[3].rule_id = 400;
rules[3].ip.ip_type = IPV4;
rules[3].port_range.min_port = 130;
rules[3].port_range.max_port = 180;
inet_pton(AF_INET, ip1_str, &rules[3].ip.ipv4);
rules[4].rule_id = 500;
rules[4].ip.ip_type = IPV4;
rules[4].port_range.min_port = 140;
rules[4].port_range.max_port = 190;
inet_pton(AF_INET, ip1_str, &rules[4].ip.ipv4);
rules[5].rule_id = 600;
rules[5].ip.ip_type = IPV4;
rules[5].port_range.min_port = 150;
rules[5].port_range.max_port = 200;
inet_pton(AF_INET, ip1_str, &rules[5].ip.ipv4);
struct ipport_matcher *matcher = ipport_matcher_new(rules, MAX_ARRAY_SIZE);
ASSERT_TRUE(matcher != NULL);
struct ip_addr ip;
ip.ip_type = IPV4;
inet_pton(AF_INET, ip1_str, &ip.ipv4);
uint16_t port = htons(90);
//no match rule_id
struct ipport_result results[MAX_ARRAY_SIZE];
int ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 0);
//match rule_id:100
memset(results, 0, sizeof(results));
port = htons(100);
ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 100);
/* sorted port range array: [[100,150], [110,160], [120,170], [130,180], [140,190], [150,200]]
rule_id: 100, 200, 300, 400, 500, 600
match rule_id:100, 200, but return only one rule_id:200(reference binary search logic)
*/
memset(results, 0, sizeof(results));
port = htons(110);
ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 200);
// match rule_id:100, 200, 300, but return only one rule_id:200(reference binary search logic)
memset(results, 0, sizeof(results));
port = htons(120);
ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 200);
// match rule_id:100, 200, 300, 400, but return only one rule_id:400(reference binary search logic)
memset(results, 0, sizeof(results));
port = htons(130);
ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 400);
// match rule_id:100, 200, 300, 400, 500, but return only one rule_id:400(reference binary search logic)
memset(results, 0, sizeof(results));
port = htons(140);
ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 400);
// match rule_id:100, 200, 300, 400, 500, 600, but return only one rule_id:400(reference binary search logic)
memset(results, 0, sizeof(results));
port = htons(150);
ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 400);
// match rule_id:200, 300, 400, 500, 600, but return only one rule_id:400(reference binary search logic)
memset(results, 0, sizeof(results));
port = htons(160);
ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 400);
// match rule_id:300, 400, 500, 600, but return only one rule_id:400(reference binary search logic)
memset(results, 0, sizeof(results));
port = htons(170);
ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 400);
// match rule_id:400, 500, 600, but return only one rule_id:400(reference binary search logic)
memset(results, 0, sizeof(results));
port = htons(180);
ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 400);
// match rule_id:500, 600, but return only one rule_id:600(reference binary search logic)
memset(results, 0, sizeof(results));
port = htons(190);
ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 600);
// match rule_id:600, but return only one rule_id:600
memset(results, 0, sizeof(results));
port = htons(200);
ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0].rule_id, 600);
// no match rule_id
memset(results, 0, sizeof(results));
port = htons(210);
ret = ipport_matcher_match(matcher, &ip, port, results, MAX_ARRAY_SIZE);
EXPECT_EQ(ret, 0);
ipport_matcher_free(matcher);
matcher = NULL;
}
int main(int argc, char **argv)
{
int ret = 0;
::testing::InitGoogleTest(&argc, argv);
g_logger = log_handle_create("./ipport_matcher_gtest.log", 0);
ret = RUN_ALL_TESTS();
log_handle_destroy(g_logger);
return ret;
}

113
test/ipport_plugin/ipport_plugin_gtest.cpp
View File

@@ -15,7 +15,7 @@
const char *table_info_path = "./ipport_plugin_table_info.conf";
const char *log_file = "./ipport_plugin_gtest.log";
const char *g_ip_str = "192.0.2.0";
const char *g_ip_str = "192.0.1.1";
class IPPortPluginTable : public testing::Test
{
@@ -89,6 +89,37 @@ int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
reply = NULL;
}
reply = maat_wrap_redis_command(c, "SET MAAT_VERSION 1");
if (NULL == reply) {
return -1;
} else {
freeReplyObject(reply);
reply = NULL;
}
reply = maat_wrap_redis_command(c, "SET MAAT_PRE_VER 1");
if (NULL == reply) {
return -1;
} else {
freeReplyObject(reply);
reply = NULL;
}
reply = maat_wrap_redis_command(c, "SET SEQUENCE_REGION 1");
if (NULL == reply) {
return -1;
} else {
freeReplyObject(reply);
reply = NULL;
}
reply = maat_wrap_redis_command(c, "SET SEQUENCE_GROUP 1");
if (NULL == reply) {
return -1;
} else {
freeReplyObject(reply);
reply = NULL;
}
size_t total_line_cnt = 0;
char iris_idx_path[PATH_MAX] = {0};
@@ -192,11 +223,11 @@ void *ipport_plugin_scan_thread(void *arg)
int table_id = maat_get_table_id(maat_inst, table_name);
struct ip_addr ipv4;
ipv4.ip_type = 4;
ipv4.ip_type = IPV4;
int ret = inet_pton(AF_INET, g_ip_str, &ipv4.ipv4);
EXPECT_EQ(ret, 1);
uint16_t port = htons(200);
uint16_t port = htons(10);
int hit_times = 0;
struct ipport_plugin_ud *results[ARRAY_SIZE];
@@ -206,7 +237,6 @@ void *ipport_plugin_scan_thread(void *arg)
(void **)results, ARRAY_SIZE);
EXPECT_EQ(ret, 1);
if (ret == 1) {
EXPECT_EQ(results[0]->rule_id, 513);
hit_times++;
}
}
@@ -270,7 +300,7 @@ void *ipport_plugin_update_thread(void *arg)
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name;
const int CMD_EXPR_NUM = 1024;
const int CMD_EXPR_NUM = 64;
long long item_id = 9000000;
for (int i = 0; i < CMD_EXPR_NUM; i++) {
@@ -292,7 +322,7 @@ void *ipport_plugin_update_thread(void *arg)
return is_all_hit;
}
TEST_F(IPPortPluginTable, basic) {
TEST_F(IPPortPluginTable, WITHOUT_SAME_IP) {
char redis_ip[32] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
@@ -312,7 +342,7 @@ TEST_F(IPPortPluginTable, basic) {
maat_options_free(opts);
int ex_data_counter = 0;
const char *table_name = "IPPORT_PLUGIN_WITH_EXDATA";
const char *table_name = "IPPORT_PLUGIN_WITHOUT_SAME_IP";
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
@@ -323,7 +353,6 @@ TEST_F(IPPortPluginTable, basic) {
ipport_plugin_ex_dup_cb,
0, &ex_data_counter);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ex_data_counter, 49999);
int i = 0;
pthread_t threads[2];
@@ -355,7 +384,73 @@ TEST_F(IPPortPluginTable, basic) {
scan_per_second = PERF_SCAN_COUNT * 1000 / time_elapse_ms;
log_info(maat_inst->logger, MODULE_IPPORT_PLUGIN_GTEST,
"IpportPluginScan match rate speed %lld lookups/s/thread",
"IpportPluginScan without same ip match rate speed %lld lookups/s/thread",
scan_per_second);
}
TEST_F(IPPortPluginTable, WITH_256SAME_IP) {
char redis_ip[32] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
struct log_handle *logger = IPPortPluginTable::logger;
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
assert(ret == 0);
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, log_file, LOG_LEVEL_INFO);
maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_rule_update_checking_interval_ms(opts, 100);
struct maat *maat_inst = maat_new(opts, table_info_path);
assert(maat_inst);
maat_options_free(opts);
int ex_data_counter = 0;
const char *table_name = "IPPORT_PLUGIN_WITH_256SAME_IP";
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
ret = maat_plugin_table_ex_schema_register(maat_inst, table_name,
ipport_plugin_ex_new_cb,
ipport_plugin_ex_free_cb,
ipport_plugin_ex_dup_cb,
0, &ex_data_counter);
EXPECT_EQ(ret, 0);
int i = 0;
pthread_t threads[2];
struct thread_param thread_params[2];
for (i = 0; i < 2; i++) {
thread_params[i].maat_inst = maat_inst;
thread_params[i].thread_id = i;
thread_params[i].table_name = table_name;
thread_params[i].test_count = PERF_SCAN_COUNT;
thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger;
}
pthread_create(&threads[0], NULL, ipport_plugin_scan_thread, thread_params);
pthread_create(&threads[1], NULL, ipport_plugin_update_thread, thread_params + 1);
int *is_all_hit = NULL;
long long time_elapse_ms = 0;
long long scan_per_second = 0;
for (i = 0; i < 2; i++) {
pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += thread_params[i].time_elapse_ms;
EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit);
}
scan_per_second = PERF_SCAN_COUNT * 1000 / time_elapse_ms;
log_info(maat_inst->logger, MODULE_IPPORT_PLUGIN_GTEST,
"IpportPluginScan with 256 same ip match rate speed %lld lookups/s/thread",
scan_per_second);
}

15
test/ipport_plugin/ipport_plugin_table_info.conf
View File

@@ -1,7 +1,20 @@
[
{
"table_id":1,
"table_name":"IPPORT_PLUGIN_WITH_EXDATA",
"table_name":"IPPORT_PLUGIN_WITH_256SAME_IP",
"table_type":"ipport_plugin",
"valid_column":6,
"custom": {
"item_id":1,
"ip_type":2,
"ip_addr":3,
"port1":4,
"port2":5
}
},
{
"table_id":2,
"table_name":"IPPORT_PLUGIN_WITHOUT_SAME_IP",
"table_type":"ipport_plugin",
"valid_column":6,
"custom": {

196609
test/ipport_plugin/test_data/IPPORT_PLUGIN_WITHOUT_SAME_IP.local Normal file
View File

File diff suppressed because it is too large Load Diff

393218
test/ipport_plugin/test_data/IPPORT_PLUGIN_WITH_EXDATA.local → test/ipport_plugin/test_data/IPPORT_PLUGIN_WITH_256SAME_IP.local
View File

File diff suppressed because it is too large Load Diff

3
test/ipport_plugin/test_data/index/full_config_index.0000000001
View File

@@ -1 +1,2 @@
IPPORT_PLUGIN_WITH_EXDATA 50000 test_data/IPPORT_PLUGIN_WITH_EXDATA.local
IPPORT_PLUGIN_WITH_256SAME_IP 196608 test_data/IPPORT_PLUGIN_WITH_256SAME_IP.local
IPPORT_PLUGIN_WITHOUT_SAME_IP 196608 test_data/IPPORT_PLUGIN_WITHOUT_SAME_IP.local

13
test/maat_framework_gtest.cpp
View File

@@ -262,9 +262,9 @@ int ip_table_set_line(struct maat *maat_inst, const char *table_name, enum maat_
return 0;
}
int ip_type = 4;
int ip_type = IPV4;
if (type == IPv6) {
ip_type = 6;
ip_type = IPV6;
}
sprintf(table_line, "%lld\t%lld\t%d\t%s\t%s\t%s\t%s\t%u\t%u\t6\t%d", item_id, group_id, ip_type,
@@ -3775,7 +3775,7 @@ TEST_F(IPPortPluginTable, EX_DATA) {
EXPECT_EQ(ex_data_counter, 4);
struct ip_addr ipv4;
ipv4.ip_type = IPv4;
ipv4.ip_type = IPV4;
ret = inet_pton(AF_INET, "192.168.100.1", &ipv4.ipv4);
EXPECT_EQ(ret, 1);
@@ -3784,9 +3784,8 @@ TEST_F(IPPortPluginTable, EX_DATA) {
struct ipport_plugin_ud *results[ARRAY_SIZE];
ret = maat_ipport_plugin_table_get_ex_data(maat_inst, table_id, &ipv4, port,
(void **)results, ARRAY_SIZE);
EXPECT_EQ(ret, 2);
EXPECT_EQ(results[0]->rule_id, 101);
EXPECT_EQ(results[1]->rule_id, 103);
EXPECT_EQ(ret, 1);
EXPECT_EQ(results[0]->rule_id, 103);
struct ip_addr ipv6;
ipv6.ip_type = IPv6;
@@ -6082,7 +6081,7 @@ TEST_F(MaatCmdTest, UpdateIPPlugin) {
struct ip_addr ipv4, ipv6;
struct ip_plugin_ud *results[ARRAY_SIZE];
ipv4.ip_type = 4;
ipv4.ip_type = IPV4;
inet_pton(AF_INET, "192.168.30.100", &(ipv4.ipv4));
memset(results, 0, sizeof(results));

4
test/maat_framework_perf_gtest.cpp
View File

@@ -238,9 +238,9 @@ static int ip_table_set_line(struct maat *maat_inst, const char *table_name, enu
return 0;
}
int ip_type = 4;
int ip_type = IPV4;
if (type == IPv6) {
ip_type = 6;
ip_type = IPV6;
}
sprintf(table_line, "%lld\t%lld\t%d\t%s\t%s\t%s\t%s\t%u\t%u\t6\t%d",