add bool_plugin & fqdn_plugin unit-test

This commit is contained in:
liuwentan
2023-02-24 17:29:38 +08:00
parent 6f4b88d690
commit d4e1670987
29 changed files with 654 additions and 159 deletions

View File

@@ -165,8 +165,9 @@ int maat_scan_string(struct maat *instance, int table_id, int thread_id,
struct maat_stream;
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id);
int maat_scan_stream(struct maat_stream **stream, const char* data, int data_len,
long long *results, size_t *n_result, struct maat_state **state);
int maat_scan_stream(struct maat_stream **stream, const char *data, int data_len,
long long *results, size_t n_result, size_t *n_hit_result,
struct maat_state **state);
void maat_scan_stream_close(struct maat_stream **stream);

View File

@@ -14,7 +14,7 @@ add_definitions(-fPIC)
set(MAAT_SRC alignment.c json2iris.c maat_api.c rcu_hash.c maat_garbage_collection.c maat_config_monitor.c
maat_rule.c maat_kv.c maat_ex_data.c maat_utils.c maat_command.c maat_redis_monitor.c
maat_table.c maat_compile.c maat_group.c maat_ip.c maat_flag.c maat_interval.c
maat_expr.c maat_fqdn.c maat_port_proto.c maat_plugin.c maat_ip_plugin.c maat_bool_plugin.c
maat_expr.c maat_fqdn.c maat_port.c maat_plugin.c maat_ip_plugin.c maat_bool_plugin.c
maat_fqdn_plugin.c maat_virtual.c)
set(LIB_SOURCE_FILES

View File

@@ -17,12 +17,10 @@ extern "C"
#endif
#include "maat.h"
#include "maat_table.h"
#include "cJSON/cJSON.h"
struct bool_plugin_runtime;
struct table_manager;
struct log_handle;
struct maat_garbage_bin;
/* bool plugin schema API */
void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,

View File

@@ -17,12 +17,10 @@ extern "C"
#endif
#include "maat_rule.h"
#include "maat_table.h"
#include "cJSON/cJSON.h"
struct expr_runtime;
struct table_manager;
struct log_handle;
struct maat_garbage_bin;
void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);

View File

@@ -20,11 +20,9 @@ extern "C"
#include "maat_rule.h"
#include "cJSON/cJSON.h"
#include "maat_table.h"
struct flag_runtime;
struct table_manager;
struct log_handle;
struct maat_garbage_bin;
void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);

View File

@@ -18,11 +18,9 @@ extern "C"
#include "maat_rule.h"
#include "cJSON/cJSON.h"
#include "maat_table.h"
struct fqdn_runtime;
struct table_manager;
struct log_handle;
struct maat_garbage_bin;
void *fqdn_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);

View File

@@ -17,12 +17,10 @@ extern "C"
#endif
#include "maat.h"
#include "maat_table.h"
#include "cJSON/cJSON.h"
struct fqdn_plugin_runtime;
struct table_manager;
struct log_handle;
struct maat_garbage_bin;
/* fqdn plugin schema API */
void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,

View File

@@ -20,11 +20,9 @@ extern "C"
#include "cJSON/cJSON.h"
#include "maat_kv.h"
#include "maat_table.h"
struct maat_group;
struct table_manager;
struct log_handle;
struct maat_garbage_bin;
/* group2group schema API */
void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr,

View File

@@ -18,13 +18,10 @@ extern "C"
#include <stdint.h>
#include "maat_rule.h"
#include "cJSON/cJSON.h"
#include "maat_table.h"
struct interval_runtime;
struct table_manager;
struct log_handle;
struct maat_garbage_bin;
void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);

View File

@@ -17,11 +17,9 @@ extern "C"
#endif
#include "cJSON/cJSON.h"
#include "maat_table.h"
struct ip_runtime;
struct table_manager;
struct log_handle;
struct maat_garbage_bin;
void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);

View File

@@ -18,11 +18,9 @@ extern "C"
#include "maat.h"
#include "cJSON/cJSON.h"
#include "maat_table.h"
struct ip_plugin_runtime;
struct table_manager;
struct log_handle;
struct maat_garbage_bin;
/* ip plugin schema API */
void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,

View File

@@ -0,0 +1,54 @@
/*
**********************************************************************************************
* File: maat_port.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_PORT_H_
#define _MAAT_PORT_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include "cJSON/cJSON.h"
#include "maat_table.h"
#include "maat_rule.h"
/* port is short for port proto */
struct port_runtime;
void *port_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void port_schema_free(void *port_schema);
/* ip runtime API */
void *port_runtime_new(void *port_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void port_runtime_free(void *port_runtime);
int port_runtime_update(void *port_runtime, void *port_schema,
const char *line, int valid_column);
int port_runtime_commit(void *port_runtime, const char *table_name);
struct ex_data_runtime *port_runtime_get_ex_data_rt(struct port_runtime *port_rt);
/* ip runtime scan API */
int port_runtime_scan(struct port_runtime *port_rt, int thread_id, int port,
int proto, int *group_ids, size_t group_id_size,
int vtable_id, struct maat_state *state);
void port_runtime_scan_hit_inc(struct port_runtime *port_rt, int thread_id);
long long port_runtime_scan_hit_sum(struct port_runtime *port_rt, int n_thread);
#ifdef __cplusplus
}
#endif
#endif

View File

@@ -1,56 +0,0 @@
/*
**********************************************************************************************
* File: maat_port_proto.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_PORT_PROTO_H_
#define _MAAT_PORT_PROTO_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include "maat_rule.h"
#include "cJSON/cJSON.h"
/* pp is short for port proto */
struct pp_runtime;
struct table_manager;
struct log_handle;
struct maat_garbage_bin;
void *pp_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void pp_schema_free(void *pp_schema);
/* ip runtime API */
void *pp_runtime_new(void *pp_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void pp_runtime_free(void *pp_runtime);
int pp_runtime_update(void *pp_runtime, void *pp_schema,
const char *line, int valid_column);
int pp_runtime_commit(void *pp_runtime, const char *table_name);
struct ex_data_runtime *pp_runtime_get_ex_data_rt(struct pp_runtime *pp_rt);
/* ip runtime scan API */
int pp_runtime_scan(struct pp_runtime *pp_rt, int thread_id, int port,
int proto, int *group_ids, size_t group_id_size,
int vtable_id, struct maat_state *state);
void pp_runtime_scan_hit_inc(struct pp_runtime *pp_rt, int thread_id);
long long pp_runtime_scan_hit_sum(struct pp_runtime *pp_rt, int n_thread);
#ifdef __cplusplus
}
#endif
#endif

View File

@@ -24,7 +24,6 @@ extern "C"
#include <sys/queue.h>
#include <openssl/md5.h>
#include "log/log.h"
#include "hiredis/hiredis.h"
#include "uthash/uthash.h"
#include "maat_command.h"
@@ -32,7 +31,6 @@ extern "C"
#include "maat.h"
#include "maat_kv.h"
#include "maat_table.h"
#include "maat_rule.h"
#include "maat_virtual.h"
#define MAX_TABLE_NUM 1024

View File

@@ -19,6 +19,7 @@ extern "C"
#include <stddef.h>
#include <cJSON/cJSON.h>
#include "log/log.h"
#include "maat_garbage_collection.h"
enum table_type {

View File

@@ -17,6 +17,7 @@ extern "C"
#endif
#include "cJSON/cJSON.h"
#include "maat_table.h"
enum scan_type {
SCAN_TYPE_INVALID = -1,
@@ -32,9 +33,6 @@ enum scan_type {
SCAN_TYPE_MAX
};
struct table_manager;
struct log_handle;
void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);

View File

@@ -46,9 +46,12 @@ enum district_set_flag {
};
struct maat_stream {
struct maat *maat_instance;
struct maat_runtime *ref_maat_rt;
struct maat *ref_maat_instance;
int thread_id;
int table_id;
int vtable_id;
int physical_table_ids[MAX_PHYSICAL_TABLE_NUM];
size_t n_physical_table;
};
enum scan_type maat_table_get_scan_type(enum table_type table_type)
@@ -1035,6 +1038,44 @@ int string_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, co
return sum_hit_group_cnt;
}
int stream_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, const char *data,
size_t data_len, int physical_table_ids[], int physical_table_cnt,
int vtable_id, struct maat_state *mid)
{
int sum_hit_group_cnt = 0;
for (size_t i = 0; i < physical_table_cnt; i++) {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]);
if ((table_type == TABLE_TYPE_EXPR_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
//maat_instance->scan_err_cnt++;
return -1;
}
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
continue;
}
void *expr_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]);
if (NULL == expr_rt) {
return -1;
}
int group_hit_cnt = expr_runtime_stream_scan((struct expr_runtime *)expr_rt,
data, data_len, vtable_id, mid);
if (group_hit_cnt < 0) {
return -1;
}
if (group_hit_cnt > 0) {
expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id);
}
sum_hit_group_cnt += group_hit_cnt;
}
return sum_hit_group_cnt;
}
size_t group_to_compile(struct maat *maat_instance, long long *results, size_t n_result,
struct maat_state *mid)
{
@@ -1447,20 +1488,126 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
return MAAT_SCAN_OK;
}
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id)
struct maat_stream *maat_scan_stream_open(struct maat *maat_instance, int table_id, int thread_id)
{
return NULL;
if (NULL == maat_instance || table_id < 0 || table_id > MAX_TABLE_NUM
|| thread_id < 0) {
return NULL;
}
struct maat_stream *stream = ALLOC(struct maat_stream, 1);
memset(stream->physical_table_ids, -1, sizeof(stream->physical_table_ids));
stream->ref_maat_instance = maat_instance;
stream->table_id = table_id;
stream->thread_id = thread_id;
stream->n_physical_table = vtable_get_physical_table_ids(stream->ref_maat_instance->tbl_mgr,
stream->table_id, stream->physical_table_ids,
MAX_PHYSICAL_TABLE_NUM, &stream->vtable_id);
if (stream->n_physical_table <= 0) {
return NULL;
}
enum table_type table_type = TABLE_TYPE_INVALID;
if (0 == stream->vtable_id) {
table_type = table_manager_get_table_type(stream->ref_maat_instance->tbl_mgr,
stream->physical_table_ids[0]);
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"scan stream's physical table must be expr or expr_plus");
return NULL;
}
}
for (size_t i = 0; i < stream->n_physical_table; i++) {
enum table_type table_type = table_manager_get_table_type(stream->ref_maat_instance->tbl_mgr,
stream->physical_table_ids[i]);
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"scan stream's physical table must be expr or expr_plus");
return NULL;
}
void *expr_rt = table_manager_get_runtime(stream->ref_maat_instance->tbl_mgr,
stream->physical_table_ids[i]);
if (NULL == expr_rt) {
return NULL;
}
expr_runtime_stream_open((struct expr_runtime *)expr_rt, thread_id);
}
return stream;
}
int maat_scan_stream(struct maat_stream **stream, const char *data, int data_len,
long long *results, size_t *n_result, struct maat_state **state)
int maat_scan_stream(struct maat_stream **maat_stream, const char *data, int data_len,
long long *results, size_t n_result, size_t *n_hit_result,
struct maat_state **state)
{
return 0;
if ((NULL == maat_stream) || (NULL == data) || (0 == data_len)
|| (NULL == results) || (0 == n_result) || (NULL == state)) {
return MAAT_SCAN_ERR;
}
struct maat_stream *stream = *maat_stream;
struct maat_state *mid = NULL;
mid = grab_state(state, stream->ref_maat_instance, stream->thread_id);
mid->scan_cnt++;
if (NULL == stream->ref_maat_instance->maat_rt) {
log_error(stream->ref_maat_instance->logger, MODULE_MAAT_API,
"maat_scan_string error because of maat_runtime is NULL");
return MAAT_SCAN_OK;
}
alignment_int64_array_add(stream->ref_maat_instance->thread_call_cnt, stream->thread_id, 1);
int hit_group_cnt = stream_scan_hit_group_count(stream->ref_maat_instance->tbl_mgr,
stream->thread_id, data, data_len,
stream->physical_table_ids,
stream->n_physical_table,
stream->vtable_id, mid);
if (hit_group_cnt < 0) {
return MAAT_SCAN_ERR;
}
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(mid)) {
// come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
sum_hit_compile_cnt = group_to_compile(stream->ref_maat_instance, results, n_result, mid);
*n_hit_result = sum_hit_compile_cnt;
}
if (sum_hit_compile_cnt > 0) {
alignment_int64_array_add(stream->ref_maat_instance->hit_cnt, stream->thread_id, 1);
if (0 == hit_group_cnt) {
//hit NOT group
alignment_int64_array_add(stream->ref_maat_instance->not_grp_hit_cnt, stream->thread_id, 1);
}
return MAAT_SCAN_HIT;
} else {
// n_hit_compile == 0
if (hit_group_cnt > 0) {
return MAAT_SCAN_HALF_HIT;
}
}
return sum_hit_compile_cnt;
}
void maat_scan_stream_close(struct maat_stream **stream)
void maat_scan_stream_close(struct maat_stream **maat_stream)
{
struct maat_stream *stream = *maat_stream;
for (size_t i = 0; i < stream->n_physical_table; i++) {
void *expr_rt = table_manager_get_runtime(stream->ref_maat_instance->tbl_mgr,
stream->physical_table_ids[i]);
assert(expr_rt != NULL);
expr_runtime_stream_close((struct expr_runtime *)expr_rt);
}
FREE(stream);
}
int maat_state_set_scan_district(struct maat *maat_instance,

View File

@@ -13,7 +13,6 @@
#include "maat_bool_plugin.h"
#include "bool_matcher.h"
#include "maat_ex_data.h"
#include "log/log.h"
#include "maat_utils.h"
#include "maat_rule.h"
#include "maat_garbage_collection.h"

View File

@@ -25,7 +25,7 @@ struct ex_data_runtime {
size_t cache_size;
struct rcu_hash_table *htable;
struct ex_data_schema *ex_schema;
struct ex_data_schema *ref_ex_schema;
int table_id;
struct log_handle *logger;
@@ -74,6 +74,8 @@ void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt)
void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt)
{
rcu_hash_commit(ex_data_rt->htable);
size_t count = rcu_hash_count(ex_data_rt->htable);
log_info(ex_data_rt->logger, MODULE_EX_DATA, "rcu_hash_count:%zu", count);
}
void ex_data_runtime_cache_row_put(struct ex_data_runtime *ex_data_rt, const char *row)
@@ -143,7 +145,7 @@ void ex_data_schema_free(struct ex_data_schema *ex_schema)
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt,
struct ex_data_schema *schema)
{
ex_data_rt->ex_schema = schema;
ex_data_rt->ref_ex_schema = schema;
}
void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt,
@@ -162,7 +164,7 @@ void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char
const char *key, size_t key_len)
{
void *ex_data = NULL;
struct ex_data_schema *ex_schema = ex_data_rt->ex_schema;
struct ex_data_schema *ex_schema = ex_data_rt->ref_ex_schema;
ex_schema->new_func(ex_data_rt->table_id, key, row, &ex_data,
ex_schema->argl, ex_schema->argp);
@@ -220,7 +222,11 @@ int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt,
}
rcu_hash_add(ex_data_rt->htable, key, key_len, ex_container);
tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
key, key_len);
log_info(ex_data_rt->logger, MODULE_EX_DATA,
"ex_data_runtime_add_ex_container rcu_hash_add ex_data_rt->htable:%p key:%p key_len:%zu, ex_container:%p tmp_container:%p",
ex_data_rt->htable, key, key_len, ex_container, tmp_container);
return 0;
}
@@ -254,10 +260,10 @@ void *ex_data_runtime_get_ex_data_by_key(struct ex_data_runtime *ex_data_rt,
}
void *dup_ex_data = NULL;
ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data,
ex_data_rt->ref_ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data,
&(ex_container->ex_data),
ex_data_rt->ex_schema->argl,
ex_data_rt->ex_schema->argp);
ex_data_rt->ref_ex_schema->argl,
ex_data_rt->ref_ex_schema->argp);
return dup_ex_data;
}
@@ -265,10 +271,10 @@ void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_r
struct ex_data_container *ex_container)
{
void *dup_ex_data = NULL;
ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data,
ex_data_rt->ref_ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data,
&(ex_container->ex_data),
ex_data_rt->ex_schema->argl,
ex_data_rt->ex_schema->argp);
ex_data_rt->ref_ex_schema->argl,
ex_data_rt->ref_ex_schema->argp);
return dup_ex_data;
}

View File

@@ -34,10 +34,10 @@ struct flag_schema {
};
struct flag_item {
long long item_id;
long long group_id;
long long flag;
long long flag_mask;
uint64_t item_id;
uint64_t group_id;
uint64_t flag;
uint64_t flag_mask;
};
struct flag_runtime {
@@ -176,7 +176,7 @@ void flag_runtime_free(void *flag_runtime)
}
int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_len,
long long item_id, struct flag_rule *rule, int is_valid)
uint64_t item_id, struct flag_rule *rule, int is_valid)
{
void *data = NULL;
@@ -301,7 +301,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
return -1;
} else if (0 == is_valid) {
//delete
HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(long long), item);
HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(uint64_t), item);
if (NULL == item) {
return -1;
}
@@ -317,7 +317,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
maat_garbage_bagging(flag_rt->ref_garbage_bin, u_para, maat_item_inner_free);
} else {
//add
HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(long long), item);
HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(uint64_t), item);
if (item) {
log_error(flag_rt->logger, MODULE_FLAG,
"flag runtime add item %d to item_hash failed, already exist",
@@ -332,7 +332,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
u_para = maat_item_inner_new(flag_item->group_id, item_id, 0);
item = maat_item_new(item_id, flag_item->group_id, u_para);
HASH_ADD(hh, flag_rt->item_hash, item_id, sizeof(long long), item);
HASH_ADD(hh, flag_rt->item_hash, item_id, sizeof(uint64_t), item);
flag_rule = flag_item_to_flag_rule(flag_item, u_para);
flag_item_free(flag_item);
@@ -345,7 +345,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
}
char *key = (char *)&item_id;
int ret = flag_runtime_update_row(flag_rt, key, sizeof(long long), item_id, flag_rule, is_valid);
int ret = flag_runtime_update_row(flag_rt, key, sizeof(uint64_t), item_id, flag_rule, is_valid);
if (ret < 0) {
if (flag_rule != NULL) {
flag_rule_free(flag_rule);
@@ -457,4 +457,4 @@ void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id)
long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread)
{
return alignment_int64_array_sum(flag_rt->hit_cnt, n_thread);
}
}

View File

@@ -13,7 +13,6 @@
#include "maat_fqdn_plugin.h"
#include "maat_ex_data.h"
#include "fqdn_engine.h"
#include "log/log.h"
#include "maat_utils.h"
#include "maat_table.h"
#include "maat_rule.h"

View File

@@ -10,16 +10,14 @@
#include <assert.h>
#include "maat_utils.h"
#include "log/log.h"
#include "maat_interval.h"
#include "maat_rule.h"
#include "maat_utils.h"
#include "rcu_hash.h"
#include "alignment.h"
#include "uthash/uthash.h"
#include "maat_garbage_collection.h"
#include "maat_compile.h"
#include "interval_matcher.h"
#include "maat_interval.h"
#define MODULE_INTERVAL module_name_str("maat.interval")

View File

@@ -268,26 +268,26 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt,
int ret = -1;
struct ex_data_schema *ex_schema = plugin_schema->ex_schema;
char hash_key[key_len + 1];
memset(hash_key, 0, sizeof(hash_key));
memcpy(hash_key, key, key_len);
/* already set plugin_table_schema's ex_data_schema */
if (ex_schema != NULL) {
if (is_valid == 0) {
// delete
ret = ex_data_runtime_del_ex_container(plugin_rt->ex_data_rt, hash_key, key_len);
ret = ex_data_runtime_del_ex_container(plugin_rt->ex_data_rt, key, key_len);
if (ret < 0) {
return -1;
}
} else {
// add
void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, hash_key, key_len);
void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL);
ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, hash_key, key_len, ex_container);
ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container);
if (ret < 0) {
return -1;
}
plugin_rt->acc_line_num++;
log_info(plugin_rt->logger, MODULE_PLUGIN,
"<plugin_runtime_update> add ex_data_runtime, key:%s key_len:%zu line:%s",
key, key_len, row);
}
}
@@ -301,6 +301,8 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt,
if ((NULL == ex_schema) && (0 == cb_count)) {
ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row);
log_info(plugin_rt->logger, MODULE_PLUGIN,
"<plugin_runtime_update> cache_row_put, line:%s", row);
}
return 0;
@@ -378,7 +380,6 @@ int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
schema->update_err_cnt++;
return -1;
}
plugin_rt->acc_line_num++;
return 0;
}
@@ -390,8 +391,17 @@ int plugin_runtime_commit(void *plugin_runtime, const char *table_name)
}
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
ex_data_runtime_commit(plugin_rt->ex_data_rt);
struct ex_data_container **ex_container = NULL;
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(plugin_rt->ex_data_rt,
&ex_container);
if (rule_cnt == 0) {
FREE(ex_container);
return 0;
}
ex_data_runtime_commit(plugin_rt->ex_data_rt);
return 0;
}
@@ -444,6 +454,6 @@ void *plugin_runtime_get_ex_data(void *plugin_runtime, void *plugin_schema, cons
if (NULL == schema->ex_schema) {
return NULL;
}
return ex_data_runtime_get_ex_data_by_key(plugin_rt->ex_data_rt, key, strlen(key));
}

View File

@@ -1,6 +1,6 @@
/*
**********************************************************************************************
* File: maat_ip.cpp
* File: maat_port.c
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
@@ -8,62 +8,61 @@
***********************************************************************************************
*/
#include "maat_port_proto.h"
#include "log/log.h"
#include "maat_port.h"
void *pp_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void *port_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
return NULL;
}
void pp_schema_free(void *pp_schema)
void port_schema_free(void *port_schema)
{
}
/* ip runtime API */
void *pp_runtime_new(void *pp_schema, int max_thread_num,
void *port_runtime_new(void *port_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
return NULL;
}
void pp_runtime_free(void *pp_runtime)
void port_runtime_free(void *port_runtime)
{
}
int pp_runtime_update(void *pp_runtime, void *pp_schema,
int port_runtime_update(void *port_runtime, void *port_schema,
const char *line, int valid_column)
{
return 0;
}
int pp_runtime_commit(void *pp_runtime, const char *table_name)
int port_runtime_commit(void *port_runtime, const char *table_name)
{
return 0;
}
struct ex_data_runtime *pp_runtime_get_ex_data_rt(struct pp_runtime *pp_rt)
struct ex_data_runtime *port_runtime_get_ex_data_rt(struct port_runtime *port_rt)
{
return NULL;
}
int pp_runtime_scan(struct pp_runtime *pp_rt, int thread_id, int port,
int port_runtime_scan(struct port_runtime *port_rt, int thread_id, int port,
int proto, int *group_ids, size_t group_id_size,
int vtable_id, struct maat_state *state)
{
return 0;
}
void pp_runtime_scan_hit_inc(struct pp_runtime *pp_rt, int thread_id)
void port_runtime_scan_hit_inc(struct port_runtime *port_rt, int thread_id)
{
}
long long pp_runtime_scan_hit_sum(struct pp_runtime *pp_rt, int n_thread)
long long port_runtime_scan_hit_sum(struct port_runtime *port_rt, int n_thread)
{
return 0;
}
}

View File

@@ -18,7 +18,6 @@
#include "json2iris.h"
#include "log/log.h"
#include "cJSON/cJSON.h"
#include "maat_utils.h"
#include "maat_rule.h"
#include "maat_config_monitor.h"
@@ -415,6 +414,7 @@ void *rule_monitor_loop(void *arg)
char err_str[NAME_MAX] = {0};
struct stat attrib;
while (maat_instance->is_running) {
log_info(maat_instance->logger, MODULE_MAAT_RULE, "rule_monitor_loop.............");
usleep(maat_instance->rule_update_checking_interval_ms * 1000);
if (0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
switch (maat_instance->input_mode) {

View File

@@ -18,6 +18,9 @@ target_link_libraries(maat_framework_gtest maat_frame_static gtest_static)
add_executable(adapter_hs_gtest adapter_hs_gtest.cpp)
target_link_libraries(adapter_hs_gtest maat_frame_static gtest_static)
add_executable(maat_ex_data_gtest maat_ex_data_gtest.cpp)
target_link_libraries(maat_ex_data_gtest maat_frame_static gtest_static)
file(COPY table_info.conf DESTINATION ./)
file(COPY literal_expr.conf DESTINATION ./)
file(COPY regex_expr.conf DESTINATION ./)

137
test/maat_ex_data_gtest.cpp Normal file
View File

@@ -0,0 +1,137 @@
#include "maat.h"
#include "log/log.h"
#include "maat_utils.h"
#include "json2iris.h"
#include "maat_command.h"
#include "maat_ex_data.h"
#include <gtest/gtest.h>
#include <limits.h>
const char *table_info_path = "./table_info.conf";
const char *json_filename = "maat_json.json";
struct log_handle *g_logger = NULL;
struct maat *g_maat_instance = NULL;
struct user_info {
char name[256];
char ip_addr[32];
int id;
int ref_cnt;
};
void ex_data_new_cb(int table_id, const char *key, const char *table_line,
void **ad, long argl, void *argp)
{
int *counter = (int *)argp;
struct user_info *u = ALLOC(struct user_info, 1);
int valid = 0, tag = 0;
int ret = sscanf(table_line, "%d\t%s\t%s%d\t%d", &(u->id), u->ip_addr, u->name, &valid, &tag);
EXPECT_EQ(ret, 5);
u->ref_cnt = 1;
*ad = u;
(*counter)++;
}
void ex_data_free_cb(int table_id, void **ad, long argl, void *argp)
{
struct user_info *u = (struct user_info *)(*ad);
if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0)) {
free(u);
*ad = NULL;
}
}
void ex_data_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
{
struct user_info *u = (struct user_info *)(*from);
__sync_add_and_fetch(&(u->ref_cnt), 1);
*to = u;
}
TEST(EXDataRuntime, Update) {
const char *table_name = "TEST_PLUGIN_EXDATA_TABLE";
int table_id = maat_table_get_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
int ex_data_counter = 0;
struct ex_data_runtime *ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free, g_logger);
struct ex_data_schema *ex_schema = ex_data_schema_new(ex_data_new_cb, ex_data_free_cb, ex_data_dup_cb,
0, &ex_data_counter);
ex_data_runtime_set_schema(ex_data_rt, ex_schema);
const char *row1 = "1\t192.168.0.1\tmahuateng\t1\t0";
const char *key1 = "192.168.0.1";
size_t key1_len = strlen(key1);
void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row1, key1, key1_len);
EXPECT_EQ(ex_data_counter, 1);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL);
int ret = ex_data_runtime_add_ex_container(ex_data_rt, key1, key1_len, ex_container);
EXPECT_EQ(ret, 0);
const char *row2 = "2\t192.168.0.2\tliyanhong\t1\t0";
const char *key2 = "192.168.0.2";
size_t key2_len = strlen(key2);
ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row2, key2, key2_len);
ex_container = ex_data_container_new(ex_data, NULL);
ret = ex_data_runtime_add_ex_container(ex_data_rt, key2, key2_len, ex_container);
EXPECT_EQ(ret, 0);
ex_data_runtime_commit(ex_data_rt);
void *res_data1 = ex_data_runtime_get_ex_data_by_key(ex_data_rt, "192.168.0.1", 11);
EXPECT_TRUE(res_data1 != NULL);
struct user_info *info = (struct user_info *)res_data1;
EXPECT_EQ(0, strcmp(info->name, "mahuateng"));
EXPECT_EQ(info->id, 1);
void *res_data2 = ex_data_runtime_get_ex_data_by_key(ex_data_rt, "192.168.0.2", 11);
EXPECT_TRUE(res_data2 != NULL);
info = (struct user_info *)res_data2;
EXPECT_EQ(0, strcmp(info->name, "liyanhong"));
EXPECT_EQ(info->id, 2);
}
int main(int argc, char ** argv)
{
int ret=0;
::testing::InitGoogleTest(&argc, argv);
g_logger = log_handle_create("./maat_ex_data_gtest.log", 0);
char json_iris_path[NAME_MAX] = {0};
char tmp_iris_path[PATH_MAX] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename);
if ((access(json_iris_path, F_OK)) == 0) {
system_cmd_rmdir(json_iris_path);
}
if (access(json_iris_path, F_OK) < 0) {
char *json_buff = NULL;
size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_filename, (unsigned char**)&json_buff, &json_buff_sz);
EXPECT_NE(ret, -1);
ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path),
NULL, NULL, g_logger);
EXPECT_NE(ret, -1);
}
struct maat_options *opts = maat_options_new();
char json_path[PATH_MAX] = {0};
snprintf(json_path, sizeof(json_path), "./%s", json_filename);
maat_options_set_json_file(opts, json_path);
g_maat_instance = maat_new(opts, table_info_path);
EXPECT_TRUE(g_maat_instance != NULL);
ret=RUN_ALL_TESTS();
log_handle_destroy(g_logger);
return ret;
}

View File

@@ -260,7 +260,6 @@ TEST_F(MaatStringScan, ExprAndExprPlus) {
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 195);
maat_state_free(&state);
}
//TODO:
@@ -866,6 +865,199 @@ TEST_F(IPPluginTable, EX_DATA) {
EXPECT_EQ(ret, 0);
}
class FQDNPluginTable : public testing::Test
{
protected:
static void SetUpTestCase() {
}
static void TearDownTestCase() {
}
};
#define FQDN_PLUGIN_EX_DATA
struct fqdn_plugin_ud
{
int rule_id;
int catid;
int ref_cnt;
};
void fqdn_plugin_ex_new_cb(int table_id, const char *key, const char *table_line, void **ad, long argl, void *argp)
{
int *counter = (int *)argp;
size_t column_offset = 0, column_len = 0;
struct fqdn_plugin_ud *ud = ALLOC(struct fqdn_plugin_ud, 1);
int ret = get_column_pos(table_line, 1, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->rule_id = atoi(table_line + column_offset);
ret = get_column_pos(table_line, 4, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
sscanf(table_line + column_offset, "catid=%d", &ud->catid);
ud->ref_cnt = 1;
*ad = ud;
(*counter)++;
}
void fqdn_plugin_ex_free_cb(int table_id, void **ad, long argl, void *argp)
{
struct fqdn_plugin_ud *u = (struct fqdn_plugin_ud *)(*ad);
if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0)) {
free(u);
*ad = NULL;
}
}
void fqdn_plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
{
struct fqdn_plugin_ud *u = (struct fqdn_plugin_ud *)(*from);
__sync_add_and_fetch(&(u->ref_cnt), 1);
*to = u;
}
TEST_F(FQDNPluginTable, EX_DATA) {
const char *table_name = "TEST_FQDN_PLUGIN_WITH_EXDATA";
int table_id = maat_table_get_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
int fqdn_plugin_ex_data_counter = 0;
int ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_id,
fqdn_plugin_ex_new_cb,
fqdn_plugin_ex_free_cb,
fqdn_plugin_ex_dup_cb,
0, &fqdn_plugin_ex_data_counter);
ASSERT_TRUE(ret>=0);
EXPECT_EQ(fqdn_plugin_ex_data_counter, 5);
int i = 0;
struct fqdn_plugin_ud *result[4];
ret = maat_fqdn_plugin_table_get_ex_data(g_maat_instance, table_id, "www.example1.com", (void**)result, 4);
ASSERT_EQ(ret, 2);
EXPECT_EQ(result[0]->rule_id, 201);
EXPECT_EQ(result[1]->rule_id, 202);
for (i = 0; i < ret; i++) {
fqdn_plugin_ex_free_cb(0, (void**)&(result[i]), 0, NULL);
}
ret = maat_fqdn_plugin_table_get_ex_data(g_maat_instance, table_id, "www.example3.com", (void**)result, 4);
EXPECT_EQ(ret, 0);
ret = maat_fqdn_plugin_table_get_ex_data(g_maat_instance, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4);
ASSERT_EQ(ret, 2);
EXPECT_TRUE(result[0]->rule_id == 205 || result[0]->rule_id == 204);
for (i = 0; i < ret; i++) {
fqdn_plugin_ex_free_cb(0, (void**)&(result[i]), 0, NULL);
}
}
struct bool_plugin_ud {
int id;
char *name;
int ref_cnt;
};
void bool_plugin_ex_new_cb(int table_id, const char *key, const char *table_line, void **ad, long argl, void *argp)
{
int *counter=(int *)argp;
size_t column_offset=0, column_len=0;
struct bool_plugin_ud *ud = ALLOC(struct bool_plugin_ud, 1);
int ret = get_column_pos(table_line, 1, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->id = atoi(table_line + column_offset);
ret = get_column_pos(table_line, 3, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->name = (char *)malloc(column_len+1);
memcpy(ud->name, table_line+column_offset, column_len);
ud->ref_cnt = 1;
*ad = ud;
(*counter)++;
}
void bool_plugin_ex_free_cb(int table_id, void **ad, long argl, void *argp)
{
struct bool_plugin_ud *u = (struct bool_plugin_ud *)(*ad);
if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0))
{
free(u->name);
free(u);
*ad = NULL;
}
}
void bool_plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
{
struct bool_plugin_ud *u = (struct bool_plugin_ud *)(*from);
__sync_add_and_fetch(&(u->ref_cnt), 1);
*to = u;
}
class BoolPluginTable : public testing::Test
{
protected:
static void SetUpTestCase() {
}
static void TearDownTestCase() {
}
};
TEST_F(BoolPluginTable, EX_DATA) {
int ex_data_counter = 0, i = 0;
const char *table_name = "TEST_BOOL_PLUGIN_WITH_EXDATA";
int table_id = maat_table_get_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
int ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_id,
bool_plugin_ex_new_cb,
bool_plugin_ex_free_cb,
bool_plugin_ex_dup_cb,
0, &ex_data_counter);
ASSERT_TRUE(ret >= 0);
EXPECT_EQ(ex_data_counter, 6);
struct bool_plugin_ud *result[6];
unsigned long long items_1[] = {999};
ret = maat_bool_plugin_table_get_ex_data(g_maat_instance, table_id, items_1, 1, (void**)result, 6);
EXPECT_EQ(ret, 0);
for (i = 0; i < ret; i++) {
bool_plugin_ex_free_cb(0, (void**)&(result[i]), 0, NULL);
}
unsigned long long items_2[] = {1, 2, 1000};
ret = maat_bool_plugin_table_get_ex_data(g_maat_instance, table_id, items_2, 3, (void**)result, 6);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0]->id, 301);
for (i = 0; i < ret; i++) {
bool_plugin_ex_free_cb(0, (void**)&(result[i]), 0, NULL);
}
unsigned long long items_3[]={101, 102, 1000};
ret = maat_bool_plugin_table_get_ex_data(g_maat_instance, table_id, items_3, 3, (void**)result, 6);
EXPECT_EQ(ret, 4);
for (i = 0; i < ret; i++) {
bool_plugin_ex_free_cb(0, (void**)&(result[i]), 0, NULL);
}
unsigned long long items_4[]={7, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7};
ret = maat_bool_plugin_table_get_ex_data(g_maat_instance, table_id, items_4, sizeof(items_4)/sizeof(unsigned long long), (void**)result, 6);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0]->id, 305);
for (i = 0; i < ret; i++) {
bool_plugin_ex_free_cb(0, (void**)&(result[i]), 0, NULL);
}
}
class VirtualTable : public testing::Test
{
protected:
@@ -1084,6 +1276,7 @@ void plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void *arg
__sync_add_and_fetch(&(u->ref_cnt), 1);
*to = u;
}
#if 0
TEST_F(MaatCmdTest, PluginEXData) {
const char *table_name = "TEST_PLUGIN_EXDATA_TABLE";
@@ -1135,8 +1328,8 @@ TEST_F(MaatCmdTest, PluginEXData) {
ret = maat_cmd_set_line(g_maat_instance, &line_rule);
EXPECT_GT(ret, 0);
sleep(1);
sleep(1);
int ex_data_counter = 0;
ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_id,
plugin_ex_new_cb,
@@ -1147,7 +1340,7 @@ TEST_F(MaatCmdTest, PluginEXData) {
EXPECT_EQ(ex_data_counter, TEST_CMD_LINE_NUM);
struct user_info *uinfo = NULL;
uinfo = (struct user_info *)maat_plugin_table_get_ex_data(g_maat_instance, table_id, "192.168.0.2");
uinfo = (struct user_info *)maat_plugin_table_get_ex_data(g_maat_instance, table_id, "192.168.0.4");
ASSERT_TRUE(uinfo != NULL);
EXPECT_EQ(0, strcmp(uinfo->name, "liuqiangdong"));
EXPECT_EQ(uinfo->id, 2);
@@ -1161,6 +1354,7 @@ TEST_F(MaatCmdTest, PluginEXData) {
ASSERT_TRUE(uinfo == NULL);
}
#endif
int count_line_num_cb(const char *table_name, const char *line, void *u_para)
{
(*((unsigned int *)u_para))++;

View File

@@ -90,6 +90,22 @@ TEST(rcu_hash_add_multi_node, single_thread) {
size_t key1_len = strlen(key1);
rcu_hash_add(htable, key1, key1_len, (void *)data1);
struct user_data *data2 = ALLOC(struct user_data, 1);
data2->id = 103;
char name2[64] = "mahuateng";
memcpy(data2->name, name2, strlen(name2));
char key2[64] = "192.168.0.1";
size_t key2_len = strlen(key2);
rcu_hash_add(htable, key2, key2_len, (void *)data2);
struct user_data *data3 = ALLOC(struct user_data, 1);
data3->id = 104;
char name3[64] = "liyanhong";
memcpy(data3->name, name3, strlen(name3));
char key3[64] = "192.168.0.2";
size_t key3_len = strlen(key3);
rcu_hash_add(htable, key3, key3_len, (void *)data3);
/* find in hash before commit */
void *res = rcu_hash_find(htable, key0, key0_len);
EXPECT_TRUE(res == NULL);
@@ -104,27 +120,37 @@ TEST(rcu_hash_add_multi_node, single_thread) {
void **data_array = NULL;
ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 2);
EXPECT_EQ(ret, 4);
rcu_hash_commit(htable);
/* find in hash after commit */
res = rcu_hash_find(htable, key0, key0_len);
EXPECT_TRUE(res != NULL);
struct user_data *res_data0 = (struct user_data *)res;
EXPECT_EQ(res_data0->id, 101);
EXPECT_STREQ(res_data0->name, "www.baidu.com");
res = rcu_hash_find(htable, key1, key1_len);
EXPECT_TRUE(res != NULL);
struct user_data *res_data1 = (struct user_data *)res;
EXPECT_EQ(res_data1->id, 102);
EXPECT_STREQ(res_data1->name, "127.0.0.1");
res = rcu_hash_find(htable, key2, key2_len);
EXPECT_TRUE(res != NULL);
struct user_data *res_data2 = (struct user_data *)res;
EXPECT_EQ(res_data2->id, 103);
EXPECT_STREQ(res_data2->name, "mahuateng");
res = rcu_hash_find(htable, key3, key3_len);
EXPECT_TRUE(res != NULL);
struct user_data *res_data3 = (struct user_data *)res;
EXPECT_EQ(res_data3->id, 104);
EXPECT_STREQ(res_data3->name, "liyanhong");
ret = rcu_hash_count(htable);
EXPECT_EQ(ret, 2);
EXPECT_EQ(ret, 4);
ret = rcu_hash_garbage_queue_len(htable);
EXPECT_EQ(ret, 0);